From 1eeddde6257b8d8156c5a4e2fd94548245e63b29 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 8 Apr 2016 16:55:17 +0200 Subject: [PATCH 01/26] clock() is default timer for all platforms except Windows --- programs/bench.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index cea9634c2..d6ee3d42a 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -70,24 +70,18 @@ # define SET_HIGH_PRIORITY /* disabled */ #endif -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) +#if !defined(_WIN32) typedef clock_t BMK_time_t; # define BMK_initTimer(ticksPerSecond) ticksPerSecond=0 # define BMK_getTime(x) x = clock() # define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) # define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) -#elif defined(_WIN32) +#else typedef LARGE_INTEGER BMK_time_t; # define BMK_initTimer(x) if (!QueryPerformanceFrequency(&x)) { fprintf(stderr, "ERROR: QueryPerformance not present\n"); } # define BMK_getTime(x) QueryPerformanceCounter(&x) # define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) # define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) -#else - typedef int BMK_time_t; -# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0 -# define BMK_getTimeMicro(clockStart) clockStart=1 -# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000ULL+clockEnd-clockStart) -# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000000ULL+clockEnd-clockStart) #endif #include "mem.h" From f7d210b2e9c7ef0b43b681e2f12f6e4f73c38c98 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 11 Apr 2016 16:35:13 +0200 Subject: [PATCH 02/26] cache literal prices for ZSTD_btopt --- lib/zstd_internal.h | 3 +++ lib/zstd_opt.h | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 31a99cbde..9c888e779 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -242,6 +242,9 @@ typedef struct { U32 log2litSum; U32 log2offCodeSum; U32 factor; + U32 cachedPrice; + U32 cachedLitLength; + const BYTE* cachedLiterals; ZSTD_stats_t stats; } seqStore_t; diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 4f6e4c8e9..efda97e52 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -54,6 +54,8 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) unsigned u; if (ssPtr->litLengthSum == 0) { + ssPtr->cachedLiterals = NULL; + ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; ssPtr->litSum = (2<litLengthSum = MaxLL+1; ssPtr->matchLengthSum = MaxML+1; @@ -98,17 +100,41 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) } -FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals) +FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals) { U32 price, u; if (litLength == 0) - return seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[0]+1); + return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1); /* literals */ - price = litLength * seqStorePtr->log2litSum; +#define ZSTD_CACHE_LITPRICES +#ifdef ZSTD_CACHE_LITPRICES + if (ssPtr->cachedLiterals == literals) { + // if (ssPtr->cachedLitLength > litLength) printf("ERROR: ssPtr->cachedLitLength > litLength\n"); + U32 additional = litLength - ssPtr->cachedLitLength; + const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; + price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; + for (u=0; u < additional; u++) + price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1); + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } else { + price = litLength * ssPtr->log2litSum; + for (u=0; u < litLength; u++) + price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); + + if (litLength >= 12) { + ssPtr->cachedLiterals = literals; + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } + } +#else + price = litLength * ssPtr->log2litSum; for (u=0; u < litLength; u++) - price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1); + price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); +#endif /* literal Length */ { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, @@ -121,7 +147,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co 24, 24, 24, 24, 24, 24, 24, 24 }; const BYTE LL_deltaCode = 19; const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; - price += LL_bits[llCode] + seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[llCode]+1); + price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1); } return price; @@ -465,7 +491,11 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; inr = ip; +#ifdef ZSTD_CACHE_LITPRICES + litstart = anchor; +#else litstart = ((U32)(ip - anchor) > 128) ? ip - 128 : anchor; +#endif opt[0].litlen = (U32)(ip - litstart); /* check repCode */ From c0d5f4eb2eef308732ee16090f338e4ad0850c9f Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 13 Apr 2016 10:48:04 +0200 Subject: [PATCH 03/26] bench.c: ignore directories from a file list for benchmark --- programs/bench.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/programs/bench.c b/programs/bench.c index fcc0934ae..bb03c0b35 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -500,10 +500,21 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, size_t* fileSizes, const char** fileNamesTable, unsigned nbFiles) { - size_t pos = 0; + size_t pos = 0, totalSize = 0; unsigned n; for (n=0; n Date: Thu, 14 Apr 2016 13:43:51 +0200 Subject: [PATCH 04/26] removed ZSTD_compressBegin_targetSrcSize --- lib/zstd_compress.c | 11 ----------- lib/zstd_internal.h | 1 - programs/bench.c | 32 +++++++++++++++++++++----------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 8ce18f811..86d83ee6d 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -2309,17 +2309,6 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict } -size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel) -{ - ZSTD_parameters params; - params.cParams = ZSTD_getCParams(compressionLevel, targetSrcSize, dictSize); - params.fParams.contentSizeFlag = 1; - ZSTD_adjustCParams(¶ms.cParams, targetSrcSize, dictSize); - ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel); - return ZSTD_compressBegin_internal(zc, dict, dictSize, params, targetSrcSize); -} - - size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel); diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index a06e79f2f..d5cc255a9 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -250,7 +250,6 @@ typedef struct { const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); -size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/programs/bench.c b/programs/bench.c index bb03c0b35..4da51f637 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -86,7 +86,6 @@ #include "mem.h" #include "zstd_static.h" -#include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */ #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" @@ -204,6 +203,19 @@ static U64 BMK_getFileSize(const char* infilename) return (U64)statbuf.st_size; } +static U32 BMK_isDirectory(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (!r && S_ISDIR(statbuf.st_mode)) return 1; + return 0; +} /* ******************************************************** * Bench functions @@ -317,8 +329,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { U32 nbLoops; for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) { U32 blockNb; - ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel); - // ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); + { ZSTD_parameters params; + params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); + params.fParams.contentSizeFlag = 1; + ZSTD_adjustCParams(¶ms.cParams, blockSize, dictBufferSize); + { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); + if (ZSTD_isError(initResult)) break; + } } for (blockNb=0; blockNb Date: Fri, 15 Apr 2016 13:44:46 +0200 Subject: [PATCH 05/26] introduced ZSTD_DEAFULT_CLEVEL for (compressionLevel<=0) --- lib/zstd_compress.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 86d83ee6d..463ae11ae 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -2431,7 +2431,8 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS /*-===== Pre-defined compression levels =====-*/ -#define ZSTD_MAX_CLEVEL 22 +#define ZSTD_DEAFULT_CLEVEL 5 +#define ZSTD_MAX_CLEVEL 22 unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { @@ -2550,7 +2551,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si size_t const addedSize = srcSize ? 0 : 500; U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ - if (compressionLevel<=0) compressionLevel = 1; + if (compressionLevel < 0) compressionLevel = ZSTD_DEAFULT_CLEVEL; + if (compressionLevel==0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; cp = ZSTD_defaultCParameters[tableID][compressionLevel]; if (MEM_32bits()) { /* auto-correction, for 32-bits mode */ From 6d157f1fbe40d9294253e117d56fab9cd09fa0a3 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 15 Apr 2016 16:54:11 +0200 Subject: [PATCH 06/26] bench.c: fixed rare compression and decompression speed bug concerns only big files with compression or decompression time longer than 100 seconds --- programs/bench.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 4da51f637..de9e16e95 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -295,7 +295,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); /* Bench */ - { double fastestC = 100000000., fastestD = 100000000.; + { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); U64 crcCheck = 0; BMK_time_t coolTime; @@ -344,7 +344,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].cSize = rSize; } } { U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond); - if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops; + if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops; } } cSize = 0; @@ -382,7 +382,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].resSize = regenSize; } } { U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond); - if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops; + if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops; } } DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", From c5e1d295eeec565cdac42f69666bdde1e8ea02d2 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 19 Apr 2016 09:37:59 +0200 Subject: [PATCH 07/26] bench.c: force at least one compression and decompression loop fix for -i0 with small files --- .gitignore | 1 + lib/zstd_opt.h | 5 +++-- programs/bench.c | 18 +++++++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 6ca7f8a7e..8af4bbe96 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ ipch/ .directory _codelite _zstdbench +zlib_wrapper diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 553465e06..f584cf430 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -53,9 +53,10 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) { unsigned u; + ssPtr->cachedLiterals = NULL; + ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; + if (ssPtr->litLengthSum == 0) { - ssPtr->cachedLiterals = NULL; - ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; ssPtr->litSum = (2<litLengthSum = MaxLL+1; ssPtr->matchLengthSum = MaxML+1; diff --git a/programs/bench.c b/programs/bench.c index de9e16e95..2892ef742 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -307,7 +307,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, DISPLAYLEVEL(2, "\r%79s\r", ""); for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) { BMK_time_t clockStart, clockEnd; - U64 clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10; + U64 clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 1; /* overheat protection */ if (BMK_clockSpan(coolTime, ticksPerSecond) > ACTIVEPERIOD_S*1000000ULL) { @@ -326,8 +326,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); BMK_getTime(clockStart); - { U32 nbLoops; - for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) { + { U32 nbLoops = 0; + do { U32 blockNb; { ZSTD_parameters params; params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); @@ -342,7 +342,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize); if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize)); blockTable[blockNb].cSize = rSize; - } } + } + nbLoops++; + } while (BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop); { U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond); if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops; } } @@ -365,8 +367,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); BMK_getTime(clockStart); - { U32 nbLoops; - for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) { + { U32 nbLoops = 0; + do { U32 blockNb; ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize); for (blockNb=0; blockNb Date: Thu, 21 Apr 2016 11:08:43 +0200 Subject: [PATCH 08/26] zst_opt.h: minor compression speed improvement --- lib/zstd_compress.c | 19 +++++----- lib/zstd_opt.h | 87 ++++++++++++++++----------------------------- 2 files changed, 41 insertions(+), 65 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 463ae11ae..6ca1bafa3 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1619,9 +1619,6 @@ FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( } } -/* The optimal parser */ -#include "zstd_opt.h" - /* ******************************* * Common parser - lazy strategy @@ -1756,12 +1753,6 @@ _storeSequence: } - -static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) -{ - ZSTD_compressBlock_opt_generic(ctx, src, srcSize); -} - static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); @@ -1975,6 +1966,16 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); } + + +/* The optimal parser */ +#include "zstd_opt.h" + +static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_opt_generic(ctx, src, srcSize); +} + static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize); diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index f584cf430..122818523 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -109,10 +109,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1); /* literals */ -#define ZSTD_CACHE_LITPRICES -#ifdef ZSTD_CACHE_LITPRICES if (ssPtr->cachedLiterals == literals) { - // if (ssPtr->cachedLitLength > litLength) printf("ERROR: ssPtr->cachedLitLength > litLength\n"); U32 additional = litLength - ssPtr->cachedLitLength; const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; @@ -131,11 +128,6 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY ssPtr->cachedLitLength = litLength; } } -#else - price = litLength * ssPtr->log2litSum; - for (u=0; u < litLength; u++) - price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); -#endif /* literal Length */ { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, @@ -455,7 +447,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const BYTE* litstart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; const BYTE* const base = ctx->base; @@ -469,7 +460,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; const BYTE* inr; - U32 cur, match_num, last_pos, litlen, price; /* init */ U32 rep[ZSTD_REP_INIT]; @@ -484,36 +474,25 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* Match Loop */ while (ip < ilimit) { - U32 u; - U32 mlen=0; - U32 best_mlen=0; - U32 best_off=0; + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off; memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; - inr = ip; -#ifdef ZSTD_CACHE_LITPRICES - litstart = anchor; -#else - litstart = ((U32)(ip - anchor) > 128) ? ip - 128 : anchor; -#endif - opt[0].litlen = (U32)(ip - litstart); + litlen = (U32)(ip - anchor); /* check repCode */ { U32 i; for (i=0; i sufficient_len || mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; goto _storeSequence; } - best_off = (i<=1 && ip == anchor) ? 1-i : i; - litlen = opt[0].litlen; do { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ mlen--; @@ -525,10 +504,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } - { U32 i ; for (i=0; i sufficient_len) { + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; cur = 0; @@ -536,23 +512,26 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, goto _storeSequence; } - best_mlen = (last_pos) ? last_pos : minMatch; - - // set prices using matches at position = 0 - for (u = 0; u < match_num; u++) { + /* set prices using matches at position = 0 */ + best_mlen = (last_pos) ? last_pos : minMatch; + for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM; + best_mlen = matches[u].len; ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); - litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ mlen++; } } if (last_pos < minMatch) { ip++; continue; } + /* initialize opt[0] */ + { U32 i ; for (i=0; i litlen) { price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, litstart); + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); } else { litlen = 1; price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); @@ -590,8 +569,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); - best_mlen = 0; + best_mlen = minMatch; { U32 i; for (i=0; i litlen) { price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); } else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); } else { litlen = 0; price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); } - best_mlen = mlen; + if (mlen > best_mlen) best_mlen = mlen; ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); do { @@ -629,19 +608,17 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); - if (match_num > 0 && matches[match_num-1].len > sufficient_len) { + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; last_pos = cur + 1; goto _storeSequence; } - best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; - /* set prices using matches at position = cur */ for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; + best_mlen = matches[u].len; // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); while (mlen <= best_mlen) { @@ -650,7 +627,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, if (cur > litlen) price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); } else { litlen = 0; price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); @@ -754,7 +731,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const BYTE* litstart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; const BYTE* const base = ctx->base; @@ -794,8 +770,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; inr = ip; - litstart = ((U32)(ip - anchor) > 128) ? ip - 128 : anchor; - opt[0].litlen = (U32)(ip - litstart); + opt[0].litlen = (U32)(ip - anchor); /* check repCode */ { U32 i; for (i=0; i last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ mlen--; @@ -832,7 +807,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, { U32 i; for (i=0; i sufficient_len) { + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; cur = 0; @@ -845,11 +820,11 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, // set prices using matches at position = 0 for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM; + best_mlen = matches[u].len; ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; @@ -869,7 +844,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, if (cur > litlen) { price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, litstart); + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); } else { litlen = 1; price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); @@ -922,7 +897,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, if (cur > litlen) { price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); } else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); } else { litlen = 0; price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); @@ -962,7 +937,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, if (cur > litlen) price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); } else { litlen = 0; price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); From 38654988f3d568df7e07700622da2397de314b3d Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 21 Apr 2016 12:18:47 +0200 Subject: [PATCH 09/26] minor speed improvements 2 bench.c: block size has to be bigger than 32 bytes zstdcli.c: support for e.g. -B16k -B16m --- lib/zstd_opt.h | 41 ++++++++++++++++++++--------------------- programs/bench.c | 8 ++++---- programs/zstdcli.c | 10 +++++----- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 122818523..ae09a2693 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -260,7 +260,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, U32 nbCompares, const U32 mls, - U32 extDict, ZSTD_match_t* matches) + U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen) { const BYTE* const base = zc->base; const U32 current = (U32)(ip-base); @@ -285,7 +285,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( U32 mnum = 0; const U32 minMatch = (mls == 3) ? 3 : 4; - size_t bestLength = minMatch-1; + size_t bestLength = minMatchLen-1; if (minMatch == 3) { /* HC3 match finder */ U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); @@ -385,26 +385,26 @@ update: static U32 ZSTD_BtGetAllMatches ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, - const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches) + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) { if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); } static U32 ZSTD_BtGetAllMatches_selectMLS ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* ip, const BYTE* const iHighLimit, - const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches) + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) { switch(matchLengthSearch) { - case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches); + case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); default : - case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches); - case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches); - case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches); + case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); } } @@ -412,26 +412,26 @@ static U32 ZSTD_BtGetAllMatches_selectMLS ( static U32 ZSTD_BtGetAllMatches_extDict ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, - const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches) + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) { if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); } static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* ip, const BYTE* const iHighLimit, - const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches) + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) { switch(matchLengthSearch) { - case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches); + case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); default : - case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches); - case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches); - case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches); + case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); } } @@ -499,7 +499,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } while (mlen >= minMatch); } } - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -604,8 +604,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } while (mlen >= minMatch); } } - - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches); + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { @@ -799,7 +798,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } while (mlen >= minMatch); } } } - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -913,7 +912,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } while (mlen >= minMatch); } } } - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches); + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && matches[match_num-1].len > sufficient_len) { diff --git a/programs/bench.c b/programs/bench.c index 2892ef742..8470ed324 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -248,7 +248,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) { - size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ + size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ @@ -488,9 +488,9 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, dictBuffer, dictBufferSize, &result); if (g_displayLevel == 1) { if (g_additionalParam) - DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam); + DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam); else - DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); total.cSize += result.cSize; total.cSpeed += result.cSpeed; total.dSpeed += result.dSpeed; @@ -501,7 +501,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, total.cSpeed /= 1+cLevelLast-cLevel; total.dSpeed /= 1+cLevelLast-cLevel; total.ratio /= 1+cLevelLast-cLevel; - DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); + DISPLAY("avg%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); } } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3019edcf2..5529cf8e1 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -41,6 +41,7 @@ #include /* fprintf, getchar */ #include /* exit, calloc, free */ #include /* strcmp, strlen */ +#include /* toupper */ #include "fileio.h" #ifndef ZSTD_NOBENCH # include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ @@ -304,9 +305,9 @@ int main(int argCount, const char** argv) argument++; while ((*argument >='0') && (*argument <='9')) bSize *= 10, bSize += *argument++ - '0'; - if (*argument=='K') bSize<<=10, argument++; /* allows using KB notation */ - if (*argument=='M') bSize<<=20, argument++; - if (*argument=='B') argument++; + if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ + if (toupper(*argument)=='M') bSize<<=20, argument++; + if (toupper(*argument)=='B') argument++; BMK_setNotificationLevel(displayLevel); BMK_SetBlockSize(bSize); } @@ -368,8 +369,7 @@ int main(int argCount, const char** argv) maxDictSize = 0; while ((*argument>='0') && (*argument<='9')) maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++; - if (*argument=='k' || *argument=='K') - maxDictSize <<= 10; + if (toupper(*argument)=='K') maxDictSize <<= 10; continue; } From 23a0889301632f0b278041f9aab0c9c85150273e Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 22 Apr 2016 12:43:18 +0200 Subject: [PATCH 10/26] separation of lib/ into common/, compress/, decompress/, dictBuilder/, legacy/ --- lib/Makefile | 13 +- lib/README.md | 6 +- lib/{ => common}/bitstream.h | 0 lib/{ => common}/error_private.h | 0 lib/{ => common}/error_public.h | 0 lib/{ => common}/fse.h | 0 lib/{ => common}/fse_static.h | 48 ++ lib/{huff0.h => common/huf.h} | 12 +- lib/{huff0_static.h => common/huf_static.h} | 107 +++- lib/{ => common}/mem.h | 0 lib/{ => common}/zbuff.h | 0 lib/{ => common}/zbuff_static.h | 8 + lib/{ => common}/zstd.h | 0 lib/{ => common}/zstd_internal.h | 0 lib/{ => common}/zstd_static.h | 0 lib/{ => compress}/.debug/zstd_stats.h | 0 lib/{fse.c => compress/fse_compress.c} | 388 +------------ lib/compress/huf_compress.c | 560 ++++++++++++++++++ lib/{zbuff.c => compress/zbuff_compress.c} | 213 ------- lib/{ => compress}/zstd_compress.c | 2 +- lib/{ => compress}/zstd_opt.h | 0 lib/decompress/fse_decompress.c | 438 ++++++++++++++ lib/{huff0.c => decompress/huf_decompress.c} | 580 +------------------ lib/decompress/zbuff_decompress.c | 258 +++++++++ lib/{ => decompress}/zstd_decompress.c | 2 +- lib/{ => dictBuilder}/divsufsort.c | 0 lib/{ => dictBuilder}/divsufsort.h | 0 lib/{ => dictBuilder}/zdict.c | 2 +- lib/{ => dictBuilder}/zdict.h | 0 lib/{ => dictBuilder}/zdict_static.h | 0 lib/legacy/zstd_v04.c | 5 - programs/Makefile | 30 +- programs/dibio.c | 2 +- programs/dibio.h | 2 +- programs/fileio.c | 8 - 35 files changed, 1468 insertions(+), 1216 deletions(-) rename lib/{ => common}/bitstream.h (100%) rename lib/{ => common}/error_private.h (100%) rename lib/{ => common}/error_public.h (100%) rename lib/{ => common}/fse.h (100%) rename lib/{ => common}/fse_static.h (89%) rename lib/{huff0.h => common/huf.h} (94%) rename lib/{huff0_static.h => common/huf_static.h} (63%) rename lib/{ => common}/mem.h (100%) rename lib/{ => common}/zbuff.h (100%) rename lib/{ => common}/zbuff_static.h (91%) rename lib/{ => common}/zstd.h (100%) rename lib/{ => common}/zstd_internal.h (100%) rename lib/{ => common}/zstd_static.h (100%) rename lib/{ => compress}/.debug/zstd_stats.h (100%) rename lib/{fse.c => compress/fse_compress.c} (69%) create mode 100644 lib/compress/huf_compress.c rename lib/{zbuff.c => compress/zbuff_compress.c} (57%) rename lib/{ => compress}/zstd_compress.c (99%) rename lib/{ => compress}/zstd_opt.h (100%) create mode 100644 lib/decompress/fse_decompress.c rename lib/{huff0.c => decompress/huf_decompress.c} (65%) create mode 100644 lib/decompress/zbuff_decompress.c rename lib/{ => decompress}/zstd_decompress.c (99%) rename lib/{ => dictBuilder}/divsufsort.c (100%) rename lib/{ => dictBuilder}/divsufsort.h (100%) rename lib/{ => dictBuilder}/zdict.c (99%) rename lib/{ => dictBuilder}/zdict.h (100%) rename lib/{ => dictBuilder}/zdict_static.h (100%) diff --git a/lib/Makefile b/lib/Makefile index 9bf83f1a1..e096caf71 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -31,9 +31,9 @@ # ################################################################ # Version numbers -LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` -LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` -LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./common/zstd.h` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./common/zstd.h` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./common/zstd.h` LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) @@ -43,7 +43,7 @@ VERSION?= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I. +CPPFLAGS= -I./common CFLAGS ?= -O3 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1 FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -51,7 +51,10 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include -ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c zbuff.c zdict.c divsufsort.c +ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c +ZSTDDECOMP_FILES := decompress/zstd_decompress.c decompress/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c +ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES) ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c ifeq ($(ZSTD_LEGACY_SUPPORT), 0) diff --git a/lib/README.md b/lib/README.md index 91ef00c40..ef446427a 100644 --- a/lib/README.md +++ b/lib/README.md @@ -22,9 +22,9 @@ as their definition may change in future version of the library. - fse.c - fse.h - fse_static.h -- huff0.c -- huff0.h -- huff0_static.h +- huf.c +- huf.h +- huf_static.h - zstd_compress.c - zstd_decompress.c - zstd_internal.h diff --git a/lib/bitstream.h b/lib/common/bitstream.h similarity index 100% rename from lib/bitstream.h rename to lib/common/bitstream.h diff --git a/lib/error_private.h b/lib/common/error_private.h similarity index 100% rename from lib/error_private.h rename to lib/common/error_private.h diff --git a/lib/error_public.h b/lib/common/error_public.h similarity index 100% rename from lib/error_public.h rename to lib/common/error_public.h diff --git a/lib/fse.h b/lib/common/fse.h similarity index 100% rename from lib/fse.h rename to lib/common/fse.h diff --git a/lib/fse_static.h b/lib/common/fse_static.h similarity index 89% rename from lib/fse_static.h rename to lib/common/fse_static.h index f3c3d44e0..0661dbd3e 100644 --- a/lib/fse_static.h +++ b/lib/common/fse_static.h @@ -334,6 +334,54 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) } + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#define FSE_MAX_MEMORY_USAGE 14 +#define FSE_DEFAULT_MEMORY_USAGE 13 + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#define FSE_MAX_SYMBOL_VALUE 255 + + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + #if defined (__cplusplus) } #endif diff --git a/lib/huff0.h b/lib/common/huf.h similarity index 94% rename from lib/huff0.h rename to lib/common/huf.h index 9d6e11f76..d07080b15 100644 --- a/lib/huff0.h +++ b/lib/common/huf.h @@ -1,5 +1,5 @@ /* ****************************************************************** - Huff0 : Huffman coder, part of New Generation Entropy library + Huffman coder, part of New Generation Entropy library header file Copyright (C) 2013-2016, Yann Collet. @@ -31,8 +31,8 @@ You can contact the author at : - Source repository : https://github.com/Cyan4973/FiniteStateEntropy ****************************************************************** */ -#ifndef HUFF0_H -#define HUFF0_H +#ifndef HUF_H +#define HUF_H #if defined (__cplusplus) extern "C" { @@ -46,7 +46,7 @@ extern "C" { /* **************************************** -* Huff0 simple functions +* HUF simple functions ******************************************/ size_t HUF_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -63,7 +63,7 @@ HUF_compress() : if HUF_isError(return), compression failed (more details using HUF_getErrorName()) HUF_decompress() : - Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize', + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', into already allocated destination buffer 'dst', of size 'dstSize'. `dstSize` : must be the **exact** size of original (uncompressed) data. Note : in contrast with FSE, HUF_decompress can regenerate @@ -94,4 +94,4 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize } #endif -#endif /* HUFF0_H */ +#endif /* HUF_H */ diff --git a/lib/huff0_static.h b/lib/common/huf_static.h similarity index 63% rename from lib/huff0_static.h rename to lib/common/huf_static.h index 32d66d352..d2d29a42b 100644 --- a/lib/huff0_static.h +++ b/lib/common/huf_static.h @@ -1,5 +1,5 @@ /* ****************************************************************** - Huff0 : Huffman codec, part of New Generation Entropy library + Huffman codec, part of New Generation Entropy library header file, for static linking only Copyright (C) 2013-2016, Yann Collet @@ -31,8 +31,8 @@ You can contact the author at : - Source repository : https://github.com/Cyan4973/FiniteStateEntropy ****************************************************************** */ -#ifndef HUFF0_STATIC_H -#define HUFF0_STATIC_H +#ifndef HUF_STATIC_H +#define HUF_STATIC_H #if defined (__cplusplus) extern "C" { @@ -42,24 +42,26 @@ extern "C" { /* **************************************** * Dependency ******************************************/ -#include "huff0.h" +#include "huf.h" +#include "fse.h" +#include "bitstream.h" /* **************************************** * Static allocation ******************************************/ -/* Huff0 buffer bounds */ +/* HUF buffer bounds */ #define HUF_CTABLEBOUND 129 #define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ -/* static allocation of Huff0's Compression Table */ +/* static allocation of HUF's Compression Table */ #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ U32 name##hb[maxSymbolValue+1]; \ void* name##hv = &(name##hb); \ HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ -/* static allocation of Huff0's DTable */ +/* static allocation of HUF's DTable */ #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1< HUF_ABSOLUTEMAX_TABLELOG) +# error "HUF_MAX_TABLELOG is too large !" +#endif + + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` +*/ +MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + U32 tableLog; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + size_t oSize; + + //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + if (iSize >= (242)) { /* RLE */ + static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; + oSize = l[iSize-242]; + memset(huffWeight, 1, hwSize); + iSize = 0; + } + else { /* Incompressible */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + }} + + /* get last non-null symbol weight (implied, total must be 2^n) */ + tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + *tableLogPtr = tableLog; + return iSize+1; +} + + + #if defined (__cplusplus) } #endif -#endif /* HUFF0_STATIC_H */ +#endif /* HUF_STATIC_H */ diff --git a/lib/mem.h b/lib/common/mem.h similarity index 100% rename from lib/mem.h rename to lib/common/mem.h diff --git a/lib/zbuff.h b/lib/common/zbuff.h similarity index 100% rename from lib/zbuff.h rename to lib/common/zbuff.h diff --git a/lib/zbuff_static.h b/lib/common/zbuff_static.h similarity index 91% rename from lib/zbuff_static.h rename to lib/common/zbuff_static.h index 9fb522e5f..4c52eef22 100644 --- a/lib/zbuff_static.h +++ b/lib/common/zbuff_static.h @@ -46,6 +46,7 @@ extern "C" { ***************************************/ #include "zstd_static.h" /* ZSTD_parameters */ #include "zbuff.h" +#include "zstd_internal.h" /* MIN */ /* ************************************* @@ -55,6 +56,13 @@ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); +MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + #if defined (__cplusplus) } diff --git a/lib/zstd.h b/lib/common/zstd.h similarity index 100% rename from lib/zstd.h rename to lib/common/zstd.h diff --git a/lib/zstd_internal.h b/lib/common/zstd_internal.h similarity index 100% rename from lib/zstd_internal.h rename to lib/common/zstd_internal.h diff --git a/lib/zstd_static.h b/lib/common/zstd_static.h similarity index 100% rename from lib/zstd_static.h rename to lib/common/zstd_static.h diff --git a/lib/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h similarity index 100% rename from lib/.debug/zstd_stats.h rename to lib/compress/.debug/zstd_stats.h diff --git a/lib/fse.c b/lib/compress/fse_compress.c similarity index 69% rename from lib/fse.c rename to lib/compress/fse_compress.c index 63898ab1c..dad0be794 100644 --- a/lib/fse.c +++ b/lib/compress/fse_compress.c @@ -1,5 +1,5 @@ /* ****************************************************************** - FSE : Finite State Entropy coder + FSE : Finite State Entropy encoder Copyright (C) 2013-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -32,35 +32,6 @@ - Public forum : https://groups.google.com/forum/#!forum/lz4c ****************************************************************** */ -#ifndef FSE_COMMONDEFS_ONLY - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSE_MAX_MEMORY_USAGE 14 -#define FSE_DEFAULT_MEMORY_USAGE 13 - -/*!FSE_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#define FSE_MAX_SYMBOL_VALUE 255 - - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSE_FUNCTION_TYPE BYTE -#define FSE_FUNCTION_EXTENSION -#define FSE_DECODE_TYPE FSE_decode_t - - -#endif /* !FSE_COMMONDEFS_ONLY */ - /* ************************************************************** * Compiler specifics ****************************************************************/ @@ -89,21 +60,6 @@ #include "fse_static.h" -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) -#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX -#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - - /* ************************************************************** * Error Management ****************************************************************/ @@ -114,7 +70,6 @@ * Complex types ****************************************************************/ typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; -typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; /* ************************************************************** @@ -141,8 +96,6 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; /* Function templates */ -static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } - size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { U32 const tableSize = 1 << tableLog; @@ -151,12 +104,15 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned U16* const tableU16 = ( (U16*) ptr) + 2; void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - U32 const step = FSE_tableStep(tableSize); + U32 const step = FSE_TABLESTEP(tableSize); U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; + FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ U32 highThreshold = tableSize-1; /* CTable header */ + + tableU16[-2] = (U16) tableLog; tableU16[-1] = (U16) maxSymbolValue; @@ -186,6 +142,7 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned position = (position + step) & tableMask; while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */ } } + if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ } @@ -202,6 +159,7 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned switch (normalizedCounter[s]) { case 0: break; + case -1: case 1: symbolTT[s].deltaNbBits = (tableLog << 16) - (1< FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - free(dt); -} - -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - FSE_DTableHeader DTableH; - void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */ - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); - const U32 tableSize = 1 << tableLog; - const U32 tableMask = tableSize-1; - const U32 step = FSE_tableStep(tableSize); - U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; - U32 highThreshold = tableSize-1; - S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 noLarge = 1; - U32 s; - - /* Sanity Checks */ - if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - DTableH.tableLog = (U16)tableLog; - for (s=0; s<=maxSymbolValue; s++) { - if (normalizedCounter[s]==-1) { - tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; - symbolNext[s] = 1; - } else { - if (normalizedCounter[s] >= largeLimit) noLarge=0; - symbolNext[s] = normalizedCounter[s]; - } } - - /* Spread symbols */ - { U32 position = 0; - for (s=0; s<=maxSymbolValue; s++) { - int i; - for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ - } } - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { U32 u; - for (u=0; u FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1<1) && (charnum<=*maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0+=24; - if (ip < iend-5) { - ip+=2; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount+=16; - } } - while ((bitStream & 3) == 3) { - n0+=3; - bitStream>>=2; - bitCount+=2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; - } - else - bitStream >>= 2; - } - { short const max = (short)((2*threshold-1)-remaining); - short count; - - if ((bitStream & (threshold-1)) < (U32)max) { - count = (short)(bitStream & (threshold-1)); - bitCount += nbBits-1; - } else { - count = (short)(bitStream & (2*threshold-1)); - if (count >= threshold) count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= FSE_abs(count); - normalizedCounter[charnum++] = count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } - if (remaining != 1) return ERROR(GENERIC); - *maxSVPtr = charnum-1; - - ip += (bitCount+7)>>3; - if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); - return ip-istart; -} - /*-************************************************************** * Counting histogram @@ -519,6 +312,7 @@ static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, unsigned maxSymbolValue = *maxSymbolValuePtr; unsigned max=0; + memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } @@ -542,6 +336,7 @@ static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr, unsigned maxSymbolValue = *maxSymbolValuePtr; unsigned max=0; + U32 Counting1[256] = { 0 }; U32 Counting2[256] = { 0 }; U32 Counting3[256] = { 0 }; @@ -619,6 +414,7 @@ size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, } + /*-************************************************************** * FSE Compression Code ****************************************************************/ @@ -763,6 +559,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + U64 const scale = 62 - tableLog; U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ U64 const vStep = 1ULL<<(scale-20); @@ -839,11 +636,13 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) /* Build Symbol Transformation Table */ { const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); + for (s=0; s<=maxSymbolValue; s++) { symbolTT[s].deltaNbBits = deltaNbBits; symbolTT[s].deltaFindState = s-1; } } + return 0; } @@ -878,6 +677,8 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize, const BYTE* const istart = (const BYTE*) src; const BYTE* const iend = istart + srcSize; const BYTE* ip=iend; + + BIT_CStream_t bitC; FSE_CState_t CState1, CState2; @@ -908,6 +709,7 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize, /* 2 or 4 encoding per loop */ for ( ; ip>istart ; ) { + FSE_encodeSymbol(&bitC, &CState2, *--ip); if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ @@ -998,162 +800,4 @@ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize) } -/*-******************************************************* -* Decompression (Byte symbols) -*********************************************************/ -size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const cell = (FSE_decode_t*)dPtr; - - DTableH->tableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSymbolValue = tableMask; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s<=maxSymbolValue; s++) { - dinfo[s].newState = 0; - dinfo[s].symbol = (BYTE)s; - dinfo[s].nbBits = (BYTE)nbBits; - } - - return 0; -} - -FORCE_INLINE size_t FSE_decompress_usingDTable_generic( - void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt, const unsigned fast) -{ - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const omax = op + maxDstSize; - BYTE* const olimit = omax-3; - - BIT_DStream_t bitD; - FSE_DState_t state1; - FSE_DState_t state2; - size_t errorCode; - - /* Init */ - errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ - if (FSE_isError(errorCode)) return errorCode; - - FSE_initDState(&state1, &bitD, dt); - FSE_initDState(&state2, &bitD, dt); - -#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) - - /* 4 symbols per loop */ - for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[1] = FSE_GETSYMBOL(&state2); - - if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } - - op[2] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[3] = FSE_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ - while (1) { - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSE_GETSYMBOL(&state1); - - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state2); - break; - } - - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSE_GETSYMBOL(&state2); - - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state1); - break; - } } - - return op-ostart; -} - - -size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* const istart = (const BYTE*)cSrc; - const BYTE* ip = istart; - short counting[FSE_MAX_SYMBOL_VALUE+1]; - DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ - unsigned tableLog; - unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - size_t errorCode; - - if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ - - /* normal FSE decoding mode */ - errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (FSE_isError(errorCode)) return errorCode; - if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ - ip += errorCode; - cSrcSize -= errorCode; - - errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); - if (FSE_isError(errorCode)) return errorCode; - - /* always return, even if it is an error code */ - return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); -} - - - #endif /* FSE_COMMONDEFS_ONLY */ diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c new file mode 100644 index 000000000..d126305c6 --- /dev/null +++ b/lib/compress/huf_compress.c @@ -0,0 +1,560 @@ +/* ****************************************************************** + Huffman encoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + + +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "huf_static.h" +#include "bitstream.h" +#include "fse.h" /* header compression */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt within huf_static.h */ + +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + +/*! HUF_writeCTable() : + `CTable` : huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) +{ + BYTE bitsToWeight[HUF_MAX_TABLELOG + 1]; + BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; + U32 n; + BYTE* op = (BYTE*)dst; + size_t size; + + /* check conditions */ + if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1) + return ERROR(GENERIC); + + /* convert to weight */ + bitsToWeight[0] = 0; + for (n=1; n<=huffLog; n++) + bitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; n= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */ + if ((size <= 1) || (size >= maxSymbolValue/2)) { + if (size==1) { /* RLE */ + /* only possible case : serie of 1 (because there are at least 2) */ + /* can only be 2^n or (2^n-1), otherwise not an huffman tree */ + BYTE code; + switch(maxSymbolValue) + { + case 1: code = 0; break; + case 2: code = 1; break; + case 3: code = 2; break; + case 4: code = 3; break; + case 7: code = 4; break; + case 8: code = 5; break; + case 15: code = 6; break; + case 16: code = 7; break; + case 31: code = 8; break; + case 32: code = 9; break; + case 63: code = 10; break; + case 64: code = 11; break; + case 127: code = 12; break; + case 128: code = 13; break; + default : return ERROR(corruption_detected); + } + op[0] = (BYTE)(255-13 + code); + return 1; + } + /* Not compressible */ + if (maxSymbolValue > (241-128)) return ERROR(GENERIC); /* not implemented (not possible with current format) */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1)); + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ + for (n=0; n HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill nbBits */ + { U32 n; for (n=0; n0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } + } + + return readSize; +} + + +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + U32 n = lastNonNull; + + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n --; + } /* n stops at huffNode[n].nbBits <= maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ + + /* renorm totalCost */ + totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_MAX_TABLELOG+1]; + int pos; + + /* Get pos of last (smallest) symbol per rank */ + memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = pos; + } } + + while (totalCost > 0) { + U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 highPos = rankLast[nBitsToDecrease]; + U32 lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + nBitsToDecrease ++; + totalCost -= 1 << (nBitsToDecrease-1); + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ + huffNode[rankLast[nBitsToDecrease]].nbBits ++; + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } } /* while (totalCost > 0) */ + + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + rankLast[1] = n+1; + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } } } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + + +typedef struct { + U32 base; + U32 current; +} rankPos; + +static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) +{ + rankPos rank[32]; + U32 n; + + memset(rank, 0, sizeof(rank)); + for (n=0; n<=maxSymbolValue; n++) { + U32 r = BIT_highbit32(count[n] + 1); + rank[r].base ++; + } + for (n=30; n>0; n--) rank[n-1].base += rank[n].base; + for (n=0; n<32; n++) rank[n].current = rank[n].base; + for (n=0; n<=maxSymbolValue; n++) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c+1) + 1; + U32 pos = rank[r].current++; + while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + + +#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1) +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) +{ + nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1]; + nodeElt* huffNode = huffNode0 + 1; + U32 n, nonNullRank; + int lowS, lowN; + U16 nodeNb = STARTNODE; + U32 nodeRoot; + + /* safety checks */ + if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; + if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC); + memset(huffNode0, 0, sizeof(huffNode0)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue); + + /* init for parents */ + nonNullRank = maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); + + /* create parents */ + while (nodeNb <= nodeRoot) { + U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + + /* fill result into tree (val, nbBits) */ + { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; + U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; + if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */ + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine stating value per rank */ + { U16 min = 0; + for (n=maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; n<=maxSymbolValue; n++) + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n=0; n<=maxSymbolValue; n++) + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ + } + + return maxNbBits; +} + +static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream) + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op); + if (HUF_isError(errorCode)) return 0; } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) + { + case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); + HUF_FLUSHBITS_2(&bitC); + case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); + HUF_FLUSHBITS_1(&bitC); + case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); + HUF_FLUSHBITS(&bitC); + case 0 : + default: ; + } + + for (; n>0; n-=4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n- 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + size_t segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t errorCode; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + MEM_writeLE16(ostart, (U16)errorCode); + + ip += segmentSize; + op += errorCode; + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + MEM_writeLE16(ostart+2, (U16)errorCode); + + ip += segmentSize; + op += errorCode; + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + MEM_writeLE16(ostart+4, (U16)errorCode); + + ip += segmentSize; + op += errorCode; + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + + op += errorCode; + return op-ostart; +} + + +static size_t HUF_compress_internal ( + void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + unsigned singleStream) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + U32 count[HUF_MAX_SYMBOL_VALUE+1]; + HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1]; + size_t errorCode; + + /* checks & inits */ + if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */ + if (dstSize < 1) return 0; /* not compressible within dst budget */ + if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE; + if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG; + + /* Scan input and build symbol stats */ + errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } + if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */ + + /* Build Huffman Tree */ + errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog); + if (HUF_isError(errorCode)) return errorCode; + huffLog = (U32)errorCode; + + /* Write table description header */ + errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */ + op += errorCode; + + /* Compress */ + if (singleStream) + errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */ + else + errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + op += errorCode; + + /* check compressibility */ + if ((size_t)(op-ostart) >= srcSize-1) + return 0; + + return op-ostart; +} + + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1); +} + +size_t HUF_compress2 (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0); +} + + +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG); +} diff --git a/lib/zbuff.c b/lib/compress/zbuff_compress.c similarity index 57% rename from lib/zbuff.c rename to lib/compress/zbuff_compress.c index eab0c482b..260aca087 100644 --- a/lib/zbuff.c +++ b/lib/compress/zbuff_compress.c @@ -169,13 +169,6 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) /* *** Compression *** */ -static size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr, @@ -290,215 +283,9 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) } -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFF_DCtx object is required to track streaming operations. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation, -* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFF_DCtx objects can be re-init multiple times. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFF_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - -typedef enum { ZBUFFds_init, ZBUFFds_readHeader, - ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; - -/* *** Resource management *** */ -struct ZBUFF_DCtx_s { - ZSTD_DCtx* zd; - ZSTD_frameParams fParams; - size_t blockSize; - char* inBuff; - size_t inBuffSize; - size_t inPos; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - ZBUFF_dStage stage; -}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ - - -ZBUFF_DCtx* ZBUFF_createDCtx(void) -{ - ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); - if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(*zbd)); - zbd->zd = ZSTD_createDCtx(); - zbd->stage = ZBUFFds_init; - return zbd; -} - -size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) -{ - if (zbd==NULL) return 0; /* support free on null */ - ZSTD_freeDCtx(zbd->zd); - free(zbd->inBuff); - free(zbd->outBuff); - free(zbd); - return 0; -} - - -/* *** Initialization *** */ - -size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) -{ - zbd->stage = ZBUFFds_readHeader; - zbd->inPos = zbd->outStart = zbd->outEnd = 0; - return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); -} - -size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) -{ - return ZBUFF_decompressInitDictionary(zbd, NULL, 0); -} - - -/* *** Decompression *** */ - -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr) -{ - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; - U32 notDone = 1; - - while (notDone) { - switch(zbd->stage) - { - case ZBUFFds_init : - return ERROR(init_missing); - - case ZBUFFds_readHeader : - /* read header from src */ - { size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr); - if (ZSTD_isError(headerSize)) return headerSize; - if (headerSize) { - /* not enough input to decode header : needs headerSize > *srcSizePtr */ - *dstCapacityPtr = 0; - *srcSizePtr = 0; - return headerSize; - } } - - /* Frame header instruct buffer sizes */ - { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); - zbd->blockSize = blockSize; - if (zbd->inBuffSize < blockSize) { - free(zbd->inBuff); - zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)malloc(blockSize); - if (zbd->inBuff == NULL) return ERROR(memory_allocation); - } - { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; - if (zbd->outBuffSize < neededOutSize) { - free(zbd->outBuff); - zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)malloc(neededOutSize); - if (zbd->outBuff == NULL) return ERROR(memory_allocation); - } } } - zbd->stage = ZBUFFds_read; - - case ZBUFFds_read: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (neededInSize==0) { /* end of frame */ - zbd->stage = ZBUFFds_init; - notDone = 0; - break; - } - if ((size_t)(iend-ip) >= neededInSize) { - /* directly decode from src */ - size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, - ip, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; - ip += neededInSize; - if (!decodedSize) break; /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - break; - } - if (ip==iend) { notDone = 0; break; } /* no more input */ - zbd->stage = ZBUFFds_load; - } - - case ZBUFFds_load: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); - size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ - size_t loadedSize; - if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); - ip += loadedSize; - zbd->inPos += loadedSize; - if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ - /* decode loaded input */ - { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, - zbd->inBuff, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; - zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - // break; /* ZBUFFds_flush follows */ - } } - - case ZBUFFds_flush: - { size_t const toFlushSize = zbd->outEnd - zbd->outStart; - size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); - op += flushedSize; - zbd->outStart += flushedSize; - if (flushedSize == toFlushSize) { - zbd->stage = ZBUFFds_read; - if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) - zbd->outStart = zbd->outEnd = 0; - break; - } - /* cannot flush everything */ - notDone = 0; - break; - } - default: return ERROR(GENERIC); /* impossible */ - } } - - /* result */ - *srcSizePtr = ip-istart; - *dstCapacityPtr = op-ostart; - { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ - nextSrcSizeHint -= zbd->inPos; /* already loaded*/ - return nextSrcSizeHint; - } -} - - /* ************************************* * Tool functions ***************************************/ -unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } -const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } - size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_MAX; } size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; } -size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; } -size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } diff --git a/lib/zstd_compress.c b/lib/compress/zstd_compress.c similarity index 99% rename from lib/zstd_compress.c rename to lib/compress/zstd_compress.c index 6ca1bafa3..1475d5fb3 100644 --- a/lib/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -55,7 +55,7 @@ #include /* memset */ #include "mem.h" #include "fse_static.h" -#include "huff0_static.h" +#include "huf_static.h" #include "zstd_internal.h" diff --git a/lib/zstd_opt.h b/lib/compress/zstd_opt.h similarity index 100% rename from lib/zstd_opt.h rename to lib/compress/zstd_opt.h diff --git a/lib/decompress/fse_decompress.c b/lib/decompress/fse_decompress.c new file mode 100644 index 000000000..e86a37204 --- /dev/null +++ b/lib/decompress/fse_decompress.c @@ -0,0 +1,438 @@ +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# ifdef __GNUC__ +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#include "fse_static.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Complex types +****************************************************************/ +typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + free(dt); +} + +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + FSE_DTableHeader DTableH; + void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + const U32 tableSize = 1 << tableLog; + const U32 tableMask = tableSize-1; + const U32 step = FSE_TABLESTEP(tableSize); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + + U32 highThreshold = tableSize-1; + S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 noLarge = 1; + U32 s; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + DTableH.tableLog = (U16)tableLog; + for (s=0; s<=maxSymbolValue; s++) { + if (normalizedCounter[s]==-1) { + tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) noLarge=0; + symbolNext[s] = normalizedCounter[s]; + } } + + /* Spread symbols */ + { U32 position = 0; + for (s=0; s<=maxSymbolValue; s++) { + int i; + for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<1) && (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0+=24; + if (ip < iend-5) { + ip+=2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount+=16; + } } + while ((bitStream & 3) == 3) { + n0+=3; + bitStream>>=2; + bitCount+=2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } + else + bitStream >>= 2; + } + { short const max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } else { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSE_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } + if (remaining != 1) return ERROR(GENERIC); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); + return ip-istart; +} + + + + +/*-******************************************************* +* Decompression (Byte symbols) +*********************************************************/ +size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const cell = (FSE_decode_t*)dPtr; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSymbolValue = tableMask; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<=maxSymbolValue; s++) { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE size_t FSE_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + BIT_DStream_t bitD; + FSE_DState_t state1; + FSE_DState_t state2; + size_t errorCode; + + /* Init */ + errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ + if (FSE_isError(errorCode)) return errorCode; + + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); + +#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSE_GETSYMBOL(&state1); + + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSE_GETSYMBOL(&state2); + + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + size_t errorCode; + + if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ + + /* normal FSE decoding mode */ + errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(errorCode)) return errorCode; + if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ + ip += errorCode; + cSrcSize -= errorCode; + + errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; + + /* always return, even if it is an error code */ + return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); +} + + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/lib/huff0.c b/lib/decompress/huf_decompress.c similarity index 65% rename from lib/huff0.c rename to lib/decompress/huf_decompress.c index 505adceca..bcd6f022d 100644 --- a/lib/huff0.c +++ b/lib/decompress/huf_decompress.c @@ -1,5 +1,5 @@ /* ****************************************************************** - Huff0 : Huffman coder, part of New Generation Entropy library + Huffman decoder, part of New Generation Entropy library Copyright (C) 2013-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -28,7 +28,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - FSE+Huff0 source repository : https://github.com/Cyan4973/FiniteStateEntropy + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - Public forum : https://groups.google.com/forum/#!forum/lz4c ****************************************************************** */ @@ -62,22 +62,11 @@ #include /* malloc, free, qsort */ #include /* memcpy, memset */ #include /* printf (debug) */ -#include "huff0_static.h" +#include "huf_static.h" #include "bitstream.h" #include "fse.h" /* header compression */ -/* ************************************************************** -* Constants -****************************************************************/ -#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */ -#define HUF_MAX_SYMBOL_VALUE 255 -#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG) -# error "HUF_MAX_TABLELOG is too large !" -#endif - /* ************************************************************** * Error Management @@ -87,499 +76,9 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -/* ******************************************************* -* Huff0 : Huffman block compression -*********************************************************/ -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt within huff0_static.h */ - -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - -/*! HUF_writeCTable() : - `CTable` : huffman tree to save, using huff0 representation. - @return : size of saved CTable */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, - const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) -{ - BYTE bitsToWeight[HUF_MAX_TABLELOG + 1]; - BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; - U32 n; - BYTE* op = (BYTE*)dst; - size_t size; - - /* check conditions */ - if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1) - return ERROR(GENERIC); - - /* convert to weight */ - bitsToWeight[0] = 0; - for (n=1; n<=huffLog; n++) - bitsToWeight[n] = (BYTE)(huffLog + 1 - n); - for (n=0; n= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */ - if ((size <= 1) || (size >= maxSymbolValue/2)) { - if (size==1) { /* RLE */ - /* only possible case : serie of 1 (because there are at least 2) */ - /* can only be 2^n or (2^n-1), otherwise not an huffman tree */ - BYTE code; - switch(maxSymbolValue) - { - case 1: code = 0; break; - case 2: code = 1; break; - case 3: code = 2; break; - case 4: code = 3; break; - case 7: code = 4; break; - case 8: code = 5; break; - case 15: code = 6; break; - case 16: code = 7; break; - case 31: code = 8; break; - case 32: code = 9; break; - case 63: code = 10; break; - case 64: code = 11; break; - case 127: code = 12; break; - case 128: code = 13; break; - default : return ERROR(corruption_detected); - } - op[0] = (BYTE)(255-13 + code); - return 1; - } - /* Not compressible */ - if (maxSymbolValue > (241-128)) return ERROR(GENERIC); /* not implemented (not possible with current format) */ - if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ - for (n=0; n HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); - - /* Prepare base value per rank */ - { U32 n, nextRankStart = 0; - for (n=1; n<=tableLog; n++) { - U32 current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } } - - /* fill nbBits */ - { U32 n; for (n=0; n0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - /* assign value within rank, symbol order */ - { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } - } - - return readSize; -} - - -static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) -{ - const U32 largestBits = huffNode[lastNonNull].nbBits; - if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ - - /* there are several too large elements (at least >= 2) */ - { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); - U32 n = lastNonNull; - - while (huffNode[n].nbBits > maxNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; - n --; - } /* n stops at huffNode[n].nbBits <= maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ - - /* renorm totalCost */ - totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ - - /* repay normalized cost */ - { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_MAX_TABLELOG+1]; - int pos; - - /* Get pos of last (smallest) symbol per rank */ - memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = maxNbBits; - for (pos=n ; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = pos; - } } - - while (totalCost > 0) { - U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; - for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 highPos = rankLast[nBitsToDecrease]; - U32 lowPos = rankLast[nBitsToDecrease-1]; - if (highPos == noSymbol) continue; - if (lowPos == noSymbol) break; - { U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) break; - } } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - nBitsToDecrease ++; - totalCost -= 1 << (nBitsToDecrease-1); - if (rankLast[nBitsToDecrease-1] == noSymbol) - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ - huffNode[rankLast[nBitsToDecrease]].nbBits ++; - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } } /* while (totalCost > 0) */ - - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ - while (huffNode[n].nbBits == maxNbBits) n--; - huffNode[n+1].nbBits--; - rankLast[1] = n+1; - totalCost++; - continue; - } - huffNode[ rankLast[1] + 1 ].nbBits--; - rankLast[1]++; - totalCost ++; - } } } /* there are several too large elements (at least >= 2) */ - - return maxNbBits; -} - - -typedef struct { - U32 base; - U32 current; -} rankPos; - -static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) -{ - rankPos rank[32]; - U32 n; - - memset(rank, 0, sizeof(rank)); - for (n=0; n<=maxSymbolValue; n++) { - U32 r = BIT_highbit32(count[n] + 1); - rank[r].base ++; - } - for (n=30; n>0; n--) rank[n-1].base += rank[n].base; - for (n=0; n<32; n++) rank[n].current = rank[n].base; - for (n=0; n<=maxSymbolValue; n++) { - U32 const c = count[n]; - U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rank[r].current++; - while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } -} - - -#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1) -size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) -{ - nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1]; - nodeElt* huffNode = huffNode0 + 1; - U32 n, nonNullRank; - int lowS, lowN; - U16 nodeNb = STARTNODE; - U32 nodeRoot; - - /* safety checks */ - if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; - if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC); - memset(huffNode0, 0, sizeof(huffNode0)); - - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue); - - /* init for parents */ - nonNullRank = maxSymbolValue; - while(huffNode[nonNullRank].count == 0) nonNullRank--; - lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; - huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; - nodeNb++; lowS-=2; - for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); - huffNode0[0].count = (U32)(1U<<31); - - /* create parents */ - while (nodeNb <= nodeRoot) { - U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = nodeNb; - nodeNb++; - } - - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n=nodeRoot-1; n>=STARTNODE; n--) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - for (n=0; n<=nonNullRank; n++) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - - /* enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); - - /* fill result into tree (val, nbBits) */ - { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; - U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; - if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */ - for (n=0; n<=nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine stating value per rank */ - { U16 min = 0; - for (n=maxNbBits; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - for (n=0; n<=maxSymbolValue; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n=0; n<=maxSymbolValue; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ - } - - return maxNbBits; -} - -static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) -{ - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -} - -size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } - -#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) - -#define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream) - -#define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream) - -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) -{ - const BYTE* ip = (const BYTE*) src; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t n; - const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); - BIT_CStream_t bitC; - - /* init */ - if (dstSize < 8) return 0; /* not enough space to compress */ - { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op); - if (HUF_isError(errorCode)) return 0; } - - n = srcSize & ~3; /* join to mod 4 */ - switch (srcSize & 3) - { - case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); - HUF_FLUSHBITS_2(&bitC); - case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); - HUF_FLUSHBITS_1(&bitC); - case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); - HUF_FLUSHBITS(&bitC); - case 0 : - default: ; - } - - for (; n>0; n-=4) { /* note : n&3==0 at this stage */ - HUF_encodeSymbol(&bitC, ip[n- 1], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 2], CTable); - HUF_FLUSHBITS_2(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 3], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 4], CTable); - HUF_FLUSHBITS(&bitC); - } - - return BIT_closeCStream(&bitC); -} - - -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) -{ - size_t segmentSize = (srcSize+3)/4; /* first 3 segments */ - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t errorCode; - - if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ - if (srcSize < 12) return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart, (U16)errorCode); - - ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart+2, (U16)errorCode); - - ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart+4, (U16)errorCode); - - ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - - op += errorCode; - return op-ostart; -} - - -static size_t HUF_compress_internal ( - void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - unsigned singleStream) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - U32 count[HUF_MAX_SYMBOL_VALUE+1]; - HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1]; - size_t errorCode; - - /* checks & inits */ - if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */ - if (dstSize < 1) return 0; /* not compressible within dst budget */ - if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */ - if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE; - if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG; - - /* Scan input and build symbol stats */ - errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } - if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */ - - /* Build Huffman Tree */ - errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog); - if (HUF_isError(errorCode)) return errorCode; - huffLog = (U32)errorCode; - - /* Write table description header */ - errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */ - op += errorCode; - - /* Compress */ - if (singleStream) - errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */ - else - errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - op += errorCode; - - /* check compressibility */ - if ((size_t)(op-ostart) >= srcSize-1) - return 0; - - return op-ostart; -} - - -size_t HUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1); -} - -size_t HUF_compress2 (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0); -} - - -size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) -{ - return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG); -} - /* ******************************************************* -* Huff0 : Huffman block decompression +* HUF : Huffman block decompression *********************************************************/ typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ @@ -587,77 +86,6 @@ typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* doubl typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` -*/ -static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - U32 tableLog; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - size_t oSize; - - //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - if (iSize >= (242)) { /* RLE */ - static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; - oSize = l[iSize-242]; - memset(huffWeight, 1, hwSize); - iSize = 0; - } - else { /* Incompressible */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } } - else { /* header compressed with FSE (normal case) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSE_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - }} - - /* get last non-null symbol weight (implied, total must be 2^n) */ - tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - *tableLogPtr = tableLog; - return iSize+1; -} /*-***************************/ diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c new file mode 100644 index 000000000..614a79240 --- /dev/null +++ b/lib/decompress/zbuff_decompress.c @@ -0,0 +1,258 @@ +/* + Buffered version of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + +/* ************************************* +* Dependencies +***************************************/ +#include +#include "error_private.h" +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ +#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ +#include "zbuff_static.h" + + +/* ************************************* +* Constants +***************************************/ +static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; + + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFF_DCtx object is required to track streaming operations. +* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. +* +* Use ZBUFF_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + +typedef enum { ZBUFFds_init, ZBUFFds_readHeader, + ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; + +/* *** Resource management *** */ +struct ZBUFF_DCtx_s { + ZSTD_DCtx* zd; + ZSTD_frameParams fParams; + size_t blockSize; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + ZBUFF_dStage stage; +}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ + + +ZBUFF_DCtx* ZBUFF_createDCtx(void) +{ + ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); + if (zbd==NULL) return NULL; + memset(zbd, 0, sizeof(*zbd)); + zbd->zd = ZSTD_createDCtx(); + zbd->stage = ZBUFFds_init; + return zbd; +} + +size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) +{ + if (zbd==NULL) return 0; /* support free on null */ + ZSTD_freeDCtx(zbd->zd); + free(zbd->inBuff); + free(zbd->outBuff); + free(zbd); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) +{ + zbd->stage = ZBUFFds_readHeader; + zbd->inPos = zbd->outStart = zbd->outEnd = 0; + return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); +} + +size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) +{ + return ZBUFF_decompressInitDictionary(zbd, NULL, 0); +} + + +/* *** Decompression *** */ + +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + U32 notDone = 1; + + while (notDone) { + switch(zbd->stage) + { + case ZBUFFds_init : + return ERROR(init_missing); + + case ZBUFFds_readHeader : + /* read header from src */ + { size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr); + if (ZSTD_isError(headerSize)) return headerSize; + if (headerSize) { + /* not enough input to decode header : needs headerSize > *srcSizePtr */ + *dstCapacityPtr = 0; + *srcSizePtr = 0; + return headerSize; + } } + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); + zbd->blockSize = blockSize; + if (zbd->inBuffSize < blockSize) { + free(zbd->inBuff); + zbd->inBuffSize = blockSize; + zbd->inBuff = (char*)malloc(blockSize); + if (zbd->inBuff == NULL) return ERROR(memory_allocation); + } + { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; + if (zbd->outBuffSize < neededOutSize) { + free(zbd->outBuff); + zbd->outBuffSize = neededOutSize; + zbd->outBuff = (char*)malloc(neededOutSize); + if (zbd->outBuff == NULL) return ERROR(memory_allocation); + } } } + zbd->stage = ZBUFFds_read; + + case ZBUFFds_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); + if (neededInSize==0) { /* end of frame */ + zbd->stage = ZBUFFds_init; + notDone = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { + /* directly decode from src */ + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize) break; /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + break; + } + if (ip==iend) { notDone = 0; break; } /* no more input */ + zbd->stage = ZBUFFds_load; + } + + case ZBUFFds_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); + size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zbd->inPos += loadedSize; + if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ + /* decode loaded input */ + { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zbd->inPos = 0; /* input is consumed */ + if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + // break; /* ZBUFFds_flush follows */ + } } + + case ZBUFFds_flush: + { size_t const toFlushSize = zbd->outEnd - zbd->outStart; + size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); + op += flushedSize; + zbd->outStart += flushedSize; + if (flushedSize == toFlushSize) { + zbd->stage = ZBUFFds_read; + if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) + zbd->outStart = zbd->outEnd = 0; + break; + } + /* cannot flush everything */ + notDone = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + *srcSizePtr = ip-istart; + *dstCapacityPtr = op-ostart; + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); + if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ + nextSrcSizeHint -= zbd->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + + +/* ************************************* +* Tool functions +***************************************/ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + +size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; } +size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } diff --git a/lib/zstd_decompress.c b/lib/decompress/zstd_decompress.c similarity index 99% rename from lib/zstd_decompress.c rename to lib/decompress/zstd_decompress.c index 5d3f44224..6515c68bf 100644 --- a/lib/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -59,7 +59,7 @@ #include "mem.h" /* low level memory routines */ #include "zstd_internal.h" #include "fse_static.h" -#include "huff0_static.h" +#include "huf_static.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_legacy.h" diff --git a/lib/divsufsort.c b/lib/dictBuilder/divsufsort.c similarity index 100% rename from lib/divsufsort.c rename to lib/dictBuilder/divsufsort.c diff --git a/lib/divsufsort.h b/lib/dictBuilder/divsufsort.h similarity index 100% rename from lib/divsufsort.h rename to lib/dictBuilder/divsufsort.h diff --git a/lib/zdict.c b/lib/dictBuilder/zdict.c similarity index 99% rename from lib/zdict.c rename to lib/dictBuilder/zdict.c index 122ac8cb2..2ebf2b49c 100644 --- a/lib/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -61,7 +61,7 @@ #include "mem.h" /* read */ #include "error_private.h" #include "fse.h" -#include "huff0_static.h" +#include "huf_static.h" #include "zstd_internal.h" #include "divsufsort.h" #include "zdict_static.h" diff --git a/lib/zdict.h b/lib/dictBuilder/zdict.h similarity index 100% rename from lib/zdict.h rename to lib/dictBuilder/zdict.h diff --git a/lib/zdict_static.h b/lib/dictBuilder/zdict_static.h similarity index 100% rename from lib/zdict_static.h rename to lib/dictBuilder/zdict_static.h diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 57d724c27..58dbe71a5 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -2004,11 +2004,6 @@ extern "C" { #endif -/* **************************************** -* Dependency -******************************************/ -#include "huff0.h" - /* **************************************** * Static allocation macros diff --git a/programs/Makefile b/programs/Makefile index 36ab3d864..e6c4e8047 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -32,9 +32,9 @@ # ########################################################################## # Version numbers -LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h` -LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h` -LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h` +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h` LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) @@ -44,7 +44,7 @@ VERSION?= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I../lib -DZSTD_VERSION=\"$(VERSION)\" +CPPFLAGS= -I../lib/common -DZSTD_VERSION=\"$(VERSION)\" CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -std=c99 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -53,7 +53,11 @@ BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib -ZSTD_FILES := $(ZSTDDIR)/huff0.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_compress.c +ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c +ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c +ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c +ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) ifeq ($(ZSTD_LEGACY_SUPPORT), 0) CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 @@ -85,11 +89,11 @@ default: zstd all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen -zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ +zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) -zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ +zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) @@ -107,18 +111,16 @@ zstd-pgo : clean zstd rm zstd $(MAKE) zstd MOREFLAGS=-fprofile-use -zstd-frugal: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c zstdcli.c fileio.c +zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) zstd-small: clean CFLAGS="-Os -s" $(MAKE) zstd-frugal -fullbench : $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \ - datagen.c fullbench.c +fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fullbench32: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \ - datagen.c fullbench.c +fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) fuzzer : $(ZSTD_FILES) \ @@ -129,11 +131,11 @@ fuzzer32: $(ZSTD_FILES) \ datagen.c xxhash.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -zbufftest : $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \ +zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) \ datagen.c xxhash.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) -zbufftest32: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \ +zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \ datagen.c xxhash.c zbufftest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) diff --git a/programs/dibio.c b/programs/dibio.c index 17f89586b..9641ac299 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -52,7 +52,7 @@ #include "mem.h" /* read */ #include "error_private.h" -#include "zdict_static.h" +#include "dibio.h" /*-************************************* diff --git a/programs/dibio.h b/programs/dibio.h index 0ccec4135..fd1f21df0 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,7 @@ /*-************************************* * Dependencies ***************************************/ -#include "zdict_static.h" /* ZDICT_params_t */ +#include "../lib/dictBuilder/zdict_static.h" /* ZDICT_params_t */ /*-************************************* diff --git a/programs/fileio.c b/programs/fileio.c index ec0d19f21..edd917e81 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -95,10 +95,6 @@ /*-************************************* * Constants ***************************************/ -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - #define _1BIT 0x01 #define _2BITS 0x03 #define _3BITS 0x07 @@ -106,9 +102,6 @@ #define _6BITS 0x3F #define _8BITS 0xFF -#define BIT6 0x40 -#define BIT7 0x80 - #define BLOCKSIZE (128 KB) #define ROLLBUFFERSIZE (BLOCKSIZE*8*64) @@ -135,7 +128,6 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } static const unsigned refreshRate = 150; static clock_t g_time = 0; -#define MAX(a,b) ((a)>(b)?(a):(b)) /*-************************************* From 3c7c3527d0a84bc794819173a681e415fa4e08e0 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 22 Apr 2016 13:59:05 +0200 Subject: [PATCH 11/26] introduced ZSTD_NOCOMPRESS to generate decompressor only --- lib/compress/zstd_compress.c | 4 ++-- lib/decompress/zbuff_decompress.c | 6 ------ programs/Makefile | 3 +++ programs/fileio.c | 6 +++++- programs/zstdcli.c | 22 ++++++++++++++++------ 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1475d5fb3..a8ebaa549 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2432,7 +2432,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS /*-===== Pre-defined compression levels =====-*/ -#define ZSTD_DEAFULT_CLEVEL 5 +#define ZSTD_DEFAULT_CLEVEL 5 #define ZSTD_MAX_CLEVEL 22 unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } @@ -2552,7 +2552,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si size_t const addedSize = srcSize ? 0 : 500; U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ - if (compressionLevel < 0) compressionLevel = ZSTD_DEAFULT_CLEVEL; + if (compressionLevel < 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; if (compressionLevel==0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; cp = ZSTD_defaultCParameters[tableID][compressionLevel]; diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 614a79240..fef32f76d 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -40,12 +40,6 @@ #include "zbuff_static.h" -/* ************************************* -* Constants -***************************************/ -static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; - - /*-*************************************************************************** * Streaming decompression howto * diff --git a/programs/Makefile b/programs/Makefile index e6c4e8047..e0a26127c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -114,6 +114,9 @@ zstd-pgo : clean zstd zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) +zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) + zstd-small: clean CFLAGS="-Os -s" $(MAKE) zstd-frugal diff --git a/programs/fileio.c b/programs/fileio.c index edd917e81..e0ccee7f8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -112,6 +112,8 @@ #define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */ +#define FNSPACE 30 + /*-************************************* * Macros @@ -267,6 +269,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName) return (size_t)fileSize; } +#ifndef ZSTD_NOCOMPRESS /*-********************************************************************** * Compression @@ -460,7 +463,6 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, } -#define FNSPACE 30 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, const char* suffix, const char* dictFileName, int compressionLevel) @@ -499,6 +501,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile return missed_files; } +#endif // #ifndef ZSTD_NOCOMPRESS + /* ************************************************************************** * Decompression diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e43b94541..feaddc339 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -111,7 +111,9 @@ static int usage(const char* programName) DISPLAY( "FILE : a filename\n"); DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); DISPLAY( "Arguments :\n"); +#ifndef ZSTD_NOCOMPRESS DISPLAY( " -# : # compression level (1-%u, default:1) \n", ZSTD_maxCLevel()); +#endif DISPLAY( " -d : decompression \n"); DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); @@ -131,7 +133,9 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); +#ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); +#endif #ifndef ZSTD_NODICT DISPLAY( "Dictionary builder :\n"); DISPLAY( "--train : create a dictionary from a training set of files \n"); @@ -194,6 +198,7 @@ int main(int argCount, const char** argv) /* init */ (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ @@ -233,6 +238,7 @@ int main(int argCount, const char** argv) argument++; while (argument[0]!=0) { +#ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { cLevel = 0; @@ -246,6 +252,7 @@ int main(int argCount, const char** argv) CLEAN_RETURN(badusage(programName)); continue; } +#endif switch(argument[0]) { @@ -417,16 +424,19 @@ int main(int argCount, const char** argv) /* IO Stream/File */ FIO_setNotificationLevel(displayLevel); - if (decode) { - if (filenameIdx==1 && outFileName) - operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); - else - operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName); - } else { /* compression */ +#ifndef ZSTD_NOCOMPRESS + if (!decode) { if (filenameIdx==1 && outFileName) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); else operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel); + } else +#endif + { /* decompression */ + if (filenameIdx==1 && outFileName) + operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); + else + operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName); } _end: From d6be2751a8a6cc60e131ff10774e933ddbcd2204 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 22 Apr 2016 13:59:21 +0200 Subject: [PATCH 12/26] updated CMakeLists.txt --- .gitignore | 9 ++++-- contrib/cmake/lib/CMakeLists.txt | 51 +++++++++++++++++--------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 8af4bbe96..12e903e05 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,9 @@ ipch/ # Other files .directory -_codelite -_zstdbench -zlib_wrapper +_codelite/ +_zstdbench/ +zlib_wrapper/ + +# CMake +contrib/cmake/ diff --git a/contrib/cmake/lib/CMakeLists.txt b/contrib/cmake/lib/CMakeLists.txt index bb2c05755..24181ce37 100644 --- a/contrib/cmake/lib/CMakeLists.txt +++ b/contrib/cmake/lib/CMakeLists.txt @@ -47,40 +47,43 @@ SET(ROOT_DIR ../../..) # Define library directory, where sources and header files are located SET(LIBRARY_DIR ${ROOT_DIR}/lib) -INCLUDE_DIRECTORIES(${LIBRARY_DIR}) +INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common) # Read file content -FILE(READ ${LIBRARY_DIR}/zstd.h HEADER_CONTENT) +FILE(READ ${LIBRARY_DIR}/common/zstd.h HEADER_CONTENT) # Parse version GetLibraryVersion("${HEADER_CONTENT}" LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE) MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") SET(Sources - ${LIBRARY_DIR}/divsufsort.c - ${LIBRARY_DIR}/fse.c - ${LIBRARY_DIR}/huff0.c - ${LIBRARY_DIR}/zbuff.c - ${LIBRARY_DIR}/zdict.c - ${LIBRARY_DIR}/zstd_compress.c - ${LIBRARY_DIR}/zstd_decompress.c) + ${LIBRARY_DIR}/compress/fse_compress.c + ${LIBRARY_DIR}/compress/huf_compress.c + ${LIBRARY_DIR}/compress/zbuff_compress.c + ${LIBRARY_DIR}/compress/zstd_compress.c + ${LIBRARY_DIR}/decompress/fse_decompress.c + ${LIBRARY_DIR}/decompress/huf_decompress.c + ${LIBRARY_DIR}/decompress/zbuff_decompress.c + ${LIBRARY_DIR}/decompress/zstd_decompress.c + ${LIBRARY_DIR}/dictBuilder/divsufsort.c + ${LIBRARY_DIR}/dictBuilder/zdict.c) SET(Headers - ${LIBRARY_DIR}/bitstream.h - ${LIBRARY_DIR}/error_private.h - ${LIBRARY_DIR}/error_public.h - ${LIBRARY_DIR}/fse.h - ${LIBRARY_DIR}/fse_static.h - ${LIBRARY_DIR}/huff0.h - ${LIBRARY_DIR}/huff0_static.h - ${LIBRARY_DIR}/mem.h - ${LIBRARY_DIR}/zbuff.h - ${LIBRARY_DIR}/zbuff_static.h - ${LIBRARY_DIR}/zdict.h - ${LIBRARY_DIR}/zdict_static.h - ${LIBRARY_DIR}/zstd_internal.h - ${LIBRARY_DIR}/zstd_static.h - ${LIBRARY_DIR}/zstd.h) + ${LIBRARY_DIR}/common/bitstream.h + ${LIBRARY_DIR}/common/error_private.h + ${LIBRARY_DIR}/common/error_public.h + ${LIBRARY_DIR}/common/fse.h + ${LIBRARY_DIR}/common/fse_static.h + ${LIBRARY_DIR}/common/huf.h + ${LIBRARY_DIR}/common/huf_static.h + ${LIBRARY_DIR}/common/mem.h + ${LIBRARY_DIR}/common/zbuff.h + ${LIBRARY_DIR}/common/zbuff_static.h + ${LIBRARY_DIR}/common/zstd_internal.h + ${LIBRARY_DIR}/common/zstd_static.h + ${LIBRARY_DIR}/common/zstd.h + ${LIBRARY_DIR}/dictBuilder/zdict.h + ${LIBRARY_DIR}/dictBuilder/zdict_static.h) IF (ZSTD_LEGACY_SUPPORT) SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy) From a1febea01d96bdb72474230537d26152ac560187 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 22 Apr 2016 17:14:25 +0200 Subject: [PATCH 13/26] Error functions moved to common/zstd_common.c --- Makefile | 2 +- contrib/cmake/lib/CMakeLists.txt | 5 +- contrib/cmake/programs/CMakeLists.txt | 2 +- lib/Makefile | 8 +-- lib/common/zstd_common.c | 85 +++++++++++++++++++++++++++ lib/decompress/fse_decompress.c | 7 --- lib/decompress/huf_decompress.c | 2 - lib/decompress/zbuff_decompress.c | 3 - lib/decompress/zstd_decompress.c | 14 +---- programs/Makefile | 9 ++- 10 files changed, 101 insertions(+), 36 deletions(-) create mode 100644 lib/common/zstd_common.c diff --git a/Makefile b/Makefile index 494f59dca..3f32deeb1 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ # ################################################################ # force a version number : uncomment below export (otherwise, default to the one declared into zstd.h) -#export VERSION := 0.5.1 +#export VERSION := 0.6.1 PRGDIR = programs ZSTDDIR = lib diff --git a/contrib/cmake/lib/CMakeLists.txt b/contrib/cmake/lib/CMakeLists.txt index 24181ce37..6d1496683 100644 --- a/contrib/cmake/lib/CMakeLists.txt +++ b/contrib/cmake/lib/CMakeLists.txt @@ -57,6 +57,7 @@ GetLibraryVersion("${HEADER_CONTENT}" LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE) MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") SET(Sources + ${LIBRARY_DIR}/common/zstd_common.c ${LIBRARY_DIR}/compress/fse_compress.c ${LIBRARY_DIR}/compress/huf_compress.c ${LIBRARY_DIR}/compress/zbuff_compress.c @@ -120,8 +121,8 @@ ENDIF (MSVC) # Define include directories IF (NOT WORKAROUND_OUTDATED_CODE_STYLE) - TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}) - TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}) + TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common) + TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common) IF (ZSTD_LEGACY_SUPPORT) TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR}) TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR}) diff --git a/contrib/cmake/programs/CMakeLists.txt b/contrib/cmake/programs/CMakeLists.txt index ebee7c210..3b4f51d79 100644 --- a/contrib/cmake/programs/CMakeLists.txt +++ b/contrib/cmake/programs/CMakeLists.txt @@ -44,7 +44,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR}) IF (WORKAROUND_OUTDATED_CODE_STYLE) # Define library directory, where sources and header files are located SET(LIBRARY_DIR ${ROOT_DIR}/lib) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}) + INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common) ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) IF (ZSTD_LEGACY_SUPPORT) diff --git a/lib/Makefile b/lib/Makefile index e096caf71..121a0ce37 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -54,7 +54,7 @@ INCLUDEDIR=$(PREFIX)/include ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c ZSTDDECOMP_FILES := decompress/zstd_decompress.c decompress/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES) +ZSTD_FILES := $(ZSTDDECOMP_FILES) common/zstd_common.c $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES) ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c ifeq ($(ZSTD_LEGACY_SUPPORT), 0) @@ -120,9 +120,9 @@ install: libzstd libzstd.pc @cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a - @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h - @install -m 644 zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h + @install -m 644 common/zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h + @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed uninstall: diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c new file mode 100644 index 000000000..c833bdacf --- /dev/null +++ b/lib/common/zstd_common.c @@ -0,0 +1,85 @@ +/* + Common functions of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + +/* ************************************* +* Dependencies +***************************************/ +#include +#include "error_private.h" +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ +#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ +#include "zbuff_static.h" + + +/* ************************************* +* Constants +***************************************/ + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +/*! ZSTD_isError() : +* tells if a return value is an error code */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getError() : +* convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); } + +/*! ZSTD_getErrorName() : +* provides error code string (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-**************************************** +* FSE Error Management +******************************************/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } + +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* HUF Error Management +****************************************************************/ +unsigned HUF_isError(size_t code) { return ERR_isError(code); } + +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* ZBUFF Error Management +****************************************************************/ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } diff --git a/lib/decompress/fse_decompress.c b/lib/decompress/fse_decompress.c index e86a37204..ec699ab91 100644 --- a/lib/decompress/fse_decompress.c +++ b/lib/decompress/fse_decompress.c @@ -169,13 +169,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned #ifndef FSE_COMMONDEFS_ONLY -/*-**************************************** -* FSE helper functions -******************************************/ -unsigned FSE_isError(size_t code) { return ERR_isError(code); } - -const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } - /*-************************************************************** * FSE NCount encoding-decoding diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index bcd6f022d..79e6fbae3 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -71,8 +71,6 @@ /* ************************************************************** * Error Management ****************************************************************/ -unsigned HUF_isError(size_t code) { return ERR_isError(code); } -const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index fef32f76d..faa813707 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -245,8 +245,5 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, /* ************************************* * Tool functions ***************************************/ -unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } -const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } - size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; } size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6515c68bf..d3232ca73 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -94,18 +94,6 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } ***************************************/ unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } -/*! ZSTD_isError() : -* tells if a return value is an error code */ -unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } - -/*! ZSTD_getError() : -* convert a `size_t` function result into a proper ZSTD_errorCode enum */ -ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); } - -/*! ZSTD_getErrorName() : -* provides error code string (useful for debugging) */ -const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } - /*-************************************************************* * Context management @@ -867,7 +855,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, BYTE* op = ostart; BYTE* const oend = ostart + dstCapacity; size_t remainingSize = srcSize; - blockProperties_t blockProperties; + blockProperties_t blockProperties = {0}; /* check */ if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); diff --git a/programs/Makefile b/programs/Makefile index e0a26127c..094026c6b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -53,8 +53,8 @@ BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c +ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c +ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/common/zstd_common.c ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) @@ -87,7 +87,7 @@ ZSTDRTTEST= --test-large-data default: zstd -all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen +all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32 zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c @@ -114,6 +114,9 @@ zstd-pgo : clean zstd zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) +zstd-compress: $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) + zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) From db3964382cb55fce39848cdc4caa66e5f954ba6b Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 22 Apr 2016 18:22:30 +0200 Subject: [PATCH 14/26] introduced ZSTD_NODECOMPRESS to link only compressor --- lib/common/zstd_common.c | 107 +++++++++++++++++++++++++++++-- lib/decompress/fse_decompress.c | 100 ----------------------------- lib/decompress/zstd_decompress.c | 2 +- programs/Makefile | 2 +- programs/fileio.c | 4 ++ programs/zstdcli.c | 6 ++ 6 files changed, 112 insertions(+), 109 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index c833bdacf..1393b7b5c 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -34,16 +34,11 @@ * Dependencies ***************************************/ #include +#include "mem.h" +#include "fse_static.h" /* FSE_MIN_TABLELOG */ #include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ -#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ -#include "zbuff_static.h" -/* ************************************* -* Constants -***************************************/ - /*-**************************************** * ZSTD Error Management @@ -83,3 +78,101 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static short FSE_abs(short a) { return a<0 ? -a : a; } + +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<1) && (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0+=24; + if (ip < iend-5) { + ip+=2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount+=16; + } } + while ((bitStream & 3) == 3) { + n0+=3; + bitStream>>=2; + bitCount+=2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } + else + bitStream >>= 2; + } + { short const max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } else { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSE_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } + if (remaining != 1) return ERROR(GENERIC); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); + return ip-istart; +} diff --git a/lib/decompress/fse_decompress.c b/lib/decompress/fse_decompress.c index ec699ab91..698dbfc24 100644 --- a/lib/decompress/fse_decompress.c +++ b/lib/decompress/fse_decompress.c @@ -170,106 +170,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned #ifndef FSE_COMMONDEFS_ONLY -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -static short FSE_abs(short a) { return a<0 ? -a : a; } - -size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, - const void* headerBuffer, size_t hbSize) -{ - const BYTE* const istart = (const BYTE*) headerBuffer; - const BYTE* const iend = istart + hbSize; - const BYTE* ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) return ERROR(srcSize_wrong); - bitStream = MEM_readLE32(ip); - nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1<1) && (charnum<=*maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0+=24; - if (ip < iend-5) { - ip+=2; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount+=16; - } } - while ((bitStream & 3) == 3) { - n0+=3; - bitStream>>=2; - bitCount+=2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; - } - else - bitStream >>= 2; - } - { short const max = (short)((2*threshold-1)-remaining); - short count; - - if ((bitStream & (threshold-1)) < (U32)max) { - count = (short)(bitStream & (threshold-1)); - bitCount += nbBits-1; - } else { - count = (short)(bitStream & (2*threshold-1)); - if (count >= threshold) count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= FSE_abs(count); - normalizedCounter[charnum++] = count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } - if (remaining != 1) return ERROR(GENERIC); - *maxSVPtr = charnum-1; - - ip += (bitCount+7)>>3; - if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); - return ip-istart; -} - - - - /*-******************************************************* * Decompression (Byte symbols) *********************************************************/ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d3232ca73..c34a7eb22 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -855,7 +855,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, BYTE* op = ostart; BYTE* const oend = ostart + dstCapacity; size_t remainingSize = srcSize; - blockProperties_t blockProperties = {0}; + blockProperties_t blockProperties = { bt_compressed, 0 }; /* check */ if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); diff --git a/programs/Makefile b/programs/Makefile index 094026c6b..d54d3fd25 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -53,7 +53,7 @@ BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c +ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/decompress/fse_decompress.c ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/common/zstd_common.c ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c diff --git a/programs/fileio.c b/programs/fileio.c index e0ccee7f8..1d86494cd 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -504,6 +504,9 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile #endif // #ifndef ZSTD_NOCOMPRESS + +#ifndef ZSTD_NODECOMPRESS + /* ************************************************************************** * Decompression ****************************************************************************/ @@ -721,3 +724,4 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles return missingFiles + skippedFiles; } +#endif // #ifndef ZSTD_NODECOMPRESS diff --git a/programs/zstdcli.c b/programs/zstdcli.c index feaddc339..33e3992c3 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -114,7 +114,9 @@ static int usage(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( " -# : # compression level (1-%u, default:1) \n", ZSTD_maxCLevel()); #endif +#ifndef ZSTD_NODECOMPRESS DISPLAY( " -d : decompression \n"); +#endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); DISPLAY( " -f : overwrite output without prompting \n"); @@ -433,10 +435,14 @@ int main(int argCount, const char** argv) } else #endif { /* decompression */ +#ifndef ZSTD_NODECOMPRESS if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); else operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName); +#else + DISPLAY("Decompression not supported\n"); +#endif } _end: From f0668169984d3f205a45bf966ab41b8a4be75cee Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 22 Apr 2016 18:54:05 +0200 Subject: [PATCH 15/26] fix for g++ compilation --- lib/common/zstd_common.c | 5 ++++- programs/Makefile | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 1393b7b5c..679e3ace6 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -37,7 +37,10 @@ #include "mem.h" #include "fse_static.h" /* FSE_MIN_TABLELOG */ #include "error_private.h" - +#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */ +#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ +#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ +#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ /*-**************************************** diff --git a/programs/Makefile b/programs/Makefile index d54d3fd25..acf6a562d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -154,7 +154,7 @@ datagen : datagen.c datagencli.c clean: @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ - zstd$(EXT) zstd32$(EXT) \ + zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) From 49794316c2cb290e5082c16490c20406bf30b4c8 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 25 Apr 2016 11:31:28 +0200 Subject: [PATCH 16/26] updated lib/README.md --- lib/README.md | 61 ++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/lib/README.md b/lib/README.md index ef446427a..80e8b98de 100644 --- a/lib/README.md +++ b/lib/README.md @@ -5,53 +5,58 @@ The __lib__ directory contains several files, but depending on target use case, #### Minimal library files -##### Shared ressources +To build the zstd library the following files are required: -- [mem.h](mem.h) -- [error_private.h](error_private.h) -- [error_public.h](error_public.h) - -##### zstd core compression +- [common/bitstream.h](common/bitstream.h) +- [common/error_private.h](common/error_private.h) +- [common/error_public.h](common/error_public.h) +- common/fse.h +- common/fse_static.h +- common/huf.h +- common/huf_static.h +- [common/mem.h](common/mem.h) +- [common/zstd.h](common/zstd.h) +- common/zstd_internal.h +- common/zstd_static.h +- compress/fse_compress.c +- compress/huf_compress.c +- compress/zstd_compress.c +- compress/zstd_opt.h +- decompress/fse_decompress.c +- decompress/huf_decompress.c +- decompress/zstd_decompress.c Stable API is exposed in [zstd.h]. Advanced and experimental API is exposed in `zstd_static.h`. `zstd_static.h` API elements should be used with static linking only, as their definition may change in future version of the library. -- [bitstream.h](bitstream.h) -- fse.c -- fse.h -- fse_static.h -- huf.c -- huf.h -- huf_static.h -- zstd_compress.c -- zstd_decompress.c -- zstd_internal.h -- zstd_opt.h -- [zstd.h] -- zstd_static.h -[zstd.h]: zstd.h +#### Separate compressor and decompressor + +To build a separate zstd compressor all files from common/ and compressor/ directories are required. +In similar way to build a separate zstd decompressor all files from common/ and decompressor/ directories are needed. + #### Buffered streaming This complementary API makes streaming integration easier. It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) : -- zbuff.c -- zbuff.h -- zbuff_static.h +- common/zbuff.h +- common/zbuff_static.h +- compress/zbuff_compress.c +- decompress/zbuff_decompress.c #### Dictionary builder To create dictionaries from training sets : -- divsufsort.c -- divsufsort.h -- zdict.c -- zdict.h -- zdict_static.h +- dictBuilder/divsufsort.c +- dictBuilder/divsufsort.h +- dictBuilder/zdict.c +- dictBuilder/zdict.h +- dictBuilder/zdict_static.h #### Miscellaneous From a8138fd767f8fa568ad97ea7e52d5303ce3012f0 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 25 Apr 2016 11:36:44 +0200 Subject: [PATCH 17/26] updated lib/README.md part 2 --- lib/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/README.md b/lib/README.md index 80e8b98de..6002118e0 100644 --- a/lib/README.md +++ b/lib/README.md @@ -15,7 +15,7 @@ To build the zstd library the following files are required: - common/huf.h - common/huf_static.h - [common/mem.h](common/mem.h) -- [common/zstd.h](common/zstd.h) +- [common/zstd.h] - common/zstd_internal.h - common/zstd_static.h - compress/fse_compress.c @@ -26,16 +26,18 @@ To build the zstd library the following files are required: - decompress/huf_decompress.c - decompress/zstd_decompress.c -Stable API is exposed in [zstd.h]. -Advanced and experimental API is exposed in `zstd_static.h`. -`zstd_static.h` API elements should be used with static linking only, +Stable API is exposed in [common/zstd.h]. +Advanced and experimental API is exposed in `common/zstd_static.h`. +`common/zstd_static.h` API elements should be used with static linking only, as their definition may change in future version of the library. +[common/zstd.h]: common/zstd.h + #### Separate compressor and decompressor -To build a separate zstd compressor all files from common/ and compressor/ directories are required. -In similar way to build a separate zstd decompressor all files from common/ and decompressor/ directories are needed. +To build a separate zstd compressor all files from `common/` and `compressor/` directories are required. +In a similar way to build a separate zstd decompressor all files from `common/` and `decompressor/` directories are needed. #### Buffered streaming From ea4ee3eee0f9e89a3937c2e46f1a36eb5c0313a3 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 25 Apr 2016 13:09:06 +0200 Subject: [PATCH 18/26] added compatibility with Visual C++ 2012 --- lib/README.md | 5 +- lib/compress/zstd_opt.h | 28 +-- lib/legacy/zstd_v05.c | 12 +- programs/bench.c | 15 +- visual/2013/fullbench/fullbench.vcxproj | 39 ++-- .../2013/fullbench/fullbench.vcxproj.filters | 95 +++++---- visual/2013/fuzzer/fuzzer.vcxproj | 40 ++-- visual/2013/fuzzer/fuzzer.vcxproj.filters | 106 +++++----- visual/2013/zstd/zstd.vcxproj | 57 ++--- visual/2013/zstd/zstd.vcxproj.filters | 195 +++++++++--------- visual/2013/zstdlib/zstdlib.vcxproj | 57 ++--- visual/2013/zstdlib/zstdlib.vcxproj.filters | 81 +++++--- 12 files changed, 380 insertions(+), 350 deletions(-) diff --git a/lib/README.md b/lib/README.md index 6002118e0..84e17a2f1 100644 --- a/lib/README.md +++ b/lib/README.md @@ -27,11 +27,12 @@ To build the zstd library the following files are required: - decompress/zstd_decompress.c Stable API is exposed in [common/zstd.h]. -Advanced and experimental API is exposed in `common/zstd_static.h`. -`common/zstd_static.h` API elements should be used with static linking only, +Advanced and experimental API is exposed in [common/zstd_static.h]. +API elements of [common/zstd_static.h] should be used with static linking only, as their definition may change in future version of the library. [common/zstd.h]: common/zstd.h +[common/zstd_static.h]: common/zstd_static.h #### Separate compressor and decompressor diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index ae09a2693..86fa13067 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -196,9 +196,10 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B } /* match offset */ - seqStorePtr->offCodeSum++; - BYTE offCode = (BYTE)ZSTD_highbit(offset+1); - seqStorePtr->offCodeFreq[offCode]++; + { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + seqStorePtr->offCodeSum++; + seqStorePtr->offCodeFreq[offCode]++; + } /* match Length */ { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, @@ -462,7 +463,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, const BYTE* inr; /* init */ - U32 rep[ZSTD_REP_INIT]; + U32 offset, rep[ZSTD_REP_INIT]; { U32 i; for (i=0; inextToUpdate3 = ctx->nextToUpdate; @@ -475,7 +476,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* Match Loop */ while (ip < ilimit) { U32 cur, match_num, last_pos, litlen, price; - U32 u, mlen, best_mlen, best_off; + U32 u, mlen, best_mlen, best_off, litLength; memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; litlen = (U32)(ip - anchor); @@ -650,7 +651,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); opt[0].mlen = 1; - U32 offset; while (1) { mlen = opt[cur].mlen; @@ -674,8 +674,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - - U32 const litLength = (U32)(ip - anchor); + litLength = (U32)(ip - anchor); // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); if (offset >= ZSTD_REP_NUM) { @@ -746,10 +745,9 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; const BYTE* inr; - U32 cur, match_num, last_pos, litlen, price; /* init */ - U32 rep[ZSTD_REP_INIT]; + U32 offset, rep[ZSTD_REP_INIT]; { U32 i; for (i=0; inextToUpdate3 = ctx->nextToUpdate; @@ -761,10 +759,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* Match Loop */ while (ip < ilimit) { - U32 u; - U32 mlen=0; - U32 best_mlen=0; - U32 best_off=0; + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; U32 current = (U32)(ip-base); memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; @@ -960,7 +956,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); opt[0].mlen = 1; - U32 offset; while (1) { mlen = opt[cur].mlen; @@ -984,8 +979,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - - U32 const litLength = (U32)(ip - anchor); + litLength = (U32)(ip - anchor); // ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); if (offset >= ZSTD_REP_NUM) { diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index d50724793..75dc2a926 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2372,9 +2372,6 @@ size_t HUFv05_decompress4X2_usingDTable( const U32 dtLog = DTable[0]; size_t errorCode; - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - /* Init */ BITv05_DStream_t bitD1; BITv05_DStream_t bitD2; @@ -2398,6 +2395,9 @@ size_t HUFv05_decompress4X2_usingDTable( BYTE* op4 = opStart4; U32 endSignal; + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ errorCode = BITv05_initDStream(&bitD1, istart1, length1); @@ -3124,9 +3124,6 @@ size_t HUFv05_decompress4X6_usingDTable( const HUFv05_DSeqX6* ds = (const HUFv05_DSeqX6*)dsPtr; size_t errorCode; - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - /* Init */ BITv05_DStream_t bitD1; BITv05_DStream_t bitD2; @@ -3150,6 +3147,9 @@ size_t HUFv05_decompress4X6_usingDTable( BYTE* op4 = opStart4; U32 endSignal; + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ errorCode = BITv05_initDStream(&bitD1, istart1, length1); diff --git a/programs/bench.c b/programs/bench.c index 8470ed324..33c58c16e 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -209,12 +209,13 @@ static U32 BMK_isDirectory(const char* infilename) #if defined(_MSC_VER) struct _stat64 statbuf; r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; #else struct stat statbuf; r = stat(infilename, &statbuf); -#endif if (!r && S_ISDIR(statbuf.st_mode)) return 1; - return 0; +#endif + return 0; } /* ******************************************************** @@ -467,12 +468,12 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, benchResult_t result, total; int l; - SET_HIGH_PRIORITY; - const char* pch = strrchr(displayName, '\\'); /* Windows */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; + SET_HIGH_PRIORITY; + memset(&result, 0, sizeof(result)); memset(&total, 0, sizeof(total)); @@ -522,15 +523,15 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, const char** fileNamesTable, unsigned nbFiles) { size_t pos = 0, totalSize = 0; - + FILE* f; unsigned n; for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ diff --git a/visual/2013/fullbench/fullbench.vcxproj b/visual/2013/fullbench/fullbench.vcxproj index 8252d03d0..6b878a99d 100644 --- a/visual/2013/fullbench/fullbench.vcxproj +++ b/visual/2013/fullbench/fullbench.vcxproj @@ -28,8 +28,8 @@ Application true - v120 Unicode + v110 Application @@ -69,7 +69,7 @@ true - $(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) true @@ -160,27 +160,30 @@ - - - - - + + + + + + + + + - - - - + + + + + + + + + + - - - - - - - diff --git a/visual/2013/fullbench/fullbench.vcxproj.filters b/visual/2013/fullbench/fullbench.vcxproj.filters index 1d5f9c59b..323b6446e 100644 --- a/visual/2013/fullbench/fullbench.vcxproj.filters +++ b/visual/2013/fullbench/fullbench.vcxproj.filters @@ -1,81 +1,86 @@  - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - + {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hpp;hxx;hm;inl;inc;xsd - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - Fichiers sources + + Source Files + + + Source Files + + + Source Files - Fichiers sources + Source Files - Fichiers sources + Source Files - - Fichiers sources + + Source Files - - Fichiers sources + + Source Files - - Fichiers sources + + Source Files - - Fichiers sources + + Source Files + + + Source Files + + + Source Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête + + Header Files - Fichiers d%27en-tête + Header Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête - - - Fichiers d%27en-tête + + Header Files - Fichiers d%27en-tête + Header Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête + + Header Files \ No newline at end of file diff --git a/visual/2013/fuzzer/fuzzer.vcxproj b/visual/2013/fuzzer/fuzzer.vcxproj index 49738e0ca..4b1875a9f 100644 --- a/visual/2013/fuzzer/fuzzer.vcxproj +++ b/visual/2013/fuzzer/fuzzer.vcxproj @@ -1,4 +1,4 @@ - + @@ -28,8 +28,8 @@ Application true - v120 Unicode + v110 Application @@ -69,7 +69,7 @@ true - $(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) true @@ -160,29 +160,29 @@ - - - - - - - + + + + + + + - - - - + + + + + + + + + + - - - - - - diff --git a/visual/2013/fuzzer/fuzzer.vcxproj.filters b/visual/2013/fuzzer/fuzzer.vcxproj.filters index 500da8756..3e0cbbedc 100644 --- a/visual/2013/fuzzer/fuzzer.vcxproj.filters +++ b/visual/2013/fuzzer/fuzzer.vcxproj.filters @@ -1,90 +1,80 @@  - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - + {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hpp;hxx;hm;inl;inc;xsd - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - Fichiers sources - - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources + Source Files - - Fichiers sources + + Source Files - - Fichiers sources + + Source Files - - Fichiers sources + + Source Files - - Fichiers sources + + Source Files - - Fichiers sources - - - Fichiers sources + + Source Files - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête + Header Files - - Fichiers d%27en-tête + + Header Files - - Fichiers d%27en-tête + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files \ No newline at end of file diff --git a/visual/2013/zstd/zstd.vcxproj b/visual/2013/zstd/zstd.vcxproj index cced867e8..7b2ebf6bb 100755 --- a/visual/2013/zstd/zstd.vcxproj +++ b/visual/2013/zstd/zstd.vcxproj @@ -1,4 +1,4 @@ - + @@ -19,18 +19,22 @@ - - - + + + + + + + + + + + - - - - @@ -40,26 +44,25 @@ - - - - - + + + + + + + + + + + + + - - - - - - - - - @@ -78,7 +81,7 @@ Application true Unicode - v120 + v110 Application @@ -118,22 +121,22 @@ true - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true true - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false false - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false diff --git a/visual/2013/zstd/zstd.vcxproj.filters b/visual/2013/zstd/zstd.vcxproj.filters index eb8943423..45f486fd1 100755 --- a/visual/2013/zstd/zstd.vcxproj.filters +++ b/visual/2013/zstd/zstd.vcxproj.filters @@ -1,156 +1,161 @@  - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - + {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hpp;hxx;hm;inl;inc;xsd - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - Fichiers sources - - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources - - - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources - - - Fichiers sources - - - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources - - - Fichiers sources - - - Fichiers sources - - - Fichiers sources + Source Files - Fichiers sources + Source Files - Fichiers sources + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête - - - Fichiers d%27en-tête - - - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête + Header Files - Fichiers d%27en-tête - - - Fichiers sources - - - Fichiers sources - - - Fichiers sources - - - Fichiers sources - - - Fichiers sources + Header Files - Fichiers d%27en-tête + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files - Fichiers d%27en-tête + Header Files \ No newline at end of file diff --git a/visual/2013/zstdlib/zstdlib.vcxproj b/visual/2013/zstdlib/zstdlib.vcxproj index e44699d4a..89133f012 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj +++ b/visual/2013/zstdlib/zstdlib.vcxproj @@ -1,5 +1,5 @@ - - + + Debug @@ -19,24 +19,31 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -53,7 +60,7 @@ DynamicLibrary true Unicode - v120 + v110 DynamicLibrary @@ -95,28 +102,28 @@ true zstdlib_x86 $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true true zstdlib_x64 $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false zstdlib_x86 $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false false zstdlib_x64 $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false diff --git a/visual/2013/zstdlib/zstdlib.vcxproj.filters b/visual/2013/zstdlib/zstdlib.vcxproj.filters index ffb457b0b..fa5bb166a 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj.filters +++ b/visual/2013/zstdlib/zstdlib.vcxproj.filters @@ -15,63 +15,84 @@ - + Source Files - + Source Files - + Source Files - + Source Files - + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + Source Files - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + Header Files From 6cb083fe3d097bed68bf2d81f56b403f2694c2d7 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 25 Apr 2016 14:42:15 +0200 Subject: [PATCH 19/26] fixed 64-bit compilation with Visual C++ --- .gitattributes | 5 +- lib/compress/zstd_compress.c | 4 +- visual/2013/fullbench/fullbench.vcxproj | 382 ++++++++++----------- visual/2013/fuzzer/fuzzer.vcxproj | 382 ++++++++++----------- visual/2013/zstd.sln | 10 +- visual/2013/zstd/zstd.vcxproj | 436 ++++++++++++------------ visual/2013/zstdlib/zstdlib.rc | Bin 5204 -> 5196 bytes visual/2013/zstdlib/zstdlib.vcxproj | 436 ++++++++++++------------ 8 files changed, 828 insertions(+), 827 deletions(-) diff --git a/.gitattributes b/.gitattributes index b9d54fbfd..b36a354d7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,8 +8,9 @@ # Denote files that should not be modified. *.odt binary *.png binary + # Visual Studio *.sln binary *.suo binary -*.vcxproj* binary - +*.vcxproj* text eol=crlf +*.rc text eol=crlf diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a8ebaa549..6e75d10ed 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -218,7 +218,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t maxNbSeq = blockSize / divider; const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); - const size_t hSize = 1 << params.cParams.hashLog; + const size_t hSize = (size_t)(1 << params.cParams.hashLog); const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -291,7 +291,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) /* copy tables */ { const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); - const size_t hSize = 1 << srcCCtx->params.cParams.hashLog; + const size_t hSize = (size_t)(1 << srcCCtx->params.cParams.hashLog); const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); diff --git a/visual/2013/fullbench/fullbench.vcxproj b/visual/2013/fullbench/fullbench.vcxproj index 6b878a99d..e98766713 100644 --- a/visual/2013/fullbench/fullbench.vcxproj +++ b/visual/2013/fullbench/fullbench.vcxproj @@ -1,192 +1,192 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8} - Win32Proj - fullbench - $(SolutionDir)bin\$(Platform)\$(Configuration)\ - - - - Application - true - Unicode - v110 - - - Application - true - v120 - Unicode - - - Application - false - v120 - true - Unicode - - - Application - false - v120 - true - Unicode - - - - - - - - - - - - - - - - - - - true - $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) - true - - - true - $(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8} + Win32Proj + fullbench + $(SolutionDir)bin\$(Platform)\$(Configuration)\ + + + + Application + true + Unicode + v110 + + + Application + true + v110 + Unicode + + + Application + false + v110 + true + Unicode + + + Application + false + v110 + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + true + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + false + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + false + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/2013/fuzzer/fuzzer.vcxproj b/visual/2013/fuzzer/fuzzer.vcxproj index 4b1875a9f..19f9cabeb 100644 --- a/visual/2013/fuzzer/fuzzer.vcxproj +++ b/visual/2013/fuzzer/fuzzer.vcxproj @@ -1,192 +1,192 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {6FD4352B-346C-4703-96EA-D4A8B9A6976E} - Win32Proj - fuzzer - $(SolutionDir)bin\$(Platform)\$(Configuration)\ - - - - Application - true - Unicode - v110 - - - Application - true - v120 - Unicode - - - Application - false - v120 - true - Unicode - - - Application - false - v120 - true - Unicode - - - - - - - - - - - - - - - - - - - true - $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) - true - - - true - $(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {6FD4352B-346C-4703-96EA-D4A8B9A6976E} + Win32Proj + fuzzer + $(SolutionDir)bin\$(Platform)\$(Configuration)\ + + + + Application + true + Unicode + v110 + + + Application + true + v110 + Unicode + + + Application + false + v110 + true + Unicode + + + Application + false + v110 + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + true + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + false + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + false + $(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/2013/zstd.sln b/visual/2013/zstd.sln index 3186fc670..e3d56b9d9 100644 --- a/visual/2013/zstd.sln +++ b/visual/2013/zstd.sln @@ -13,20 +13,20 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib. EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Release|x64 = Release|x64 Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 - Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.ActiveCfg = Release|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.Build.0 = Release|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.ActiveCfg = Release|Win32 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.Build.0 = Release|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.ActiveCfg = Debug|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.Build.0 = Debug|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.ActiveCfg = Debug|x64 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.Build.0 = Debug|x64 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.ActiveCfg = Release|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.Build.0 = Release|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.ActiveCfg = Release|x64 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.Build.0 = Release|x64 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.ActiveCfg = Debug|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.Build.0 = Debug|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|x64.ActiveCfg = Debug|x64 diff --git a/visual/2013/zstd/zstd.vcxproj b/visual/2013/zstd/zstd.vcxproj index 7b2ebf6bb..8e2c47798 100755 --- a/visual/2013/zstd/zstd.vcxproj +++ b/visual/2013/zstd/zstd.vcxproj @@ -1,219 +1,219 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C} - Win32Proj - zstd - $(SolutionDir)bin\$(Platform)\$(Configuration)\ - - - - Application - true - Unicode - v110 - - - Application - true - v120 - Unicode - - - Application - false - true - v120_xp - Unicode - - - Application - false - true - v120 - Unicode - - - - - - - - - - - - - - - - - - - true - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - true - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - false - - - false - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - false - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - true - /analyze:stacksize19000 %(AdditionalOptions) - - - Console - true - - - - - Level4 - - - Full - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - true - MultiThreaded - - - Console - true - true - true - setargv.obj; kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - false - MultiThreaded - - - Console - true - true - true - setargv.obj;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C} + Win32Proj + zstd + $(SolutionDir)bin\$(Platform)\$(Configuration)\ + + + + Application + true + Unicode + v110 + + + Application + true + v110 + Unicode + + + Application + false + true + v110 + Unicode + + + Application + false + true + v110 + Unicode + + + + + + + + + + + + + + + + + + + true + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + true + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + false + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize19000 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + Full + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + MultiThreaded + + + Console + true + true + true + setargv.obj; kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + setargv.obj;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + \ No newline at end of file diff --git a/visual/2013/zstdlib/zstdlib.rc b/visual/2013/zstdlib/zstdlib.rc index 93e221d9f59c8e72720b0094614e27b8925e42ba..6c0fb2fc7194e3adf0031ce1ea344e5c58e7a386 100644 GIT binary patch delta 57 zcmcbjaYkc92*+eQPPWMhxWpzGaPe%e0FoN}8BI9eD%jL8CA31Iph7u)1XTnZre=5t(ioB)1H5A*;4 diff --git a/visual/2013/zstdlib/zstdlib.vcxproj b/visual/2013/zstdlib/zstdlib.vcxproj index 89133f012..76944cf4d 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj +++ b/visual/2013/zstdlib/zstdlib.vcxproj @@ -1,219 +1,219 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} - Win32Proj - zstdlib - $(SolutionDir)bin\$(Platform)\$(Configuration)\ - - - - DynamicLibrary - true - Unicode - v110 - - - DynamicLibrary - true - v120 - Unicode - - - DynamicLibrary - false - true - v120 - Unicode - - - DynamicLibrary - false - true - v120 - Unicode - - - - - - - - - - - - - - - - - - - true - zstdlib_x86 - $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - true - zstdlib_x64 - $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - zstdlib_x86 - $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - false - - - false - zstdlib_x64 - $(Platform)\$(Configuration)\ - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - false - - - - - - Level4 - Disabled - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebugDLL - EditAndContinue - true - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - MachineX86 - - - - - - - Level4 - Disabled - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - false - EnableFastChecks - MultiThreadedDebugDLL - ProgramDatabase - true - /analyze:stacksize25000 %(AdditionalOptions) - - - Console - true - - - - - Level4 - - - MaxSpeed - true - true - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - true - /analyze:stacksize25000 %(AdditionalOptions) - MultiThreadedDLL - ProgramDatabase - All - - - Console - true - true - true - MachineX86 - - - - - Level4 - - - MaxSpeed - true - true - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - false - false - MultiThreadedDLL - ProgramDatabase - true - true - All - - - Console - true - true - true - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} + Win32Proj + zstdlib + $(SolutionDir)bin\$(Platform)\$(Configuration)\ + + + + DynamicLibrary + true + Unicode + v110 + + + DynamicLibrary + true + v110 + Unicode + + + DynamicLibrary + false + true + v110 + Unicode + + + DynamicLibrary + false + true + v110 + Unicode + + + + + + + + + + + + + + + + + + + true + zstdlib_x86 + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + true + zstdlib_x64 + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + zstdlib_x86 + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + false + zstdlib_x64 + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + + + + Level4 + Disabled + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + EditAndContinue + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + MachineX86 + + + + + + + Level4 + Disabled + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + false + EnableFastChecks + MultiThreadedDebugDLL + ProgramDatabase + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + /analyze:stacksize25000 %(AdditionalOptions) + MultiThreadedDLL + ProgramDatabase + All + + + Console + true + true + true + MachineX86 + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + false + false + MultiThreadedDLL + ProgramDatabase + true + true + All + + + Console + true + true + true + + + + + \ No newline at end of file From 1007a1fe3dd5cdc5cc3dab76d0d3861b2ecfa18a Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 25 Apr 2016 15:23:09 +0200 Subject: [PATCH 20/26] get rid of some Visual C++ warnings --- .gitattributes | 6 +++--- lib/compress/zstd_compress.c | 4 ++-- visual/2013/zstd/zstd.vcxproj | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index b36a354d7..3959727b8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,7 +10,7 @@ *.png binary # Visual Studio -*.sln binary -*.suo binary +*.sln text eol=crlf *.vcxproj* text eol=crlf -*.rc text eol=crlf +*.suo binary +*.rc binary diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 6e75d10ed..7b07eb454 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -218,7 +218,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t maxNbSeq = blockSize / divider; const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); - const size_t hSize = (size_t)(1 << params.cParams.hashLog); + const size_t hSize = ((size_t)1) << params.cParams.hashLog; const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -291,7 +291,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) /* copy tables */ { const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); - const size_t hSize = (size_t)(1 << srcCCtx->params.cParams.hashLog); + const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); diff --git a/visual/2013/zstd/zstd.vcxproj b/visual/2013/zstd/zstd.vcxproj index 8e2c47798..d31994d99 100755 --- a/visual/2013/zstd/zstd.vcxproj +++ b/visual/2013/zstd/zstd.vcxproj @@ -164,7 +164,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) false true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console From 69fcd7c0ae4c7423658e935dd322293a7a4a5e7c Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 28 Apr 2016 12:23:33 +0200 Subject: [PATCH 21/26] getFileSize moved to common/util.h --- lib/common/util.h | 101 ++++++ lib/dictBuilder/zdict.c | 7 - programs/bench.c | 43 +-- programs/dibio.c | 28 +- programs/fileio.c | 24 +- programs/fullbench.c | 27 +- programs/paramgrill.c | 30 +- visual/2013/zstd.sln | 116 +++---- visual/2013/zstd/zstd.vcxproj.filters | 320 ++++++++++---------- visual/2013/zstdlib/zstdlib.vcxproj.filters | 206 ++++++------- 10 files changed, 440 insertions(+), 462 deletions(-) create mode 100644 lib/common/util.h diff --git a/lib/common/util.h b/lib/common/util.h new file mode 100644 index 000000000..1d2840dc0 --- /dev/null +++ b/lib/common/util.h @@ -0,0 +1,101 @@ +/* ****************************************************************** + util.h + utility functions + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef UTIL_H_MODULE +#define UTIL_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline +#else +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Utility functions +******************************************/ +UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (U64)statbuf.st_size; +} + + +UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + +#if defined (__cplusplus) +} +#endif + +#endif /* UTIL_H_MODULE */ + diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 2ebf2b49c..74d5f36a9 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -67,13 +67,6 @@ #include "zdict_static.h" -/*-************************************* -* Compiler specifics -***************************************/ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - /*-************************************* * Constants diff --git a/programs/bench.c b/programs/bench.c index 33c58c16e..77d7c533a 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -86,17 +86,14 @@ #include "mem.h" #include "zstd_static.h" -#include "datagen.h" /* RDG_genBuffer */ +#include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" +#include "util.h" /* UTIL_GetFileSize */ /* ************************************* * Compiler specifics ***************************************/ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - #if defined(_MSC_VER) # define snprintf sprintf_s #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) @@ -189,34 +186,6 @@ static U64 BMK_clockSpan( BMK_time_t clockStart, BMK_time_t ticksPerSecond ) return BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); } -static U64 BMK_getFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - -static U32 BMK_isDirectory(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; -} /* ******************************************************** * Bench functions @@ -511,7 +480,7 @@ static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) U64 total = 0; unsigned n; for (n=0; n 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName); dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); diff --git a/programs/dibio.c b/programs/dibio.c index 9641ac299..0e8324103 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -53,14 +53,7 @@ #include "mem.h" /* read */ #include "error_private.h" #include "dibio.h" - - -/*-************************************* -* Compiler specifics -***************************************/ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif +#include "util.h" /* UTIL_GetFileSize */ /*-************************************* @@ -115,27 +108,12 @@ const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCo /* ******************************************************** * File related operations **********************************************************/ -static unsigned long long DiB_getFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (unsigned long long)statbuf.st_size; -} - - static unsigned long long DiB_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) { unsigned long long total = 0; unsigned n; for (n=0; n MAX_DICT_SIZE) { int seekResult; if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */ @@ -333,7 +315,7 @@ static int FIO_compressFilename_internal(cRess_t ress, size_t dictSize = ress.dictBufferSize; size_t sizeCheck; ZSTD_parameters params; - U64 const fileSize = FIO_getFileSize(srcFileName); + U64 const fileSize = UTIL_getFileSize(srcFileName); /* init */ params.cParams = ZSTD_getCParams(cLevel, fileSize, dictSize); diff --git a/programs/fullbench.c b/programs/fullbench.c index d9bce181f..08ff8973f 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -53,15 +53,7 @@ #include "fse_static.h" #include "zbuff.h" #include "datagen.h" - - -/*_************************************ -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif +#include "util.h" /* UTIL_GetFileSize */ /*_************************************ @@ -135,21 +127,6 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - /*_******************************************************* * Benchmark wrappers *********************************************************/ @@ -446,7 +423,7 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory allocation & restrictions */ - inFileSize = BMK_GetFileSize(inFileName); + inFileSize = UTIL_getFileSize(inFileName); benchedSize = BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 3dcb2e102..21f695240 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -37,7 +37,7 @@ # define _LARGEFILE64_SOURCE #endif -/* S_ISREG & gettimeofday() are not supported by MSVC */ +/* gettimeofday() are not supported by MSVC */ #if defined(_MSC_VER) || defined(_WIN32) # define BMK_LEGACY_TIMER 1 #endif @@ -68,15 +68,7 @@ #include "zstd_static.h" #include "datagen.h" #include "xxhash.h" - - -/*-************************************ -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif +#include "util.h" /* UTIL_GetFileSize */ /*-************************************ @@ -197,20 +189,6 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - # define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) U32 FUZ_rand(U32* src) { @@ -790,7 +768,7 @@ int benchFiles(char** fileNamesTable, int nbFiles) } /* Memory allocation & restrictions */ - inFileSize = BMK_GetFileSize(inFileName); + inFileSize = UTIL_getFileSize(inFileName); benchedSize = BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) @@ -841,7 +819,7 @@ int optimizeForSize(char* inFileName) } /* Memory allocation & restrictions */ - inFileSize = BMK_GetFileSize(inFileName); + inFileSize = UTIL_getFileSize(inFileName); benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) diff --git a/visual/2013/zstd.sln b/visual/2013/zstd.sln index e3d56b9d9..97d52ceb5 100644 --- a/visual/2013/zstd.sln +++ b/visual/2013/zstd.sln @@ -1,58 +1,58 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.40629.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcxproj", "{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{6FD4352B-346C-4703-96EA-D4A8B9A6976E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Release|x64 = Release|x64 - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.ActiveCfg = Release|x64 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.Build.0 = Release|x64 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.ActiveCfg = Release|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.Build.0 = Release|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.ActiveCfg = Debug|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.Build.0 = Debug|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.ActiveCfg = Debug|x64 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.Build.0 = Debug|x64 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.ActiveCfg = Debug|Win32 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.Build.0 = Debug|Win32 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|x64.ActiveCfg = Debug|x64 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|x64.Build.0 = Debug|x64 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Win32.ActiveCfg = Release|Win32 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Win32.Build.0 = Release|Win32 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|x64.ActiveCfg = Release|x64 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|x64.Build.0 = Release|x64 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Win32.ActiveCfg = Debug|Win32 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Win32.Build.0 = Debug|Win32 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|x64.ActiveCfg = Debug|x64 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|x64.Build.0 = Debug|x64 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.ActiveCfg = Release|Win32 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.Build.0 = Release|Win32 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.ActiveCfg = Release|x64 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.Build.0 = Release|x64 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.ActiveCfg = Debug|Win32 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.Build.0 = Debug|Win32 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.ActiveCfg = Debug|x64 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.Build.0 = Debug|x64 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.ActiveCfg = Release|Win32 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64 - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.40629.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcxproj", "{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{6FD4352B-346C-4703-96EA-D4A8B9A6976E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Release|x64 = Release|x64 + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.ActiveCfg = Release|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.Build.0 = Release|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.ActiveCfg = Release|Win32 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.Build.0 = Release|Win32 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.ActiveCfg = Debug|Win32 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.Build.0 = Debug|Win32 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.ActiveCfg = Debug|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.Build.0 = Debug|x64 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.ActiveCfg = Debug|Win32 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.Build.0 = Debug|Win32 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|x64.ActiveCfg = Debug|x64 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|x64.Build.0 = Debug|x64 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Win32.ActiveCfg = Release|Win32 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Win32.Build.0 = Release|Win32 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|x64.ActiveCfg = Release|x64 + {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|x64.Build.0 = Release|x64 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Win32.ActiveCfg = Debug|Win32 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Win32.Build.0 = Debug|Win32 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|x64.ActiveCfg = Debug|x64 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|x64.Build.0 = Debug|x64 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.ActiveCfg = Release|Win32 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.Build.0 = Release|Win32 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.ActiveCfg = Release|x64 + {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.Build.0 = Release|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.ActiveCfg = Debug|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.Build.0 = Debug|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.ActiveCfg = Debug|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.Build.0 = Debug|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.ActiveCfg = Release|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/visual/2013/zstd/zstd.vcxproj.filters b/visual/2013/zstd/zstd.vcxproj.filters index 45f486fd1..8891f9ccf 100755 --- a/visual/2013/zstd/zstd.vcxproj.filters +++ b/visual/2013/zstd/zstd.vcxproj.filters @@ -1,161 +1,161 @@ - - - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - + + + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + \ No newline at end of file diff --git a/visual/2013/zstdlib/zstdlib.vcxproj.filters b/visual/2013/zstdlib/zstdlib.vcxproj.filters index fa5bb166a..7aeda6e43 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj.filters +++ b/visual/2013/zstdlib/zstdlib.vcxproj.filters @@ -1,104 +1,104 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - - - Resource Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Resource Files + + \ No newline at end of file From 83c76b45945fe8276cd89c9ecd8bce4bcec33ade Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 28 Apr 2016 13:16:01 +0200 Subject: [PATCH 22/26] bench.c: time functions moved to common/util.h --- lib/common/util.h | 54 ++++++++++++++++++++++++++ programs/bench.c | 97 +++++++++++------------------------------------ programs/fileio.c | 2 +- 3 files changed, 78 insertions(+), 75 deletions(-) diff --git a/lib/common/util.h b/lib/common/util.h index 1d2840dc0..b85c4f9e4 100644 --- a/lib/common/util.h +++ b/lib/common/util.h @@ -42,6 +42,7 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ +#include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ /*-**************************************** @@ -57,10 +58,63 @@ extern "C" { # define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ #endif +/* Sleep functions: posix - windows - others */ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) +# include +# include /* setpriority */ +# define UTIL_sleep(s) sleep(s) +# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } +# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +#elif defined(_WIN32) +# include +# define UTIL_sleep(s) Sleep(1000*s) +# define UTIL_sleepMilli(milli) Sleep(milli) +# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) +#else +# define UTIL_sleep(s) /* disabled */ +# define UTIL_sleepMilli(milli) /* disabled */ +# define SET_HIGH_PRIORITY /* disabled */ +#endif + +#if !defined(_WIN32) + typedef clock_t UTIL_time_t; +# define UTIL_initTimer(ticksPerSecond) ticksPerSecond=0 +# define UTIL_getTime(x) x = clock() +# define UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) +# define UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) +#else + typedef LARGE_INTEGER UTIL_time_t; +# define UTIL_initTimer(x) if (!QueryPerformanceFrequency(&x)) { fprintf(stderr, "ERROR: QueryPerformance not present\n"); } +# define UTIL_getTime(x) QueryPerformanceCounter(&x) +# define UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) +# define UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) +#endif + + /*-**************************************** * Utility functions ******************************************/ +/* returns time span in microseconds */ +UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) +{ + UTIL_time_t clockEnd; + + (void)ticksPerSecond; + UTIL_getTime(clockEnd); + return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); +} + + +UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) +{ + UTIL_time_t clockStart, clockEnd; + UTIL_getTime(clockStart); + do { UTIL_getTime(clockEnd); } + while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); +} + + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; diff --git a/programs/bench.c b/programs/bench.c index 77d7c533a..a7a4c579c 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -50,39 +50,6 @@ #include /* fprintf, fopen, ftello64 */ #include /* stat64 */ #include /* stat64 */ -#include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ - -/* sleep : posix - windows - others */ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) -# include -# include /* setpriority */ -# define BMK_sleep(s) sleep(s) -# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000ULL; nanosleep(&t, NULL); } -# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) -#elif defined(_WIN32) -# include -# define BMK_sleep(s) Sleep(1000*s) -# define mili_sleep(mili) Sleep(mili) -# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) -#else -# define BMK_sleep(s) /* disabled */ -# define mili_sleep(mili) /* disabled */ -# define SET_HIGH_PRIORITY /* disabled */ -#endif - -#if !defined(_WIN32) - typedef clock_t BMK_time_t; -# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0 -# define BMK_getTime(x) x = clock() -# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) -# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) -#else - typedef LARGE_INTEGER BMK_time_t; -# define BMK_initTimer(x) if (!QueryPerformanceFrequency(&x)) { fprintf(stderr, "ERROR: QueryPerformance not present\n"); } -# define BMK_getTime(x) QueryPerformanceCounter(&x) -# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) -# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) -#endif #include "mem.h" #include "zstd_static.h" @@ -110,10 +77,10 @@ # define ZSTD_VERSION "" #endif -#define NBLOOPS 3 -#define TIMELOOP_S 1 -#define ACTIVEPERIOD_S 70 -#define COOLPERIOD_S 10 +#define NBLOOPS 3 +#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ +#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define COOLPERIOD_SEC 10 #define KB *(1 <<10) #define MB *(1 <<20) @@ -173,20 +140,6 @@ void BMK_SetBlockSize(size_t blockSize) } -/* ******************************************************** -* Private functions -**********************************************************/ -/* returns time span in microseconds */ -static U64 BMK_clockSpan( BMK_time_t clockStart, BMK_time_t ticksPerSecond ) -{ - BMK_time_t clockEnd; - - (void)ticksPerSecond; - BMK_getTime(clockEnd); - return BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); -} - - /* ******************************************************** * Bench functions **********************************************************/ @@ -229,7 +182,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_DCtx* refDCtx = ZSTD_createDCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); U32 nbBlocks; - BMK_time_t ticksPerSecond; + UTIL_time_t ticksPerSecond; /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx) @@ -237,7 +190,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ - BMK_initTimer(ticksPerSecond); + UTIL_initTimer(ticksPerSecond); /* Init blockTable data */ { const char* srcPtr = (const char*)srcBuffer; @@ -268,33 +221,31 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); U64 crcCheck = 0; - BMK_time_t coolTime; + UTIL_time_t coolTime; U32 testNb; size_t cSize = 0; double ratio = 0.; - BMK_getTime(coolTime); + UTIL_getTime(coolTime); DISPLAYLEVEL(2, "\r%79s\r", ""); for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) { - BMK_time_t clockStart, clockEnd; - U64 clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 1; + UTIL_time_t clockStart; + U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1; /* overheat protection */ - if (BMK_clockSpan(coolTime, ticksPerSecond) > ACTIVEPERIOD_S*1000000ULL) { + if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) { DISPLAY("\rcooling down ... \r"); - BMK_sleep(COOLPERIOD_S); - BMK_getTime(coolTime); + UTIL_sleep(COOLPERIOD_SEC); + UTIL_getTime(coolTime); } /* Compression */ DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ - mili_sleep(1); /* give processor time to other processes */ - BMK_getTime(clockStart); - do { BMK_getTime(clockEnd); } - while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); - BMK_getTime(clockStart); + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(ticksPerSecond); + UTIL_getTime(clockStart); { U32 nbLoops = 0; do { @@ -314,8 +265,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].cSize = rSize; } nbLoops++; - } while (BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop); - { U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond); + } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop); + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond); if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops; } } @@ -331,11 +282,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Decompression */ memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ - mili_sleep(1); /* give processor time to other processes */ - BMK_getTime(clockStart); - do { BMK_getTime(clockEnd); } - while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); - BMK_getTime(clockStart); + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(ticksPerSecond); + UTIL_getTime(clockStart); { U32 nbLoops = 0; do { @@ -354,8 +303,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].resSize = regenSize; } nbLoops++; - } while (BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop); - { U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond); + } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop); + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond); if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops; } } diff --git a/programs/fileio.c b/programs/fileio.c index bb79dbb8e..b69a0cb77 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -63,10 +63,10 @@ #include /* stat64 */ #include /* stat64 */ #include "mem.h" +#include "util.h" /* UTIL_GetFileSize */ #include "fileio.h" #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zbuff_static.h" -#include "util.h" /* UTIL_GetFileSize */ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_legacy.h" /* ZSTD_isLegacy */ From d5ff2c3d9ae3c2d3d70cb29e3c13a53514b2ea6e Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 28 Apr 2016 14:40:45 +0200 Subject: [PATCH 23/26] ordering of #include --- lib/common/util.h | 15 +++++++++++---- lib/dictBuilder/zdict.c | 2 -- programs/bench.c | 5 +---- programs/dibio.c | 5 +---- programs/fileio.c | 5 ++--- programs/fullbench.c | 4 +--- programs/paramgrill.c | 6 ++---- 7 files changed, 18 insertions(+), 24 deletions(-) diff --git a/lib/common/util.h b/lib/common/util.h index b85c4f9e4..3eef4b1d0 100644 --- a/lib/common/util.h +++ b/lib/common/util.h @@ -42,7 +42,11 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ -#include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ +#define _POSIX_C_SOURCE 199309L /* before - needed for nanosleep() */ +#include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ +#include /* stat */ +#include /* stat */ +#include "mem.h" /* U32, U64 */ /*-**************************************** @@ -99,8 +103,8 @@ extern "C" { UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) { UTIL_time_t clockEnd; - (void)ticksPerSecond; + UTIL_getTime(clockEnd); return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); } @@ -109,9 +113,12 @@ UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPe UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) { UTIL_time_t clockStart, clockEnd; + (void)ticksPerSecond; + UTIL_getTime(clockStart); - do { UTIL_getTime(clockEnd); } - while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); + do { + UTIL_getTime(clockEnd); + } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); } diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 74d5f36a9..95d291f40 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -54,8 +54,6 @@ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ #include /* clock */ #include "mem.h" /* read */ diff --git a/programs/bench.c b/programs/bench.c index a7a4c579c..831cc27ef 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -44,18 +44,15 @@ /* ************************************* * Includes ***************************************/ -#define _POSIX_C_SOURCE 199309L /* before - needed for nanosleep() */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ +#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" -#include "util.h" /* UTIL_GetFileSize */ /* ************************************* diff --git a/programs/dibio.c b/programs/dibio.c index 0e8324103..c06d186be 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -46,14 +46,11 @@ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ -#include /* clock */ +#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" /* read */ #include "error_private.h" #include "dibio.h" -#include "util.h" /* UTIL_GetFileSize */ /*-************************************* diff --git a/programs/fileio.c b/programs/fileio.c index b69a0cb77..964fc31ff 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -60,10 +60,9 @@ #include /* strcmp, strlen */ #include /* clock */ #include /* errno */ -#include /* stat64 */ -#include /* stat64 */ -#include "mem.h" + #include "util.h" /* UTIL_GetFileSize */ +#include "mem.h" #include "fileio.h" #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zbuff_static.h" diff --git a/programs/fullbench.c b/programs/fullbench.c index 08ff8973f..3188c0f24 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -43,17 +43,15 @@ **************************************/ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ #include /* strcmp */ #include /* clock_t, clock, CLOCKS_PER_SEC */ +#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "fse_static.h" #include "zbuff.h" #include "datagen.h" -#include "util.h" /* UTIL_GetFileSize */ /*_************************************ diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 21f695240..a904726a1 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -52,8 +52,6 @@ **************************************/ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ #include /* strcmp */ #include /* log */ @@ -64,17 +62,17 @@ # include /* gettimeofday */ #endif +#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "datagen.h" #include "xxhash.h" -#include "util.h" /* UTIL_GetFileSize */ /*-************************************ * Constants **************************************/ -#define PROGRAM_DESCRIPTION "ZSTD_HC parameters tester" +#define PROGRAM_DESCRIPTION "ZSTD parameters tester" #ifndef ZSTD_VERSION # define ZSTD_VERSION "" #endif From 55d047aa92fe3498ccea6bfda007a997e403fccb Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 28 Apr 2016 16:50:13 +0200 Subject: [PATCH 24/26] getTotalFileSize moved to common/util.h --- lib/common/util.h | 10 ++++++++++ programs/bench.c | 12 ++---------- programs/dibio.c | 12 +----------- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/lib/common/util.h b/lib/common/util.h index 3eef4b1d0..1de1efd86 100644 --- a/lib/common/util.h +++ b/lib/common/util.h @@ -138,6 +138,16 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) } +UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) +{ + U64 total = 0; + unsigned n; + for (n=0; n /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ -#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "datagen.h" /* RDG_genBuffer */ @@ -421,14 +421,6 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, } } -static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) -{ - U64 total = 0; - unsigned n; - for (n=0; n Date: Fri, 29 Apr 2016 15:18:50 +0200 Subject: [PATCH 25/26] util.h must be the first include to #define _POSIX_C_SOURCE --- programs/bench.c | 1 + programs/dibio.c | 2 +- programs/fileio.c | 2 +- programs/fullbench.c | 2 +- programs/paramgrill.c | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index dd9f289b6..85c28e8c2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -436,6 +436,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); if (UTIL_isDirectory(fileNamesTable[n])) { DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]); + fileSizes[n] = 0; continue; } f = fopen(fileNamesTable[n], "rb"); diff --git a/programs/dibio.c b/programs/dibio.c index ccdf5057e..fe947bd8a 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -43,11 +43,11 @@ /*-************************************* * Includes ***************************************/ +#include "util.h" /* UTIL_GetFileSize */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ -#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" /* read */ #include "error_private.h" #include "dibio.h" diff --git a/programs/fileio.c b/programs/fileio.c index 964fc31ff..5a1b37d32 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -61,7 +61,7 @@ #include /* clock */ #include /* errno */ -#include "util.h" /* UTIL_GetFileSize */ +#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "fileio.h" #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ diff --git a/programs/fullbench.c b/programs/fullbench.c index 3188c0f24..f77276545 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -46,7 +46,7 @@ #include /* strcmp */ #include /* clock_t, clock, CLOCKS_PER_SEC */ -#include "util.h" /* UTIL_GetFileSize */ +#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "fse_static.h" diff --git a/programs/paramgrill.c b/programs/paramgrill.c index a904726a1..f36a6916f 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -50,6 +50,7 @@ /*-************************************ * Dependencies **************************************/ +#include "util.h" /* UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ @@ -62,7 +63,6 @@ # include /* gettimeofday */ #endif -#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "datagen.h" From bab43179612fda8233cd586c62145a6ef8b20d54 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 29 Apr 2016 15:19:40 +0200 Subject: [PATCH 26/26] util.h must the the first include to #define _POSIX_C_SOURCE --- programs/bench.c | 1 + programs/dibio.c | 2 +- programs/fileio.c | 2 +- programs/fullbench.c | 2 +- programs/paramgrill.c | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index dd9f289b6..85c28e8c2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -436,6 +436,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); if (UTIL_isDirectory(fileNamesTable[n])) { DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]); + fileSizes[n] = 0; continue; } f = fopen(fileNamesTable[n], "rb"); diff --git a/programs/dibio.c b/programs/dibio.c index ccdf5057e..fe947bd8a 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -43,11 +43,11 @@ /*-************************************* * Includes ***************************************/ +#include "util.h" /* UTIL_GetFileSize */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ -#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" /* read */ #include "error_private.h" #include "dibio.h" diff --git a/programs/fileio.c b/programs/fileio.c index 964fc31ff..3c06d20d3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -55,13 +55,13 @@ /*-************************************* * Includes ***************************************/ +#include "util.h" /* UTIL_GetFileSize */ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ #include /* clock */ #include /* errno */ -#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "fileio.h" #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ diff --git a/programs/fullbench.c b/programs/fullbench.c index 3188c0f24..3cc9dac77 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -41,12 +41,12 @@ /*_************************************ * Includes **************************************/ +#include "util.h" /* UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ #include /* clock_t, clock, CLOCKS_PER_SEC */ -#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "fse_static.h" diff --git a/programs/paramgrill.c b/programs/paramgrill.c index a904726a1..f36a6916f 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -50,6 +50,7 @@ /*-************************************ * Dependencies **************************************/ +#include "util.h" /* UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ @@ -62,7 +63,6 @@ # include /* gettimeofday */ #endif -#include "util.h" /* UTIL_GetFileSize */ #include "mem.h" #include "zstd_static.h" #include "datagen.h"