From 9d8297bf04715c5b6884dc533b55ed5e3525d627 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Fri, 18 Dec 2015 09:54:17 +0100 Subject: [PATCH 01/27] Update version to 0.4.5 in zstdlib resources --- lib/zstd.h | 2 +- visual/2013/zstdlib/zstdlib.rc | Bin 4610 -> 4610 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd.h b/lib/zstd.h index b0c841f3a..d6eb0b517 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -62,7 +62,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */ -#define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber (void); diff --git a/visual/2013/zstdlib/zstdlib.rc b/visual/2013/zstdlib/zstdlib.rc index d5b107e331744d414f2945514e3afa761478b127..05e5bf8d2c09fe5ed7a07aa3b00bcf8c17249408 100644 GIT binary patch delta 53 zcmZotX;Rrx!NF)cxst<>(RA}_4ryj)QwF`sfn1W4A8>O_p1{So`3LtKMws9xJ|-3b Dohc5s delta 53 zcmZotX;Rrx!NF)exst<>(RlM}4ryj)V+Os+fn1W4A8>O_p1{So`3LtKMws9xJ|-3b DoGT8l From 8972f13b4a05309c1c9c3929190d565f969a6fb7 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Fri, 18 Dec 2015 11:20:37 +0100 Subject: [PATCH 02/27] Added missing ZSTDLIB_API for ZSTD_decompress_usingDict and ZSTD_decompressDCtx --- lib/zstd_static.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index f78d464c1..542b03094 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -115,15 +115,15 @@ ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /** ZSTD_decompressDCtx * Same as ZSTD_decompress, with pre-allocated DCtx structure */ -size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); /** ZSTD_decompress_usingDict * Same as ZSTD_decompressDCtx, using a Dictionary content as prefix * Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */ -size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, - void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + const void* dict, size_t dictSize); /* ************************************** From ed73ae8c673bc016c55af35c28e08126dd4492fa Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 19 Dec 2015 19:51:42 +0100 Subject: [PATCH 03/27] fix #101 --- lib/zstd_static.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index f78d464c1..725800ff2 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -236,14 +236,14 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 18, 13, 14, 1, 7, ZSTD_fast }, /* level 0 - never used */ { 0, 18, 14, 15, 1, 6, ZSTD_fast }, /* level 1 */ { 0, 18, 14, 15, 1, 5, ZSTD_fast }, /* level 2 */ - { 0, 18, 12, 15, 3, 7, ZSTD_greedy }, /* level 3 */ - { 0, 18, 13, 15, 4, 7, ZSTD_greedy }, /* level 4 */ - { 0, 18, 14, 15, 5, 7, ZSTD_greedy }, /* level 5 */ - { 0, 18, 13, 15, 4, 7, ZSTD_lazy }, /* level 6 */ - { 0, 18, 14, 16, 5, 7, ZSTD_lazy }, /* level 7 */ - { 0, 18, 15, 16, 6, 7, ZSTD_lazy }, /* level 8 */ - { 0, 18, 15, 15, 7, 7, ZSTD_lazy }, /* level 9 */ - { 0, 18, 16, 16, 7, 7, ZSTD_lazy }, /* level 10 */ + { 0, 18, 12, 15, 3, 4, ZSTD_greedy }, /* level 3 */ + { 0, 18, 13, 15, 4, 4, ZSTD_greedy }, /* level 4 */ + { 0, 18, 14, 15, 5, 4, ZSTD_greedy }, /* level 5 */ + { 0, 18, 13, 15, 4, 4, ZSTD_lazy }, /* level 6 */ + { 0, 18, 14, 16, 5, 4, ZSTD_lazy }, /* level 7 */ + { 0, 18, 15, 16, 6, 4, ZSTD_lazy }, /* level 8 */ + { 0, 18, 15, 15, 7, 4, ZSTD_lazy }, /* level 9 */ + { 0, 18, 16, 16, 7, 4, ZSTD_lazy }, /* level 10 */ { 0, 18, 16, 16, 8, 4, ZSTD_lazy }, /* level 11 */ { 0, 18, 17, 16, 8, 4, ZSTD_lazy }, /* level 12 */ { 0, 18, 17, 16, 9, 4, ZSTD_lazy }, /* level 13 */ From c1450f721d4524568dd39cab6f67620fbf724749 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 29 Dec 2015 11:57:15 +0100 Subject: [PATCH 04/27] fix confirmation prompt --- programs/fileio.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 3867301a9..0d49af2b9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -225,7 +225,6 @@ static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr, if (*fileOutPtr != 0) { /* prompt for overwrite authorization */ - int ch = 'N'; fclose(*fileOutPtr); DISPLAY("Warning : %s already exists \n", dstFileName); if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) @@ -235,11 +234,14 @@ static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr, return 1; } DISPLAY("Overwrite ? (y/N) : "); - while((ch = getchar()) != '\n' && ch != EOF); /* flush integrated */ - if ((ch!='Y') && (ch!='y')) { - DISPLAY("No. Operation aborted : %s already exists \n", dstFileName); - return 1; + int ch = getchar(); + if ((ch!='Y') && (ch!='y')) + { + DISPLAY("No. Operation aborted : %s already exists \n", dstFileName); + return 1; + } + while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ } } } From d94efbf95cdda03efbdafd0ef900bfcfe7ecc0f6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 29 Dec 2015 14:29:08 +0100 Subject: [PATCH 05/27] fix fast mode on Windows --- lib/zstd_compress.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c6d81ebec..30e4f06f5 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -761,7 +761,8 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const BYTE* const lowest = base + zc->dictLimit; + const U32 lowIndex = zc->dictLimit; + const BYTE* const lowest = base + lowIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; @@ -772,9 +773,9 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, ZSTD_resetSeqStore(seqStorePtr); if (ip < lowest+4) { - hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = zc->dictLimit+1; - hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = zc->dictLimit+2; - hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = zc->dictLimit+3; + hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = lowIndex+1; + hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = lowIndex+2; + hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = lowIndex+3; ip = lowest+4; } @@ -784,7 +785,8 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, size_t mlCode; size_t offset; const size_t h = ZSTD_hashPtr(ip, hBits, mls); - const BYTE* match = base + hashTable[h]; + const U32 matchIndex = hashTable[h]; + const BYTE* match = base + matchIndex; hashTable[h] = (U32)(ip-base); if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) /* note : by construction, offset_1 <= (ip-base) */ @@ -795,7 +797,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, } else { - if ( (match <= lowest) || + if ( (matchIndex <= lowIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { ip += ((ip-anchor) >> g_searchStrength) + 1; From ee3f4515c1f9ece7d4f56139dc5d2485645beaab Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 29 Dec 2015 22:26:09 +0100 Subject: [PATCH 06/27] faster hc mode with repetitive data (experimental) --- lib/zstd_compress.c | 62 ++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 30e4f06f5..41932da16 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1013,7 +1013,7 @@ size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, /* ************************************* * Binary Tree search ***************************************/ -/** ZSTD_insertBt1 : add one or multiple positions to tree +/*! ZSTD_insertBt1 : add one or multiple positions to tree * @ip : assumed <= iend-8 * @return : nb of positions added */ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares) @@ -1034,14 +1034,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co U32* largerPtr = bt + 2*(current&btMask) + 1; U32 dummy32; /* to be nullified at the end */ const U32 windowLow = zc->lowLimit; - - if ( (current-matchIndex == 1) /* RLE */ - && (matchIndex > windowLow) - && (MEM_read64(match) == MEM_read64(ip)) ) - { - size_t rleLength = ZSTD_count(ip+8, match+8, iend) + 8; - return (U32)(rleLength - mls); - } + U32 matchEndIdx = current; hashTable[h] = current; /* Update Hash Table */ @@ -1054,8 +1047,11 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co if (match[matchLength] == ip[matchLength]) matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + matchLength; + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but required to not corrupt the tree */ if (match[matchLength] < ip[matchLength]) { @@ -1078,7 +1074,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co } *smallerPtr = *largerPtr = 0; - return 1; + return (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - current - zc->params.searchLength : 1; } @@ -1104,6 +1100,7 @@ size_t ZSTD_insertBtAndFindBestMatch ( U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; size_t bestLength = 0; + U32 matchEndIdx = current; U32 dummy32; /* to be nullified at the end */ hashTable[h] = current; /* Update Hash Table */ @@ -1119,6 +1116,8 @@ size_t ZSTD_insertBtAndFindBestMatch ( if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + matchLength; if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) bestLength = matchLength, *offsetPtr = current - matchIndex; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ @@ -1147,12 +1146,12 @@ size_t ZSTD_insertBtAndFindBestMatch ( *smallerPtr = *largerPtr = 0; - zc->nextToUpdate = current+1; /* current has been inserted */ + zc->nextToUpdate = (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - zc->params.searchLength : current+1; return bestLength; } -static const BYTE* ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) { const BYTE* const base = zc->base; const U32 target = (U32)(ip - base); @@ -1160,9 +1159,6 @@ static const BYTE* ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BY for( ; idx < target ; ) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares); - - zc->nextToUpdate = idx; - return base + idx; } @@ -1174,9 +1170,8 @@ size_t ZSTD_BtFindBestMatch ( size_t* offsetPtr, const U32 maxNbAttempts, const U32 mls) { - const BYTE* nextToUpdate = ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - if (nextToUpdate > ip) /* RLE data */ - { *offsetPtr = 1; return ZSTD_count(ip, ip-1, iLimit); } + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls); } @@ -1222,13 +1217,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 U32* largerPtr = bt + 2*(current&btMask) + 1; U32 dummy32; /* to be nullified at the end */ const U32 windowLow = zc->lowLimit; - - if ( (current-matchIndex == 1) /* RLE */ - && (MEM_read64(match) == MEM_read64(ip)) ) - { - size_t rleLength = ZSTD_count(ip+8, match+8, iend) + 8; - return (U32)(rleLength - mls); - } + U32 matchEndIdx = current; hashTable[h] = current; /* Update Hash Table */ @@ -1251,6 +1240,9 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ } + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + matchLength; + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ @@ -1275,11 +1267,11 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 } *smallerPtr = *largerPtr = 0; - return 1; + return (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - current - zc->params.searchLength : 1; } -static const BYTE* ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) { const BYTE* const base = zc->base; const U32 target = (U32)(ip - base); @@ -1287,9 +1279,6 @@ static const BYTE* ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, for( ; idx < target ; ) idx += ZSTD_insertBt1_extDict(zc, base+idx, mls, iend, nbCompares); - - zc->nextToUpdate = idx; - return base + idx; } @@ -1319,6 +1308,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; size_t bestLength = 0; + U32 matchEndIdx = current; U32 dummy32; /* to be nullified at the end */ hashTable[h] = current; /* Update Hash Table */ @@ -1345,6 +1335,8 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + matchLength; if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) bestLength = matchLength, *offsetPtr = current - matchIndex; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ @@ -1373,7 +1365,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( *smallerPtr = *largerPtr = 0; - zc->nextToUpdate = current+1; /* current has been inserted */ + zc->nextToUpdate = (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - zc->params.searchLength : current+1; return bestLength; } @@ -1385,9 +1377,8 @@ size_t ZSTD_BtFindBestMatch_extDict ( size_t* offsetPtr, const U32 maxNbAttempts, const U32 mls) { - const BYTE* nextToUpdate = ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - if (nextToUpdate > ip) /* RLE data */ - { *offsetPtr = 1; return ZSTD_count(ip, ip-1, iLimit); } + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); return ZSTD_insertBtAndFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls); } @@ -2067,6 +2058,7 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src case ZSTD_btlazy2: ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength); + zc->nextToUpdate = iend - zc->base; break; default: From 48da164f1428ea4448ca2478b6ecd6fb1efd3cb2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 29 Dec 2015 23:40:02 +0100 Subject: [PATCH 07/27] fixed clang warnings --- lib/zstd_compress.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 41932da16..e88568d92 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1048,7 +1048,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + matchLength; + matchEndIdx = matchIndex + (U32)matchLength; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ break; /* drop , to guarantee consistency ; miss a bit of compression, but required to not corrupt the tree */ @@ -1117,7 +1117,7 @@ size_t ZSTD_insertBtAndFindBestMatch ( if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + matchLength; + matchEndIdx = matchIndex + (U32)matchLength; if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) bestLength = matchLength, *offsetPtr = current - matchIndex; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ @@ -1241,7 +1241,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 } if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + matchLength; + matchEndIdx = matchIndex + (U32)matchLength; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ @@ -1336,7 +1336,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + matchLength; + matchEndIdx = matchIndex + (U32)matchLength; if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) bestLength = matchLength, *offsetPtr = current - matchIndex; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ @@ -2058,7 +2058,7 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src case ZSTD_btlazy2: ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength); - zc->nextToUpdate = iend - zc->base; + zc->nextToUpdate = (U32)(iend - zc->base); break; default: From 155fbbfe2588b44d405fddabb28d10fbbac416c8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 30 Dec 2015 19:31:46 +0100 Subject: [PATCH 08/27] visual dll version automatically follows zstd.h --- visual/2013/zstdlib/zstdlib.rc | Bin 4610 -> 5204 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/visual/2013/zstdlib/zstdlib.rc b/visual/2013/zstdlib/zstdlib.rc index 05e5bf8d2c09fe5ed7a07aa3b00bcf8c17249408..93e221d9f59c8e72720b0094614e27b8925e42ba 100644 GIT binary patch delta 611 zcmZotxuP*)!Q^j@BJ5QR#SA43DI2G@vrk+iP#?t*%n-uh!Vu5k!{Eu_#1PLA#^A~j z#1PEj$>7i6$DqJ~P=P8F&*018$l%4`&k)3*gIAR&gC8zcGEi&c8G;yG8GIOA85|jc z8C)537z`K`Ce9aCjbaF5aAt^S2xJIl@Mj2NaAnW{d!UFxlR<$&8N!}8U(_F_(3K&A zA%MY=!4K>y7>!*!Ob-^>iStDXhD|&;?1I2y1aT!{%Mqc8J-js;cp13BIBc>kN3c8r r^^ Date: Wed, 30 Dec 2015 19:52:08 +0100 Subject: [PATCH 09/27] Makefile version automatically follows zstd.h --- Makefile | 2 -- NEWS | 4 ++++ lib/Makefile | 23 ++++++++++++++--------- programs/Makefile | 11 ++++++++++- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index a741034ad..9caa907cb 100644 --- a/Makefile +++ b/Makefile @@ -31,8 +31,6 @@ # - Public forum : https://groups.google.com/forum/#!forum/lz4c # ################################################################ -# Version number -export VERSION := 0.4.5 PRGDIR = programs ZSTDDIR = lib diff --git a/NEWS b/NEWS index ee9a4585c..d60b363a4 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +v0.4.6 +fix : fast compression mode on Windows +Improved : high compression mode on repetitive data + v0.4.5 new : -m/--multiple : compress/decompress multiple files diff --git a/lib/Makefile b/lib/Makefile index c621f63c8..181d4a735 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -32,11 +32,15 @@ # ################################################################ # Version numbers -LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` -LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` -LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` -LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) -VERSION?= $(LIBVER) +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) +VERSION:= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local @@ -98,12 +102,13 @@ clean: #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) -libzstd.pc: libzstd.pc.in Makefile +libzstd.pc: +libzstd.pc: libzstd.pc.in @echo creating pkgconfig @sed -e 's|@PREFIX@|$(PREFIX)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ - -e 's|@VERSION@|$(VERSION)|' \ + -e 's|@LIBDIR@|$(LIBDIR)|' \ + -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@VERSION@|$(VERSION)|' \ $< >$@ install: libzstd libzstd.pc diff --git a/programs/Makefile b/programs/Makefile index 57fa87c11..b9d51adc8 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,16 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -VERSION?= 0.4.5 +# Version numbers +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) +VERSION:= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local From 722504c2a07a72d2b3ce6784fff46bc795bd97a9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 31 Dec 2015 18:32:15 +0100 Subject: [PATCH 10/27] update version to 0.4.6 --- Makefile | 2 ++ lib/Makefile | 2 +- lib/zstd.h | 2 +- programs/Makefile | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9caa907cb..62ca6683d 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,8 @@ # - Public forum : https://groups.google.com/forum/#!forum/lz4c # ################################################################ +# force a version number : uncomment below export (otherwise, default to the one declared into zstd.h) +#export VERSION := 0.4.6 PRGDIR = programs ZSTDDIR = lib diff --git a/lib/Makefile b/lib/Makefile index 181d4a735..a4a156d85 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -40,7 +40,7 @@ LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) LIBVER := $(shell echo $(LIBVER_SCRIPT)) -VERSION:= $(LIBVER) +VERSION?= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local diff --git a/lib/zstd.h b/lib/zstd.h index d6eb0b517..bba096144 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -62,7 +62,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */ -#define ZSTD_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber (void); diff --git a/programs/Makefile b/programs/Makefile index b9d51adc8..f2230d973 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -39,7 +39,7 @@ LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) LIBVER := $(shell echo $(LIBVER_SCRIPT)) -VERSION:= $(LIBVER) +VERSION?= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local From 72e84cfd0f11396bd434dbe36e686be52e698b4a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 31 Dec 2015 19:08:44 +0100 Subject: [PATCH 11/27] minor simplification --- lib/zstd_compress.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index e88568d92..2efb9289b 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1034,7 +1034,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co U32* largerPtr = bt + 2*(current&btMask) + 1; U32 dummy32; /* to be nullified at the end */ const U32 windowLow = zc->lowLimit; - U32 matchEndIdx = current; + U32 matchEndIdx = current+8; hashTable[h] = current; /* Update Hash Table */ @@ -1074,7 +1074,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co } *smallerPtr = *largerPtr = 0; - return (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - current - zc->params.searchLength : 1; + return (matchEndIdx > current + 8) ? matchEndIdx - current - 8 : 1; } @@ -1100,7 +1100,7 @@ size_t ZSTD_insertBtAndFindBestMatch ( U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; size_t bestLength = 0; - U32 matchEndIdx = current; + U32 matchEndIdx = current+8; U32 dummy32; /* to be nullified at the end */ hashTable[h] = current; /* Update Hash Table */ @@ -1146,7 +1146,7 @@ size_t ZSTD_insertBtAndFindBestMatch ( *smallerPtr = *largerPtr = 0; - zc->nextToUpdate = (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - zc->params.searchLength : current+1; + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; return bestLength; } @@ -1217,7 +1217,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 U32* largerPtr = bt + 2*(current&btMask) + 1; U32 dummy32; /* to be nullified at the end */ const U32 windowLow = zc->lowLimit; - U32 matchEndIdx = current; + U32 matchEndIdx = current+8; hashTable[h] = current; /* Update Hash Table */ @@ -1267,7 +1267,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 } *smallerPtr = *largerPtr = 0; - return (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - current - zc->params.searchLength : 1; + return (matchEndIdx > current + 8) ? matchEndIdx - current - 8 : 1; } @@ -1308,7 +1308,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; size_t bestLength = 0; - U32 matchEndIdx = current; + U32 matchEndIdx = current+8; U32 dummy32; /* to be nullified at the end */ hashTable[h] = current; /* Update Hash Table */ @@ -1365,7 +1365,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( *smallerPtr = *largerPtr = 0; - zc->nextToUpdate = (matchEndIdx > current + zc->params.searchLength) ? matchEndIdx - zc->params.searchLength : current+1; + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; return bestLength; } From 1358f91ec4a5a202158c161488d19e0f7c23925d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Jan 2016 07:29:39 +0100 Subject: [PATCH 12/27] minor refactoring --- lib/zstd_compress.c | 112 ++++++++++---------------------------------- 1 file changed, 24 insertions(+), 88 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 2efb9289b..51157306b 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1013,10 +1013,11 @@ size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, /* ************************************* * Binary Tree search ***************************************/ -/*! ZSTD_insertBt1 : add one or multiple positions to tree +/** ZSTD_insertBt1 : add one or multiple positions to tree * @ip : assumed <= iend-8 * @return : nb of positions added */ -static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares) +static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, + U32 extDict) { U32* const hashTable = zc->hashTable; const U32 hashLog = zc->params.hashLog; @@ -1027,6 +1028,10 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co U32 matchIndex = hashTable[h]; size_t commonLengthSmaller=0, commonLengthLarger=0; const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; const BYTE* match = base + matchIndex; const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; @@ -1043,17 +1048,27 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co U32* nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) - matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) + { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + } + else + { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but required to not corrupt the tree */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ - if (match[matchLength] < ip[matchLength]) + if (match[matchLength] < ip[matchLength]) /* necessarily within correct buffer */ { /* match is smaller than current */ *smallerPtr = matchIndex; /* update smaller idx */ @@ -1158,7 +1173,7 @@ static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* con U32 idx = zc->nextToUpdate; for( ; idx < target ; ) - idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares); + idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0); } @@ -1192,85 +1207,6 @@ FORCE_INLINE size_t ZSTD_BtFindBestMatch_selectMLS ( } -/** ZSTD_insertBt1_extDict : add one or multiple positions to tree -* @ip : assumed <= iend-8 -* @return : nb of positions added */ -static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares) -{ - U32* const hashTable = zc->hashTable; - const U32 hashLog = zc->params.hashLog; - const size_t h = ZSTD_hashPtr(ip, hashLog, mls); - U32* const bt = zc->contentTable; - const U32 btLog = zc->params.contentLog - 1; - const U32 btMask= (1 << btLog) - 1; - U32 matchIndex = hashTable[h]; - size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const base = zc->base; - const BYTE* const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const prefixStart = base + dictLimit; - const BYTE* match = base + matchIndex; - const U32 current = (U32)(ip-base); - const U32 btLow = btMask >= current ? 0 : current - btMask; - U32* smallerPtr = bt + 2*(current&btMask); - U32* largerPtr = bt + 2*(current&btMask) + 1; - U32 dummy32; /* to be nullified at the end */ - const U32 windowLow = zc->lowLimit; - U32 matchEndIdx = current+8; - - hashTable[h] = current; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) - { - U32* nextPtr = bt + 2*(matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - - if (matchIndex+matchLength >= dictLimit) - { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) - matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; - } - else - { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - - if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ - - if (match[matchLength] < ip[matchLength]) /* necessarily within correct buffer */ - { - /* match is smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - } - else - { - /* match is larger than current */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - return (matchEndIdx > current + 8) ? matchEndIdx - current - 8 : 1; -} - - static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) { const BYTE* const base = zc->base; @@ -1278,7 +1214,7 @@ static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const B U32 idx = zc->nextToUpdate; for( ; idx < target ; ) - idx += ZSTD_insertBt1_extDict(zc, base+idx, mls, iend, nbCompares); + idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); } From 2cc12cb3cbe84e73741c95ac6e9c69a816e6533f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Jan 2016 07:47:58 +0100 Subject: [PATCH 13/27] minor refactoring --- lib/zstd_compress.c | 166 +++++++++++++------------------------------- 1 file changed, 47 insertions(+), 119 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 51157306b..0bb04ea02 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1093,79 +1093,6 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co } -FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ -size_t ZSTD_insertBtAndFindBestMatch ( - ZSTD_CCtx* zc, - const BYTE* const ip, const BYTE* const iend, - size_t* offsetPtr, - U32 nbCompares, const U32 mls) -{ - U32* const hashTable = zc->hashTable; - const U32 hashLog = zc->params.hashLog; - const size_t h = ZSTD_hashPtr(ip, hashLog, mls); - U32* const bt = zc->contentTable; - const U32 btLog = zc->params.contentLog - 1; - const U32 btMask= (1 << btLog) - 1; - U32 matchIndex = hashTable[h]; - size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const base = zc->base; - const U32 current = (U32)(ip-base); - const U32 btLow = btMask >= current ? 0 : current - btMask; - const U32 windowLow = zc->lowLimit; - U32* smallerPtr = bt + 2*(current&btMask); - U32* largerPtr = bt + 2*(current&btMask) + 1; - size_t bestLength = 0; - U32 matchEndIdx = current+8; - U32 dummy32; /* to be nullified at the end */ - - hashTable[h] = current; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) - { - U32* nextPtr = bt + 2*(matchIndex & btMask); - const BYTE* match = base + matchIndex; - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - - if (match[matchLength] == ip[matchLength]) - matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; - - if (matchLength > bestLength) - { - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) - bestLength = matchLength, *offsetPtr = current - matchIndex; - if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - - if (match[matchLength] < ip[matchLength]) - { - /* match is smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - } - else - { - /* match is larger than current */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - - zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; - return bestLength; -} - - static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) { const BYTE* const base = zc->base; @@ -1176,54 +1103,13 @@ static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* con idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0); } - -/** Tree updater, providing best match */ FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ -size_t ZSTD_BtFindBestMatch ( - ZSTD_CCtx* zc, - const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 maxNbAttempts, const U32 mls) -{ - if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls); -} - - -FORCE_INLINE size_t ZSTD_BtFindBestMatch_selectMLS ( - ZSTD_CCtx* zc, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 maxNbAttempts, const U32 matchLengthSearch) -{ - switch(matchLengthSearch) - { - default : - case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); - case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); - case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); - } -} - - -static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) -{ - const BYTE* const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; - - for( ; idx < target ; ) - idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); -} - - -FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ -size_t ZSTD_insertBtAndFindBestMatch_extDict ( +size_t ZSTD_insertBtAndFindBestMatch ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, size_t* offsetPtr, - U32 nbCompares, const U32 mls) + U32 nbCompares, const U32 mls, + U32 extDict) { U32* const hashTable = zc->hashTable; const U32 hashLog = zc->params.hashLog; @@ -1255,7 +1141,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match; - if (matchIndex+matchLength >= dictLimit) + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { match = base + matchIndex; if (match[matchLength] == ip[matchLength]) @@ -1305,6 +1191,48 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( return bestLength; } + +/** Tree updater, providing best match */ +FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +size_t ZSTD_BtFindBestMatch ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); +} + + +FORCE_INLINE size_t ZSTD_BtFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + +static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + for( ; idx < target ; ) + idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); +} + + /** Tree updater, providing best match */ FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ size_t ZSTD_BtFindBestMatch_extDict ( @@ -1315,7 +1243,7 @@ size_t ZSTD_BtFindBestMatch_extDict ( { if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); } From 96ffa424ebc4025744aaf95f3430a8142d94ec97 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 2 Jan 2016 01:16:28 +0100 Subject: [PATCH 14/27] minor fast mode improvement --- lib/zstd_compress.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 0bb04ea02..5db0fdf5d 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -787,7 +787,8 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, const size_t h = ZSTD_hashPtr(ip, hBits, mls); const U32 matchIndex = hashTable[h]; const BYTE* match = base + matchIndex; - hashTable[h] = (U32)(ip-base); + const U32 current = (U32)(ip-base); + hashTable[h] = current; /* update hash table */ if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) /* note : by construction, offset_1 <= (ip-base) */ { @@ -818,7 +819,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, if (ip <= ilimit) { /* Fill Table */ - hashTable[ZSTD_hashPtr(ip-(mlCode+MINMATCH)+2, hBits, mls)] = (U32)(ip-(mlCode+MINMATCH)+2-base); /* here because ip-(mlCode+MINMATCH)+2 could be > iend-8 without ip <= ilimit check*/ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 without ip <= ilimit check*/ hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); /* check immediate repcode */ while ( (ip <= ilimit) From 5835e1b5ba5245dacac0e3c4625a5c538db9c151 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 5 Jan 2016 01:44:36 +0100 Subject: [PATCH 15/27] updated xxhash source --- programs/fuzzer.c | 8 +-- programs/xxhash.c | 114 ++++++++++++++++---------------- programs/xxhash.h | 152 ++++++++++++++++++++++++++----------------- programs/zbufftest.c | 8 +-- 4 files changed, 158 insertions(+), 124 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index f3fda456a..025fc5d6a 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -305,7 +305,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize; U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n; - XXH64_state_t crc64; + XXH64_CREATESTATE_STATIC(xxh64); U64 crcOrig, crcDest; int cLevel; BYTE* sampleBuffer; @@ -447,7 +447,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit } /* Streaming compression of scattered segments test */ - XXH64_reset(&crc64, 0); + XXH64_reset(xxh64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; @@ -481,14 +481,14 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; - XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize); + XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); totalTestSize += sampleSize; } errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; - crcOrig = XXH64_digest(&crc64); + crcOrig = XXH64_digest(xxh64); /* streaming decompression test */ errorCode = ZSTD_resetDCtx(dctx); diff --git a/programs/xxhash.c b/programs/xxhash.c index 511d9941a..9389321ce 100644 --- a/programs/xxhash.c +++ b/programs/xxhash.c @@ -1,6 +1,6 @@ /* xxHash - Fast Hash algorithm -Copyright (C) 2012-2015, Yann Collet +Copyright (C) 2012-2016, Yann Collet BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -32,18 +32,18 @@ You can contact the author at : */ -/************************************** +/* ************************************* * Tuning parameters -**************************************/ -/* XXH_FORCE_MEMORY_ACCESS +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. * The below switch allow to select different access method for improved performance. * Method 0 (default) : use `memcpy()`. Safe and portable. * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which generate assembly depending on alignment. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) * See http://stackoverflow.com/a/32095106/646947 for details. * Prefer these methods in priority order (0 > 1 > 2) @@ -57,14 +57,14 @@ You can contact the author at : # endif #endif -/* XXH_ACCEPT_NULL_INPUT_POINTER : +/*!XXH_ACCEPT_NULL_INPUT_POINTER : * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. * By default, this option is disabled. To enable it, uncomment below define : */ /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ -/* XXH_FORCE_NATIVE_FORMAT : +/*!XXH_FORCE_NATIVE_FORMAT : * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. * Results are therefore identical for little-endian and big-endian CPU. * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. @@ -74,9 +74,9 @@ You can contact the author at : */ #define XXH_FORCE_NATIVE_FORMAT 0 -/* XXH_USELESS_ALIGN_BRANCH : +/*!XXH_USELESS_ALIGN_BRANCH : * This is a minor performance trick, only useful with lots of very small keys. - * It means : don't make a test between aligned/unaligned, because performance will be the same. + * It means : don't check for aligned/unaligned input, because performance will be the same. * It saves one initial branch per hash. */ #if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) @@ -106,7 +106,6 @@ You can contact the author at : /************************************** * Includes & Memory related functions ***************************************/ -#include "xxhash.h" /* Modify the local functions below should you wish to use some other memory routines */ /* for malloc(), free() */ #include @@ -116,23 +115,28 @@ static void XXH_free (void* p) { free(p); } #include static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } +#include "xxhash.h" + /************************************** * Basic Types ***************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; +#ifndef MEM_MODULE +# define MEM_MODULE +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +# endif #endif @@ -221,10 +225,10 @@ static U64 XXH_swap64 (U64 x) ***************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example one the compiler command line */ +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ #ifndef XXH_CPU_LITTLE_ENDIAN - static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) + static const int g_one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) #endif @@ -263,7 +267,7 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) /*************************************** * Macros ***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ /*************************************** @@ -281,6 +285,8 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) #define PRIME64_4 9650029242287828579ULL #define PRIME64_5 2870177450012600261ULL +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + /***************************** * Simple Hash Functions @@ -362,7 +368,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH } -unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ @@ -502,7 +508,7 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH } -unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ @@ -530,12 +536,12 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed #endif } -/**************************************************** +/* ************************************************** * Advanced Hash Functions ****************************************************/ /*** Allocation ***/ -typedef struct +struct XXH32_state_s { U64 total_len; U32 seed; @@ -545,9 +551,9 @@ typedef struct U32 v4; U32 mem32[4]; /* defined as U32 for alignment */ U32 memsize; -} XXH_istate32_t; +}; /* typedef'd to XXH32_state_t within xxhash.h */ -typedef struct +struct XXH64_state_s { U64 total_len; U64 seed; @@ -557,26 +563,26 @@ typedef struct U64 v4; U64 mem64[4]; /* defined as U64 for alignment */ U32 memsize; -} XXH_istate64_t; +}; /* typedef'd to XXH64_state_t within xxhash.h */ -XXH32_state_t* XXH32_createState(void) +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */ + XXH_STATIC_ASSERT(sizeof(XXH32_stateBody_t) >= sizeof(XXH32_state_t)); /* A compilation error here means XXH32_state_t is not large enough */ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } -XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } -XXH64_state_t* XXH64_createState(void) +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */ + XXH_STATIC_ASSERT(sizeof(XXH64_stateBody_t) >= sizeof(XXH64_state_t)); /* A compilation error here means XXH64_state_t is not large enough */ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } -XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; @@ -585,9 +591,8 @@ XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) /*** Hash feed ***/ -XXH_errorcode XXH32_reset(XXH32_state_t* state_in, unsigned int seed) +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* state, unsigned int seed) { - XXH_istate32_t* state = (XXH_istate32_t*) state_in; state->seed = seed; state->v1 = seed + PRIME32_1 + PRIME32_2; state->v2 = seed + PRIME32_2; @@ -598,9 +603,8 @@ XXH_errorcode XXH32_reset(XXH32_state_t* state_in, unsigned int seed) return XXH_OK; } -XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* state, unsigned long long seed) { - XXH_istate64_t* state = (XXH_istate64_t*) state_in; state->seed = seed; state->v1 = seed + PRIME64_1 + PRIME64_2; state->v2 = seed + PRIME64_2; @@ -612,9 +616,8 @@ XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) } -FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) { - XXH_istate32_t* state = (XXH_istate32_t *) state_in; const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -701,7 +704,7 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v return XXH_OK; } -XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -713,9 +716,8 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l -FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { - const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; const BYTE * p = (const BYTE*)state->mem32; const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; @@ -755,7 +757,7 @@ FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endiane } -unsigned int XXH32_digest (const XXH32_state_t* state_in) +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -766,9 +768,8 @@ unsigned int XXH32_digest (const XXH32_state_t* state_in) } -FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) { - XXH_istate64_t * state = (XXH_istate64_t *) state_in; const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -855,7 +856,7 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v return XXH_OK; } -XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -867,9 +868,8 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { - const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; const BYTE * p = (const BYTE*)state->mem64; const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; @@ -949,7 +949,7 @@ FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endiane } -unsigned long long XXH64_digest (const XXH64_state_t* state_in) +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; diff --git a/programs/xxhash.h b/programs/xxhash.h index c60aa6157..ea582e482 100644 --- a/programs/xxhash.h +++ b/programs/xxhash.h @@ -1,7 +1,7 @@ /* xxHash - Extremely Fast Hash algorithm Header File - Copyright (C) 2012-2015, Yann Collet. + Copyright (C) 2012-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -71,34 +71,56 @@ extern "C" { #endif -/***************************** +/* **************************** * Definitions -*****************************/ +******************************/ #include /* size_t */ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; -/***************************** -* Namespace Emulation -*****************************/ -/* Motivations : +/* **************************** +* API modifier +******************************/ +/*!XXH_PRIVATE_API +* Transforms all publics symbols within `xxhash.c` into private ones. +* Methodology : +* instead of : #include "xxhash.h" +* do : +* #define XXH_PRIVATE_API +* #include "xxhash.c" // note the .c , instead of .h +* also : don't compile and link xxhash.c separately +*/ +#ifdef XXH_PRIVATE_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else +# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif -If you need to include xxHash into your library, -but wish to avoid xxHash symbols to be present on your library interface -in an effort to avoid potential name collision if another library also includes xxHash, +/*!XXH_NAMESPACE, aka Namespace Emulation : -you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash -with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values). +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with another library which also includes xxHash, -Note that no change is required within the calling program : -it can still call xxHash functions using their regular name. -They will be automatically translated by this header. +you can use XXH_NAMESPACE, to automatically prefix any public symbol from `xxhash.c` +with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). + +Note that no change is required within the calling program as long as it also includes `xxhash.h` : +regular symbol name will be automatically translated by this header. */ #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) @@ -112,63 +134,75 @@ They will be automatically translated by this header. #endif -/***************************** +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 /* for breaking interface changes */ +#define XXH_VERSION_MINOR 5 /* for new (non-breaking) interface capabilities */ +#define XXH_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** * Simple Hash Functions -*****************************/ +******************************/ -unsigned int XXH32 (const void* input, size_t length, unsigned seed); -unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); -/* +/*! XXH32() : Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". The memory between input & input+length must be valid (allocated and read-accessible). "seed" can be used to alter the result predictably. - This function successfully passes all SMHasher tests. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s XXH64() : Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - Faster on 64-bits systems. Slower on 32-bits systems. + "seed" can be used to alter the result predictably. + This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). */ - -/***************************** +/* **************************** * Advanced Hash Functions -*****************************/ -typedef struct { long long ll[ 6]; } XXH32_state_t; -typedef struct { long long ll[11]; } XXH64_state_t; - -/* -These structures allow static allocation of XXH states. -States must then be initialized using XXHnn_reset() before first use. - -If you prefer dynamic allocation, please refer to functions below. -*/ - -XXH32_state_t* XXH32_createState(void); -XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); - -XXH64_state_t* XXH64_createState(void); -XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); - -/* -These functions create and release memory for XXH state. -States must then be initialized using XXHnn_reset() before first use. -*/ +******************************/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete */ -XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); -XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -unsigned int XXH32_digest (const XXH32_state_t* statePtr); +/*!Static allocation + For static linking only, do not use in the context of DLL ! */ +typedef struct { long long ll[ 6]; } XXH32_stateBody_t; +typedef struct { long long ll[11]; } XXH64_stateBody_t; -XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -unsigned long long XXH64_digest (const XXH64_state_t* statePtr); +#define XXH32_CREATESTATE_STATIC(name) XXH32_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH32_state_t* name = (XXH32_state_t*)(name##xxhvoid) /* no final ; */ +#define XXH64_CREATESTATE_STATIC(name) XXH64_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH64_state_t* name = (XXH64_state_t*)(name##xxhvoid) /* no final ; */ -/* -These functions calculate the xxHash of an input provided in multiple smaller packets, -as opposed to an input provided as a single block. + +/*!Dynamic allocation + To be preferred in the context of DLL */ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + + +/* hash streaming */ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* statePtr); + +/*! +These functions calculate the xxHash of an input provided in multiple segments, +as opposed to provided as a single block. XXH state space must first be allocated, using either static or dynamic method provided above. @@ -178,12 +212,12 @@ Then, feed the hash state by calling XXHnn_update() as many times as necessary. Obviously, input must be valid, meaning allocated and read accessible. The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -Finally, you can produce a hash anytime, by using XXHnn_digest(). -This function returns the final nn-bits hash. -You can nonetheless continue feeding the hash state with more input, -and therefore get some new hashes, by calling again XXHnn_digest(). +Finally, a hash value can be produced anytime, by using XXHnn_digest(). +This function returns the nn-bits hash. +It's nonetheless possible to continue inserting input into the hash state +and later on generate some new hashes, by calling again XXHnn_digest(). -When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). +When you are done, don't forget to free XXH state space if it was allocated dynamically. */ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index ab8aa3486..f9677f00a 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -286,7 +286,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; - XXH64_state_t crc64; + XXH64_CREATESTATE_STATIC(xxh64); U64 crcOrig, crcDest; /* init */ @@ -313,7 +313,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit srcBuffer = cNoiseBuffer[buffNb]; /* Multi - segments compression test */ - XXH64_reset(&crc64, 0); + XXH64_reset(xxh64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; @@ -347,7 +347,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); - XXH64_update(&crc64, srcBuffer+sampleStart, readSize); + XXH64_update(xxh64, srcBuffer+sampleStart, readSize); memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize); cSize += genSize; totalTestSize += readSize; @@ -371,7 +371,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (errorCode != 0, "frame epilogue not fully consumed"); cSize += genSize; - crcOrig = XXH64_digest(&crc64); + crcOrig = XXH64_digest(xxh64); /* multi - fragments decompression test */ ZBUFF_decompressInit(zd); From 8ab94b653c8b3296d2b8f6624b5a06d4022b0e26 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jan 2016 01:37:46 +0100 Subject: [PATCH 16/27] updated xxhash --- programs/xxhash.c | 53 +++++++++++++++++++++++++---------------------- programs/xxhash.h | 14 ++++++------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/programs/xxhash.c b/programs/xxhash.c index 9389321ce..d33113fe2 100644 --- a/programs/xxhash.c +++ b/programs/xxhash.c @@ -84,7 +84,7 @@ You can contact the author at : #endif -/************************************** +/* ************************************* * Compiler Specific Options ***************************************/ #ifdef _MSC_VER /* Visual Studio */ @@ -103,7 +103,7 @@ You can contact the author at : #endif -/************************************** +/* ************************************* * Includes & Memory related functions ***************************************/ /* Modify the local functions below should you wish to use some other memory routines */ @@ -118,7 +118,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp #include "xxhash.h" -/************************************** +/* ************************************* * Basic Types ***************************************/ #ifndef MEM_MODULE @@ -178,7 +178,7 @@ static U64 XXH_read64(const void* memPtr) #endif // XXH_FORCE_DIRECT_MEMORY_ACCESS -/****************************************** +/* **************************************** * Compiler-specific Functions and Macros ******************************************/ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) @@ -220,7 +220,7 @@ static U64 XXH_swap64 (U64 x) #endif -/*************************************** +/* ************************************* * Architecture Macros ***************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; @@ -232,7 +232,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; #endif -/***************************** +/* *************************** * Memory reads *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; @@ -264,13 +264,13 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) } -/*************************************** +/* ************************************* * Macros ***************************************/ #define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -/*************************************** +/* ************************************* * Constants ***************************************/ #define PRIME32_1 2654435761U @@ -288,7 +288,7 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } -/***************************** +/* *************************** * Simple Hash Functions *****************************/ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) @@ -591,27 +591,30 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) /*** Hash feed ***/ -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* state, unsigned int seed) +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) { - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.seed = seed; + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + memcpy(statePtr, &state, sizeof(state)); return XXH_OK; } -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* state, unsigned long long seed) + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) { - state->seed = seed; - state->v1 = seed + PRIME64_1 + PRIME64_2; - state->v2 = seed + PRIME64_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME64_1; - state->total_len = 0; - state->memsize = 0; + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.seed = seed; + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + memcpy(statePtr, &state, sizeof(state)); return XXH_OK; } diff --git a/programs/xxhash.h b/programs/xxhash.h index ea582e482..4b1e1dcd6 100644 --- a/programs/xxhash.h +++ b/programs/xxhash.h @@ -137,9 +137,9 @@ regular symbol name will be automatically translated by this header. /* ************************************* * Version ***************************************/ -#define XXH_VERSION_MAJOR 0 /* for breaking interface changes */ -#define XXH_VERSION_MINOR 5 /* for new (non-breaking) interface capabilities */ -#define XXH_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 5 +#define XXH_VERSION_RELEASE 0 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) XXH_PUBLIC_API unsigned XXH_versionNumber (void); @@ -201,15 +201,15 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const v XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* statePtr); /*! -These functions calculate the xxHash of an input provided in multiple segments, +These functions generate the xxHash of an input provided in multiple segments, as opposed to provided as a single block. -XXH state space must first be allocated, using either static or dynamic method provided above. +XXH state must first be allocated, using either static or dynamic method provided above. Start a new hash by initializing state with a seed, using XXHnn_reset(). Then, feed the hash state by calling XXHnn_update() as many times as necessary. -Obviously, input must be valid, meaning allocated and read accessible. +Obviously, input must be valid, hence allocated and read accessible. The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. Finally, a hash value can be produced anytime, by using XXHnn_digest(). @@ -217,7 +217,7 @@ This function returns the nn-bits hash. It's nonetheless possible to continue inserting input into the hash state and later on generate some new hashes, by calling again XXHnn_digest(). -When you are done, don't forget to free XXH state space if it was allocated dynamically. +When done, free XXH state space if it was allocated dynamically. */ From 3b994cb465251b29782b3582c748b3c5f92bf0cd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jan 2016 01:58:37 +0100 Subject: [PATCH 17/27] updated fse --- lib/fse.c | 97 ++++++++++++++++++++++++---------------------- lib/fse.h | 46 +++++++++++----------- lib/fse_static.h | 53 +++++++++++++------------ lib/huff0.c | 81 +++++++++++++++++++------------------- lib/huff0.h | 16 ++++---- lib/huff0_static.h | 51 ++++++++++++++++++++++-- 6 files changed, 198 insertions(+), 146 deletions(-) diff --git a/lib/fse.c b/lib/fse.c index 38e40a576..e74c1e833 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -34,10 +34,10 @@ #ifndef FSE_COMMONDEFS_ONLY -/**************************************************************** +/* ************************************************************** * Tuning parameters ****************************************************************/ -/* MEMORY_USAGE : +/*!MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect @@ -45,26 +45,23 @@ #define FSE_MAX_MEMORY_USAGE 14 #define FSE_DEFAULT_MEMORY_USAGE 13 -/* FSE_MAX_SYMBOL_VALUE : +/*!FSE_MAX_SYMBOL_VALUE : * Maximum symbol value authorized. * Required for proper stack allocation */ #define FSE_MAX_SYMBOL_VALUE 255 -/**************************************************************** +/* ************************************************************** * template functions type & suffix ****************************************************************/ #define FSE_FUNCTION_TYPE BYTE #define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t -/**************************************************************** -* Byte symbol type -****************************************************************/ #endif /* !FSE_COMMONDEFS_ONLY */ - -/**************************************************************** +/* ************************************************************** * Compiler specifics ****************************************************************/ #ifdef _MSC_VER /* Visual Studio */ @@ -82,7 +79,7 @@ #endif -/**************************************************************** +/* ************************************************************** * Includes ****************************************************************/ #include /* malloc, free, qsort */ @@ -92,7 +89,7 @@ #include "fse_static.h" -/**************************************************************** +/* *************************************************************** * Constants *****************************************************************/ #define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) @@ -107,20 +104,20 @@ #endif -/**************************************************************** +/* ************************************************************** * Error Management ****************************************************************/ #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -/**************************************************************** +/* ************************************************************** * Complex types ****************************************************************/ typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; -/**************************************************************** +/* ************************************************************** * Templates ****************************************************************/ /* @@ -144,8 +141,7 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; /* Function templates */ -size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) -(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe) +size_t FSE_count_generic(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe) { const FSE_FUNCTION_TYPE* ip = source; const FSE_FUNCTION_TYPE* const iend = ip+sourceSize; @@ -226,7 +222,7 @@ size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION) (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize) { - return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0); + return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0); } size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) @@ -235,25 +231,26 @@ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255)) { *maxSymbolValuePtr = 255; - return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0); + return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0); } - return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 1); + return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 1); } static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } -size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) -(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { const unsigned tableSize = 1 << tableLog; const unsigned tableMask = tableSize - 1; - U16* tableU16 = ( (U16*) ct) + 2; - FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)ct) + 1 + (tableLog ? tableSize>>1 : 1) ); + void* const ptr = ct; + U16* const tableU16 = ( (U16*) ptr) + 2; + void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); const unsigned step = FSE_tableStep(tableSize); unsigned cumul[FSE_MAX_SYMBOL_VALUE+2]; U32 position = 0; - FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* init isn't necessary, even if static analyzer complain about it */ + FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ U32 highThreshold = tableSize-1; unsigned symbol; unsigned i; @@ -269,7 +266,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) cumul[0] = 0; for (i=1; i<=maxSymbolValue+1; i++) { - if (normalizedCounter[i-1]==-1) /* Low prob symbol */ + if (normalizedCounter[i-1]==-1) /* Low proba symbol */ { cumul[i] = cumul[i-1] + 1; tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1); @@ -287,7 +284,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) { tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; position = (position + step) & tableMask; - while (position > highThreshold) position = (position + step) & tableMask; /* Lowprob area */ + while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */ } } @@ -296,7 +293,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) /* Build table */ for (i=0; i FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); } -void FSE_FUNCTION_NAME(FSE_freeDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt) +void FSE_freeDTable (FSE_DTable* dt) { free(dt); } -size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) -(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DTableHeader DTableH; + void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); const U32 tableSize = 1 << tableLog; const U32 tableMask = tableSize-1; const U32 step = FSE_tableStep(tableSize); @@ -365,7 +360,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ - DTableH[0].tableLog = (U16)tableLog; + DTableH.tableLog = (U16)tableLog; for (s=0; s<=maxSymbolValue; s++) { if (normalizedCounter[s]==-1) @@ -406,7 +401,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) } } - DTableH->fastMode = (U16)noLarge; + DTableH.fastMode = (U16)noLarge; + memcpy(dt, &DTableH, sizeof(DTableH)); return 0; } @@ -890,8 +886,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) const unsigned tableSize = 1 << nbBits; const unsigned tableMask = tableSize - 1; const unsigned maxSymbolValue = tableMask; - U16* tableU16 = ( (U16*) ct) + 2; - FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)ct)+1) + (tableSize>>1)); + void* const ptr = ct; + U16* const tableU16 = ( (U16*) ptr) + 2; + void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); unsigned s; /* Sanity checks */ @@ -918,8 +916,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) /* fake FSE_CTable, for rle (100% always same symbol) input */ size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) { - U16* tableU16 = ( (U16*) ct) + 2; - FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)ct + 2); + void* ptr = ct; + U16* tableU16 = ( (U16*) ptr) + 2; + void* FSCTptr = (U32*)ptr + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr; /* header */ tableU16[-2] = (U16) 0; @@ -1076,8 +1076,10 @@ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize) *********************************************************/ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const cell = (FSE_decode_t*)dPtr; DTableH->tableLog = 0; DTableH->fastMode = 0; @@ -1092,8 +1094,10 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; const unsigned tableSize = 1 << nbBits; const unsigned tableMask = tableSize - 1; const unsigned maxSymbolValue = tableMask; @@ -1189,7 +1193,8 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) { - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; const U32 fastMode = DTableH->fastMode; /* select fast mode (static) */ diff --git a/lib/fse.h b/lib/fse.h index 24d1d26d4..dd1190f86 100644 --- a/lib/fse.h +++ b/lib/fse.h @@ -40,20 +40,20 @@ extern "C" { #endif -/****************************************** +/* ***************************************** * Includes ******************************************/ #include /* size_t, ptrdiff_t */ -/****************************************** +/* ***************************************** * FSE simple functions ******************************************/ size_t FSE_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize); size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize); -/* +/*! FSE_compress(): Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. 'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize) @@ -74,7 +74,7 @@ FSE_decompress(): */ -/****************************************** +/* ***************************************** * Tool functions ******************************************/ size_t FSE_compressBound(size_t size); /* maximum compressed size */ @@ -84,10 +84,10 @@ unsigned FSE_isError(size_t code); /* tells if a return value is an er const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/****************************************** +/* ***************************************** * FSE advanced functions ******************************************/ -/* +/*! FSE_compress2(): Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' Both parameters can be defined as '0' to mean : use default value @@ -99,10 +99,10 @@ FSE_compress2(): size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -/****************************************** +/* ***************************************** * FSE detailed API ******************************************/ -/* +/*! FSE_compress() does the following: 1. count symbol occurrence from source[] into table count[] 2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) @@ -122,7 +122,7 @@ or to save and provide normalized distribution using external method. /* *** COMPRESSION *** */ -/* +/*! FSE_count(): Provides the precise count of each symbol within a table 'count' 'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1). @@ -132,14 +132,14 @@ FSE_count(): if FSE_isError(return), it's an error code. */ size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize); -/* +/*! FSE_optimalTableLog(): dynamically downsize 'tableLog' when conditions are met. It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. return : recommended tableLog (necessarily <= initial 'tableLog') */ unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue); -/* +/*! FSE_normalizeCount(): normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). @@ -147,13 +147,13 @@ FSE_normalizeCount(): or an errorCode, which can be tested using FSE_isError() */ size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); -/* +/*! FSE_NCountWriteBound(): Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog' Typically useful for allocation purpose. */ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); -/* +/*! FSE_writeNCount(): Compactly save 'normalizedCounter' into 'buffer'. return : size of the compressed table @@ -161,21 +161,21 @@ FSE_writeNCount(): size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); -/* +/*! Constructor and Destructor of type FSE_CTable Note that its size depends on 'tableLog' and 'maxSymbolValue' */ -typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); void FSE_freeCTable (FSE_CTable* ct); -/* +/*! FSE_buildCTable(): Builds 'ct', which must be already allocated, using FSE_createCTable() return : 0 or an errorCode, which can be tested using FSE_isError() */ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); -/* +/*! FSE_compress_usingCTable(): Compress 'src' using 'ct' into 'dst' which must be already allocated return : size of compressed data (<= maxDstSize) @@ -183,7 +183,7 @@ FSE_compress_usingCTable(): or an errorCode, which can be tested using FSE_isError() */ size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct); -/* +/*! Tutorial : ---------- The first step is to count all symbols. FSE_count() does this job very fast. @@ -229,7 +229,7 @@ If there is an error, the function will return an ErrorCode (which can be tested /* *** DECOMPRESSION *** */ -/* +/*! FSE_readNCount(): Read compactly saved 'normalizedCounter' from 'rBuffer'. return : size read from 'rBuffer' @@ -237,21 +237,21 @@ FSE_readNCount(): maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); -/* +/*! Constructor and Destructor of type FSE_DTable Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ FSE_DTable* FSE_createDTable(unsigned tableLog); void FSE_freeDTable(FSE_DTable* dt); -/* +/*! FSE_buildDTable(): Builds 'dt', which must be already allocated, using FSE_createDTable() return : 0, or an errorCode, which can be tested using FSE_isError() */ size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); -/* +/*! FSE_decompress_usingDTable(): Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt' into 'dst' which must be already allocated. @@ -259,7 +259,7 @@ FSE_decompress_usingDTable(): or an errorCode, which can be tested using FSE_isError() */ size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); -/* +/*! Tutorial : ---------- (Note : these functions only decompress FSE-compressed blocks. diff --git a/lib/fse_static.h b/lib/fse_static.h index 01e25669f..c5c6f20c6 100644 --- a/lib/fse_static.h +++ b/lib/fse_static.h @@ -40,31 +40,31 @@ extern "C" { #endif -/****************************************** -* FSE API compatible with DLL -******************************************/ +/* ***************************************** +* Dependencies +*******************************************/ #include "fse.h" #include "bitstream.h" -/****************************************** +/* ***************************************** * Static allocation -******************************************/ +*******************************************/ /* FSE buffer bounds */ #define FSE_NCOUNTBOUND 512 #define FSE_BLOCKBOUND(size) (size + (size>>7)) #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ -/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */ +/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= 1 (otherwise, result will be corrupted) */ -/****************************************** -* Implementation of inline functions -******************************************/ +/* ***************************************** +* Implementation of inlined functions +*******************************************/ typedef struct { int deltaFindState; @@ -231,10 +231,12 @@ typedef struct MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) { - const U32 tableLog = ( (const U16*) ct) [0]; + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = *u16ptr; statePtr->value = (ptrdiff_t)1<stateTable = ((const U16*) ct) + 2; - statePtr->symbolTT = (const void*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); + statePtr->stateTable = u16ptr+2; + statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); statePtr->stateLog = tableLog; } @@ -269,7 +271,8 @@ typedef struct MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) { - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); BIT_reloadDStream(bitD); DStatePtr->table = dt + 1; diff --git a/lib/huff0.c b/lib/huff0.c index 3ddb7e5e3..26a7639ff 100644 --- a/lib/huff0.c +++ b/lib/huff0.c @@ -32,7 +32,7 @@ - Public forum : https://groups.google.com/forum/#!forum/lz4c ****************************************************************** */ -/**************************************************************** +/* ************************************************************** * Compiler specifics ****************************************************************/ #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) @@ -57,7 +57,7 @@ #endif -/**************************************************************** +/* ************************************************************** * Includes ****************************************************************/ #include /* malloc, free, qsort */ @@ -68,7 +68,7 @@ #include "fse.h" /* header compression */ -/**************************************************************** +/* ************************************************************** * Constants ****************************************************************/ #define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ @@ -80,27 +80,21 @@ #endif -/**************************************************************** +/* ************************************************************** * Error Management ****************************************************************/ +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -/****************************************** -* Helper functions -******************************************/ -unsigned HUF_isError(size_t code) { return ERR_isError(code); } - -const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/********************************************************* +/* ******************************************************* * Huff0 : Huffman block compression *********************************************************/ -typedef struct HUF_CElt_s { +struct HUF_CElt_s { U16 val; BYTE nbBits; -} HUF_CElt ; +}; /* typedef'd to HUF_CElt within huff0_static.h */ typedef struct nodeElt_s { U32 count; @@ -320,7 +314,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 /* sort, decreasing order */ HUF_sort(huffNode, count, maxSymbolValue); - // init for parents + /* init for parents */ nonNullRank = maxSymbolValue; while(huffNode[nonNullRank].count == 0) nonNullRank--; lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; @@ -330,7 +324,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); huffNode0[0].count = (U32)(1U<<31); - // create parents + /* create parents */ while (nodeNb <= nodeRoot) { U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; @@ -340,7 +334,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 nodeNb++; } - // distribute weights (unlimited tree height) + /* distribute weights (unlimited tree height) */ huffNode[nodeRoot].nbBits = 0; for (n=nodeRoot-1; n>=STARTNODE; n--) huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; @@ -368,9 +362,9 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 } } for (n=0; n<=maxSymbolValue; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; // push nbBits per symbol, symbol order + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ for (n=0; n<=maxSymbolValue; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; // assign value within rank, symbol order + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ } return maxNbBits; @@ -636,12 +630,12 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ U32 tableLog = 0; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; U32 nbSymbols = 0; U32 n; U32 nextRankStart; - HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1); + void* const dtPtr = DTable + 1; + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ @@ -730,7 +724,8 @@ size_t HUF_decompress1X2_usingDTable( BYTE* const oend = op + dstSize; size_t errorCode; const U32 dtLog = DTable[0]; - const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1; + const void* dtPtr = DTable; + const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1; BIT_DStream_t bitD; errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); if (HUF_isError(errorCode)) return errorCode; @@ -770,8 +765,8 @@ size_t HUF_decompress4X2_usingDTable( const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - - const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1; + const void* const dtPtr = DTable; + const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1; const U32 dtLog = DTable[0]; size_t errorCode; @@ -978,9 +973,9 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) rankVal_t rankVal; U32 tableLog, maxW, sizeOfSort, nbSymbols; const U32 memLog = DTable[0]; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1; + size_t iSize; + void* dtPtr = DTable; + HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1; HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); @@ -1127,7 +1122,8 @@ size_t HUF_decompress1X4_usingDTable( BYTE* const oend = ostart + dstSize; const U32 dtLog = DTable[0]; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1; + const void* const dtPtr = DTable; + const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; size_t errorCode; /* Init */ @@ -1170,8 +1166,8 @@ size_t HUF_decompress4X4_usingDTable( const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1; + const void* const dtPtr = DTable; + const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; const U32 dtLog = DTable[0]; size_t errorCode; @@ -1352,8 +1348,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) U32 tableLog, maxW, sizeOfSort, nbSymbols; rankVal_t rankVal; const U32 memLog = DTable[0]; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ @@ -1418,8 +1413,10 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) /* fill tables */ { - HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1); - HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1))); + void* ddPtr = DTable+1; + HUF_DDescX6* DDescription = (HUF_DDescX6*)ddPtr; + void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1)); + HUF_DSeqX6* DSequence = (HUF_DSeqX6*)dsPtr; HUF_DSeqX6 DSeq; HUF_DDescX6 DDesc; DSeq.sequence = 0; @@ -1478,8 +1475,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog) { - const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); - const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); + const void* const ddPtr = DTable+1; + const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr; + const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); + const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr; BYTE* const pStart = p; /* up to 16 symbols at a time */ @@ -1557,8 +1556,10 @@ size_t HUF_decompress4X6_usingDTable( BYTE* const oend = ostart + dstSize; const U32 dtLog = DTable[0]; - const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); - const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); + const void* const ddPtr = DTable+1; + const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr; + const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); + const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr; size_t errorCode; /* Init */ diff --git a/lib/huff0.h b/lib/huff0.h index 613d9b9cc..2ebd5cfbd 100644 --- a/lib/huff0.h +++ b/lib/huff0.h @@ -40,25 +40,25 @@ extern "C" { #endif -/****************************************** +/* **************************************** * Dependency ******************************************/ #include /* size_t */ -/****************************************** +/* **************************************** * Huff0 simple functions ******************************************/ size_t HUF_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize); size_t HUF_decompress(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -/* +/*! HUF_compress(): Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. 'dst' buffer must be already allocated. Compression runs faster if maxDstSize >= HUF_compressBound(srcSize). Note : srcSize must be <= 128 KB - return : size of compressed data (<= maxDstSize) + @return : size of compressed data (<= maxDstSize) Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! if return == 1, srcData is a single repeated byte symbol (RLE compression) if HUF_isError(return), compression failed (more details using HUF_getErrorName()) @@ -68,12 +68,12 @@ HUF_decompress(): into already allocated destination buffer 'dst', of size 'dstSize'. 'dstSize' must be the exact size of original (uncompressed) data. Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate. - return : size of regenerated data (== dstSize) - or an error code, which can be tested using HUF_isError() + @return : size of regenerated data (== dstSize) + or an error code, which can be tested using HUF_isError() */ -/****************************************** +/* **************************************** * Tool functions ******************************************/ size_t HUF_compressBound(size_t size); /* maximum compressed size */ @@ -83,7 +83,7 @@ unsigned HUF_isError(size_t code); /* tells if a return value is an er const char* HUF_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/****************************************** +/* **************************************** * Advanced functions ******************************************/ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); diff --git a/lib/huff0_static.h b/lib/huff0_static.h index e8af19ede..5df0727d7 100644 --- a/lib/huff0_static.h +++ b/lib/huff0_static.h @@ -40,13 +40,13 @@ extern "C" { #endif -/****************************************** +/* **************************************** * Dependency ******************************************/ #include "huff0.h" -/****************************************** +/* **************************************** * Static allocation macros ******************************************/ /* Huff0 buffer bounds */ @@ -64,14 +64,57 @@ extern "C" { unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog } -/****************************************** -* Advanced functions +/* **************************************** +* Advanced decompression functions ******************************************/ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */ +/* **************************************** +* Huff0 detailed API +******************************************/ +/*! +HUF_compress() does the following: +1. count symbol occurrence from source[] into table count[] using FSE_count() +2. build Huffman table from count using HUF_buildCTable() +3. save Huffman table to memory buffer using HUF_writeCTable() +4. encode the data stream using HUF_compress_usingCTable() + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ + +/* FSE_count() : find it within "fse.h" */ + +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + + +/*! +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable + +*/ +size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbol decoder */ + +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); +size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); + + #if defined (__cplusplus) } #endif From 218bd319425bfebffd9fb262b12f2f811973d889 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jan 2016 02:19:55 +0100 Subject: [PATCH 18/27] minor strict aliasing warning --- lib/fse_static.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fse_static.h b/lib/fse_static.h index c5c6f20c6..a881e351f 100644 --- a/lib/fse_static.h +++ b/lib/fse_static.h @@ -233,7 +233,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) { const void* ptr = ct; const U16* u16ptr = (const U16*) ptr; - const U32 tableLog = *u16ptr; + const U32 tableLog = MEM_read16(ptr); statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); From 1fdd82312d156674ec1b1fc184d391204e3ac577 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jan 2016 12:35:42 +0100 Subject: [PATCH 19/27] Fixed : strict aliasing warnings in legacy decoders --- lib/legacy/zstd_v01.c | 39 +++++++++++++++++++-------------- lib/legacy/zstd_v02.c | 51 ++++++++++++++++++++++++++++--------------- lib/legacy/zstd_v03.c | 51 ++++++++++++++++++++++++++++--------------- programs/Makefile | 2 +- 4 files changed, 90 insertions(+), 53 deletions(-) diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index ccd57d5b7..29d95e5b2 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -418,8 +418,9 @@ typedef struct { static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(ptr) + 1; /* because dt is unsigned, 32-bits aligned on 32-bits */ const U32 tableSize = 1 << tableLog; const U32 tableMask = tableSize-1; const U32 step = FSE_tableStep(tableSize); @@ -615,8 +616,9 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi *********************************************************/ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ DTableH->tableLog = 0; DTableH->fastMode = 0; @@ -631,8 +633,9 @@ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ const unsigned tableSize = 1 << nbBits; const unsigned tableMask = tableSize - 1; const unsigned maxSymbolValue = tableMask; @@ -701,7 +704,7 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t } -/* FSE_lookBits +/*!FSE_lookBits * Provides next n bits from the bitContainer. * bitContainer is not modified (bits are still present for next read/look) * On 32-bits, maxNbBits==25 @@ -726,7 +729,7 @@ static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits) } -/* FSE_readBits +/*!FSE_readBits * Read next n bits from the bitContainer. * On 32-bits, don't read more than maxNbBits==25 * On 64-bits, don't read more than maxNbBits==57 @@ -782,7 +785,8 @@ static unsigned FSE_reloadDStream(FSE_DStream_t* bitD) static void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt) { - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; DStatePtr->state = FSE_readBits(bitD, DTableH->tableLog); FSE_reloadDStream(bitD); DStatePtr->table = dt + 1; @@ -898,7 +902,8 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) { - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; const U32 fastMode = DTableH->fastMode; /* select fast mode (static) */ @@ -935,7 +940,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz -/********************************************************* +/* ******************************************************* * Huff0 : Huffman block compression *********************************************************/ #define HUF_MAX_SYMBOL_VALUE 255 @@ -959,7 +964,7 @@ typedef struct nodeElt_s { } nodeElt; -/********************************************************* +/* ******************************************************* * Huff0 : Huffman block decompression *********************************************************/ typedef struct { @@ -978,7 +983,8 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize) size_t oSize; U32 n; U32 nextRankStart; - HUF_DElt* const dt = (HUF_DElt*)(DTable + 1); + void* ptr = DTable+1; + HUF_DElt* const dt = (HUF_DElt*)ptr; FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */ //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */ @@ -1082,7 +1088,8 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */ BYTE* const omax = op + maxDstSize; BYTE* const olimit = omax-15; - const HUF_DElt* const dt = (const HUF_DElt*)(DTable+1); + const void* ptr = DTable; + const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1; const U32 dtLog = DTable[0]; size_t errorCode; U32 reloadStatus; @@ -1988,8 +1995,8 @@ static size_t ZSTD_decompressBlock( { /* blockType == blockCompressed, srcSize is trusted */ const BYTE* ip = (const BYTE*)src; - const BYTE* litPtr; - size_t litSize; + const BYTE* litPtr = NULL; + size_t litSize = 0; size_t errorCode; /* Decode literals sub-block */ diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index 67d726f64..cfb1788fb 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -891,7 +891,8 @@ typedef struct MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) { - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); BIT_reloadDStream(bitD); DStatePtr->table = dt + 1; @@ -1272,8 +1273,9 @@ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(ptr) + 1; const U32 tableSize = 1 << tableLog; const U32 tableMask = tableSize-1; const U32 step = FSE_tableStep(tableSize); @@ -1468,8 +1470,9 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi *********************************************************/ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ DTableH->tableLog = 0; DTableH->fastMode = 0; @@ -1484,8 +1487,9 @@ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ const unsigned tableSize = 1 << nbBits; const unsigned tableMask = tableSize - 1; const unsigned maxSymbolValue = tableMask; @@ -1581,7 +1585,8 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) { - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; const U32 fastMode = DTableH->fastMode; /* select fast mode (static) */ @@ -1810,7 +1815,8 @@ static size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) U32 nbSymbols = 0; U32 n; U32 nextRankStart; - HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1); + void* ptr = DTable+1; + HUF_DEltX2* const dt = (HUF_DEltX2*)ptr; HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ @@ -1903,7 +1909,8 @@ static size_t HUF_decompress4X2_usingDTable( BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1; + const void* ptr = DTable; + const HUF_DEltX2* const dt = ((const HUF_DEltX2*)ptr) +1; const U32 dtLog = DTable[0]; size_t errorCode; @@ -2112,7 +2119,8 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) const U32 memLog = DTable[0]; const BYTE* ip = (const BYTE*) src; size_t iSize = ip[0]; - HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1; + void* ptr = DTable; + HUF_DEltX4* const dt = ((HUF_DEltX4*)ptr) + 1; HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); @@ -2262,7 +2270,8 @@ static size_t HUF_decompress4X4_usingDTable( BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1; + const void* ptr = DTable; + const HUF_DEltX4* const dt = ((const HUF_DEltX4*)ptr) +1; const U32 dtLog = DTable[0]; size_t errorCode; @@ -2510,8 +2519,10 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) /* fill tables */ { - HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1); - HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1))); + void* ptr = DTable+1; + HUF_DDescX6* DDescription = (HUF_DDescX6*)(ptr); + void* dSeqStart = DTable + 1 + ((size_t)1<<(memLog-1)); + HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(dSeqStart); HUF_DSeqX6 DSeq; HUF_DDescX6 DDesc; DSeq.sequence = 0; @@ -2570,8 +2581,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog) { - const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); - const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); + const void* ddPtr = DTable+1; + const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr); + const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); + const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr); BYTE* const pStart = p; /* up to 16 symbols at a time */ @@ -2611,8 +2624,10 @@ static size_t HUF_decompress4X6_usingDTable( BYTE* const oend = ostart + dstSize; const U32 dtLog = DTable[0]; - const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); - const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); + const void* ddPtr = DTable+1; + const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr); + const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); + const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr); size_t errorCode; /* Init */ diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index ae67941c9..1aed028e8 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -891,7 +891,8 @@ typedef struct MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) { - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); BIT_reloadDStream(bitD); DStatePtr->table = dt + 1; @@ -1272,8 +1273,9 @@ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(ptr) + 1; /* because dt is unsigned, 32-bits aligned on 32-bits */ const U32 tableSize = 1 << tableLog; const U32 tableMask = tableSize-1; const U32 step = FSE_tableStep(tableSize); @@ -1468,8 +1470,9 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi *********************************************************/ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1; DTableH->tableLog = 0; DTableH->fastMode = 0; @@ -1484,8 +1487,9 @@ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) { - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; - FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1; const unsigned tableSize = 1 << nbBits; const unsigned tableMask = tableSize - 1; const unsigned maxSymbolValue = tableMask; @@ -1581,7 +1585,8 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) { - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt; + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; const U32 fastMode = DTableH->fastMode; /* select fast mode (static) */ @@ -1810,7 +1815,8 @@ static size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) U32 nbSymbols = 0; U32 n; U32 nextRankStart; - HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1); + void* ptr = DTable+1; + HUF_DEltX2* const dt = (HUF_DEltX2*)(ptr); HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ @@ -1903,7 +1909,8 @@ static size_t HUF_decompress4X2_usingDTable( BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1; + const void* ptr = DTable; + const HUF_DEltX2* const dt = ((const HUF_DEltX2*)ptr) +1; const U32 dtLog = DTable[0]; size_t errorCode; @@ -2112,7 +2119,8 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) const U32 memLog = DTable[0]; const BYTE* ip = (const BYTE*) src; size_t iSize = ip[0]; - HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1; + void* ptr = DTable; + HUF_DEltX4* const dt = ((HUF_DEltX4*)ptr) + 1; HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); @@ -2262,7 +2270,8 @@ static size_t HUF_decompress4X4_usingDTable( BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1; + const void* ptr = DTable; + const HUF_DEltX4* const dt = ((const HUF_DEltX4*)ptr) +1; const U32 dtLog = DTable[0]; size_t errorCode; @@ -2510,8 +2519,10 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) /* fill tables */ { - HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1); - HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1))); + void* ddPtr = DTable+1; + HUF_DDescX6* DDescription = (HUF_DDescX6*)(ddPtr); + void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1)); + HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(dsPtr); HUF_DSeqX6 DSeq; HUF_DDescX6 DDesc; DSeq.sequence = 0; @@ -2570,8 +2581,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog) { - const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); - const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); + const void* ddPtr = DTable+1; + const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr); + const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); + const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr); BYTE* const pStart = p; /* up to 16 symbols at a time */ @@ -2611,8 +2624,10 @@ static size_t HUF_decompress4X6_usingDTable( BYTE* const oend = ostart + dstSize; const U32 dtLog = DTable[0]; - const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); - const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); + const void* ddPtr = DTable+1; + const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr); + const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); + const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr); size_t errorCode; /* Init */ diff --git a/programs/Makefile b/programs/Makefile index f2230d973..182f91a38 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -45,7 +45,7 @@ DESTDIR?= PREFIX ?= /usr/local CPPFLAGS= -I../lib -DZSTD_VERSION=\"$(VERSION)\" CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1 FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) BINDIR = $(PREFIX)/bin From 494c786a3bf023c38818d206bb0572dd86f45ab5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jan 2016 12:54:02 +0100 Subject: [PATCH 20/27] fixed outstanding strict aliasing warnings --- lib/legacy/zstd_v01.c | 7 +++---- lib/legacy/zstd_v02.c | 24 ++++++++++++------------ lib/legacy/zstd_v03.c | 24 ++++++++++++------------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 29d95e5b2..7a8486b58 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -902,12 +902,11 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) { - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); /* memcpy() into local variable, to avoid strict aliasing warning */ /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); } diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index cfb1788fb..73136f81a 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -891,9 +891,9 @@ typedef struct MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) { - const void* ptr = dt; - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); + DStatePtr->state = BIT_readBits(bitD, DTableH.tableLog); BIT_reloadDStream(bitD); DStatePtr->table = dt + 1; } @@ -1273,9 +1273,9 @@ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(ptr) + 1; + void* ptr = dt+1; + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)ptr; + FSE_DTableHeader DTableH; const U32 tableSize = 1 << tableLog; const U32 tableMask = tableSize-1; const U32 step = FSE_tableStep(tableSize); @@ -1291,7 +1291,7 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ - DTableH[0].tableLog = (U16)tableLog; + DTableH.tableLog = (U16)tableLog; for (s=0; s<=maxSymbolValue; s++) { if (normalizedCounter[s]==-1) @@ -1332,7 +1332,8 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) } } - DTableH->fastMode = (U16)noLarge; + DTableH.fastMode = (U16)noLarge; + memcpy(dt, &DTableH, sizeof(DTableH)); /* memcpy(), to avoid strict aliasing warnings */ return 0; } @@ -1585,12 +1586,11 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) { - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); } diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 1aed028e8..5c75eb4e4 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -891,9 +891,9 @@ typedef struct MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) { - const void* ptr = dt; - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); + DStatePtr->state = BIT_readBits(bitD, DTableH.tableLog); BIT_reloadDStream(bitD); DStatePtr->table = dt + 1; } @@ -1273,9 +1273,9 @@ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(ptr) + 1; /* because dt is unsigned, 32-bits aligned on 32-bits */ + void* ptr = dt+1; + FSE_DTableHeader DTableH; + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)ptr; const U32 tableSize = 1 << tableLog; const U32 tableMask = tableSize-1; const U32 step = FSE_tableStep(tableSize); @@ -1291,7 +1291,7 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ - DTableH[0].tableLog = (U16)tableLog; + DTableH.tableLog = (U16)tableLog; for (s=0; s<=maxSymbolValue; s++) { if (normalizedCounter[s]==-1) @@ -1332,7 +1332,8 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) } } - DTableH->fastMode = (U16)noLarge; + DTableH.fastMode = (U16)noLarge; + memcpy(dt, &DTableH, sizeof(DTableH)); return 0; } @@ -1585,12 +1586,11 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) { - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); } From ecd651bd91b41b12fe389b27e3ee405164e52697 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jan 2016 15:35:18 +0100 Subject: [PATCH 21/27] added ZSTD_duplicateCCtx() --- NEWS | 1 + lib/Makefile | 2 +- lib/zstd.h | 16 +++++-- lib/zstd_buffered.c | 2 +- lib/zstd_compress.c | 101 +++++++++++++++++++++++++++++++++----------- lib/zstd_static.h | 36 ++++++++-------- programs/fuzzer.c | 14 ++++-- 7 files changed, 119 insertions(+), 53 deletions(-) diff --git a/NEWS b/NEWS index d60b363a4..f99b0376b 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v0.4.6 fix : fast compression mode on Windows Improved : high compression mode on repetitive data +Added : ZSTD_duplicateCCtx() v0.4.5 new : -m/--multiple : compress/decompress multiple files diff --git a/lib/Makefile b/lib/Makefile index a4a156d85..a7172b6f8 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -46,7 +46,7 @@ DESTDIR?= PREFIX ?= /usr/local CPPFLAGS= -I. CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1 FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) LIBDIR ?= $(PREFIX)/lib diff --git a/lib/zstd.h b/lib/zstd.h index bba096144..e201b5e0b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -107,16 +107,24 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /** provides error co /* ************************************* * Advanced functions ***************************************/ +/** Compression context management */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); -/** -ZSTD_compressCCtx() : - Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated -*/ +/** ZSTD_compressCCtx() : + Same as ZSTD_compress(), but requires an already allocated ZSTD_CCtx (see ZSTD_createCCtx()) */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel); +/** Decompression context management */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); + +/** ZSTD_decompressDCtx +* Same as ZSTD_decompress(), but requires an already allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); + #if defined (__cplusplus) } diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index 99148c297..8a7fa1d11 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -143,7 +143,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params) if (zbc->outBuff == NULL) return ERROR(memory_allocation); } - zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, zbc->outBuff, zbc->outBuffSize, params); + zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, params); if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize; zbc->inToCompress = 0; diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 5db0fdf5d..4044ac741 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -65,8 +65,9 @@ /* ************************************* * Constants ***************************************/ -unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +ZSTDLIB_API unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const U32 g_searchStrength = 8; +static const size_t g_hbSize = (((ZSTD_frameHeaderSize_max+15)/8)*8); /* ************************************* @@ -109,10 +110,14 @@ struct ZSTD_CCtx_s U32 dictLimit; /* below that point, need extDict */ U32 lowLimit; /* below that point, no more data */ U32 nextToUpdate; /* index from which to continue dictionary update */ + U32 stage; ZSTD_parameters params; void* workSpace; size_t workSpaceSize; size_t blockSize; + void* headerBuffer; + size_t hbSize; + seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -205,6 +210,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize; zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2); zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2); + zc->headerBuffer = (char*)zc->workSpace + zc->workSpaceSize - g_hbSize; + zc->hbSize = 0; return 0; } @@ -790,7 +797,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, const U32 current = (U32)(ip-base); hashTable[h] = current; /* update hash table */ - if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) /* note : by construction, offset_1 <= (ip-base) */ + if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) /* note : by construction, offset_1 <= current */ { mlCode = ZSTD_count(ip+1+MINMATCH, ip+1+MINMATCH-offset_1, iend); ip++; @@ -819,7 +826,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, if (ip <= ilimit) { /* Fill Table */ - hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 without ip <= ilimit check*/ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); /* check immediate repcode */ while ( (ip <= ilimit) @@ -1852,6 +1859,17 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, const void* src, size_t srcSize) { const BYTE* const ip = (const BYTE*) src; + size_t hbSize = 0; + + if (zc->stage==0) + { + hbSize = zc->hbSize; + if (dstSize <= hbSize) return ERROR(dstSize_tooSmall); + zc->stage = 1; + memcpy(dst, zc->headerBuffer, hbSize); + dstSize -= hbSize; + dst = (char*)dst + hbSize; + } /* Check if blocks follow each other */ if (src != zc->nextSrc) @@ -1890,8 +1908,11 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, } zc->nextSrc = ip + srcSize; - - return ZSTD_compress_generic (zc, dst, dstSize, src, srcSize); + { + size_t cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize); + if (ZSTD_isError(cSize)) return cSize; + return cSize + hbSize; + } } size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize) @@ -1934,24 +1955,49 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src } +/*! ZSTD_duplicateCCtx +* Duplicate an existing context @srcCCtx into another one @dstCCtx. +* Only works during stage 0 (i.e. before first call to ZSTD_compressContinue()) +* @return : 0, or an error code */ +size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) +{ + void* dstWorkSpace = dstCCtx->workSpace; + size_t dstWorkSpaceSize = dstCCtx->workSpaceSize; + + if (dstWorkSpaceSize < srcCCtx->workSpaceSize) + { + free(dstCCtx->workSpace); + dstWorkSpaceSize = srcCCtx->workSpaceSize; + dstWorkSpace = malloc(dstWorkSpaceSize); + if (dstWorkSpace==NULL) return ERROR(memory_allocation); + } + + memcpy(dstCCtx, srcCCtx, sizeof(*dstCCtx)); + dstCCtx->workSpace = dstWorkSpace; + dstCCtx->workSpaceSize = dstWorkSpaceSize; + + return 0; +} + + /*! ZSTD_compressBegin_advanced -* Write frame header, according to params -* @return : nb of bytes written */ +* @return : 0, or an error code */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, ZSTD_parameters params) { size_t errorCode; ZSTD_validateParams(¶ms); - if (maxDstSize < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); errorCode = ZSTD_resetCCtx_advanced(ctx, params); if (ZSTD_isError(errorCode)) return errorCode; - MEM_writeLE32(dst, ZSTD_MAGICNUMBER); /* Write Header */ - ((BYTE*)dst)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); - return ZSTD_frameHeaderSize_min; + MEM_writeLE32(ctx->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */ + ((BYTE*)ctx->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); + ctx->hbSize = ZSTD_frameHeaderSize_min; + ctx->stage = 0; + + return 0; } @@ -1970,29 +2016,38 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) } -size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel) +size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, int compressionLevel) { - return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_getParams(compressionLevel, 0)); + return ZSTD_compressBegin_advanced(ctx, ZSTD_getParams(compressionLevel, 0)); } /*! ZSTD_compressEnd * Write frame epilogue * @return : nb of bytes written into dst (or an error code) */ -size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize) +size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) { BYTE* op = (BYTE*)dst; + size_t hbSize = 0; - /* Sanity check */ - (void)ctx; + /* empty frame */ + if (zc->stage==0) + { + hbSize = zc->hbSize; + if (maxDstSize <= hbSize) return ERROR(dstSize_tooSmall); + zc->stage = 1; + memcpy(dst, zc->headerBuffer, hbSize); + maxDstSize -= hbSize; + op += hbSize; + } + + /* frame epilogue */ if (maxDstSize < 3) return ERROR(dstSize_tooSmall); - - /* End of frame */ op[0] = (BYTE)(bt_end << 6); op[1] = 0; op[2] = 0; - return 3; + return 3+hbSize; } size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, @@ -2006,10 +2061,8 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t oSize; /* Header */ - oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params); + oSize = ZSTD_compressBegin_advanced(ctx, params); if(ZSTD_isError(oSize)) return oSize; - op += oSize; - maxDstSize -= oSize; /* dictionary */ if (dict) @@ -2048,7 +2101,7 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); result = ZSTD_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel); - free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; free heap content */ + free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 32ef67eec..62bd93577 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -33,9 +33,9 @@ #ifndef ZSTD_STATIC_H #define ZSTD_STATIC_H -/* The objects defined into this file should be considered experimental. - * They are not labelled stable, as their prototype may change in the future. - * You can use them for tests, provide feedback, or if you can endure risk of future changes. +/* The objects defined into this file shall be considered experimental. + * They are not considered stable, as their prototype may change in the future. + * You can use them for tests, provide feedback, or if you can endure risks of future changes. */ #if defined (__cplusplus) @@ -108,40 +108,33 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params); -/** Decompression context management */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); -ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); - -/** ZSTD_decompressDCtx -* Same as ZSTD_decompress, with pre-allocated DCtx structure */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); - /** ZSTD_decompress_usingDict * Same as ZSTD_decompressDCtx, using a Dictionary content as prefix * Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - const void* dict, size_t dictSize); + const void* src, size_t srcSize, + const void* dict,size_t dictSize); /* ************************************** * Streaming functions (direct mode) ****************************************/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters params); + ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); /** - Streaming compression, direct mode (bufferless) + Streaming compression, synchronous mode (bufferless) A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it. - A ZSTD_CCtx object can be re-used multiple times. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. First operation is to start a new frame. Use ZSTD_compressBegin(). @@ -151,8 +144,13 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz Note that dictionary presence is a "hidden" information, the decoder needs to be aware that it is required for proper decoding, or decoding will fail. + If you want to compress multiple messages using same dictionary, + it can be beneficial to duplicate compression context rather than reloading dictionary each time. + In such case, use ZSTD_duplicateCCtx(), which will need an already created ZSTD_CCtx, + in order to duplicate compression context into it. + Then, consume your input using ZSTD_compressContinue(). - The interface is synchronous, so all input will be consumed. + The interface is synchronous, so all input will be consumed and produce a compressed output. You must ensure there is enough space in destination buffer to store compressed data under worst case scenario. Worst case evaluation is provided by ZSTD_compressBound(). diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 025fc5d6a..801ae75bb 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -268,11 +268,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; + ZSTD_CCtx* refCtx; ZSTD_CCtx* ctx; ZSTD_DCtx* dctx; U32 startTime = FUZ_GetMilliStart(); /* allocation */ + refCtx = ZSTD_createCCtx(); ctx = ZSTD_createCCtx(); dctx= ZSTD_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -284,7 +286,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit mirrorBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] - || !dstBuffer || !mirrorBuffer || !cBuffer || !ctx || !dctx, + || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ @@ -461,10 +463,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit dict = srcBuffer + sampleStart; dictSize = sampleSize; - cSize = ZSTD_compressBegin(ctx, cBuffer, cBufferSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); - errorCode = ZSTD_compress_insertDictionary(ctx, dict, dictSize); + errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); + CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode)); + errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize); CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode)); - totalTestSize = 0; + errorCode = ZSTD_duplicateCCtx(ctx, refCtx); + CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode)); + totalTestSize = 0; cSize = 0; for (n=0; n Date: Thu, 7 Jan 2016 23:07:44 +0100 Subject: [PATCH 22/27] fixed context copy --- lib/error.h | 2 +- lib/zstd_compress.c | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/error.h b/lib/error.h index 3c39c39ca..af3c1b569 100644 --- a/lib/error.h +++ b/lib/error.h @@ -69,7 +69,7 @@ extern "C" { #define ERROR_LIST(ITEM) \ ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \ ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(frameParameter_unsupported)) ITEM(PREFIX(frameParameter_unsupportedBy32bitsImplementation)) \ - ITEM(PREFIX(init_missing)) ITEM(PREFIX(memory_allocation)) \ + ITEM(PREFIX(init_missing)) ITEM(PREFIX(memory_allocation)) ITEM(PREFIX(stage_wrong)) \ ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \ ITEM(PREFIX(corruption_detected)) \ ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \ diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 4044ac741..c167d7824 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1963,11 +1963,17 @@ size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { void* dstWorkSpace = dstCCtx->workSpace; size_t dstWorkSpaceSize = dstCCtx->workSpaceSize; + const U32 contentLog = (srcCCtx->params.strategy == ZSTD_fast) ? 1 : srcCCtx->params.contentLog; + const size_t tableSpace = ((1 << contentLog) + (1 << srcCCtx->params.hashLog)) * sizeof(U32); + const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << srcCCtx->params.windowLog); + const size_t neededSpace = tableSpace + (3*blockSize); - if (dstWorkSpaceSize < srcCCtx->workSpaceSize) + if (srcCCtx->stage!=0) return ERROR(stage_wrong); + + if (dstWorkSpaceSize < neededSpace) { - free(dstCCtx->workSpace); - dstWorkSpaceSize = srcCCtx->workSpaceSize; + free(dstWorkSpace); + dstWorkSpaceSize = neededSpace; dstWorkSpace = malloc(dstWorkSpaceSize); if (dstWorkSpace==NULL) return ERROR(memory_allocation); } @@ -1976,6 +1982,8 @@ size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) dstCCtx->workSpace = dstWorkSpace; dstCCtx->workSpaceSize = dstWorkSpaceSize; + memcpy(dstWorkSpace, srcCCtx->workSpace, tableSpace); + return 0; } From 600962711da2ede9322296a76937653338126558 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jan 2016 17:27:50 +0100 Subject: [PATCH 23/27] added duplication tests --- lib/zstd_compress.c | 33 ++++++++++++------------- programs/fuzzer.c | 59 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c167d7824..6f94922f3 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -115,8 +115,8 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; - void* headerBuffer; size_t hbSize; + char headerBuffer[ZSTD_frameHeaderSize_max]; seqStore_t seqStore; /* sequences storage ptrs */ @@ -210,8 +210,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize; zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2); zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2); - zc->headerBuffer = (char*)zc->workSpace + zc->workSpaceSize - g_hbSize; zc->hbSize = 0; + zc->stage = 0; return 0; } @@ -1961,28 +1961,27 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src * @return : 0, or an error code */ size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { - void* dstWorkSpace = dstCCtx->workSpace; - size_t dstWorkSpaceSize = dstCCtx->workSpaceSize; const U32 contentLog = (srcCCtx->params.strategy == ZSTD_fast) ? 1 : srcCCtx->params.contentLog; const size_t tableSpace = ((1 << contentLog) + (1 << srcCCtx->params.hashLog)) * sizeof(U32); - const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << srcCCtx->params.windowLog); - const size_t neededSpace = tableSpace + (3*blockSize); if (srcCCtx->stage!=0) return ERROR(stage_wrong); - if (dstWorkSpaceSize < neededSpace) - { - free(dstWorkSpace); - dstWorkSpaceSize = neededSpace; - dstWorkSpace = malloc(dstWorkSpaceSize); - if (dstWorkSpace==NULL) return ERROR(memory_allocation); - } + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params); - memcpy(dstCCtx, srcCCtx, sizeof(*dstCCtx)); - dstCCtx->workSpace = dstWorkSpace; - dstCCtx->workSpaceSize = dstWorkSpaceSize; + /* copy tables */ + memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); - memcpy(dstWorkSpace, srcCCtx->workSpace, tableSpace); + /* copy frame header */ + dstCCtx->hbSize = srcCCtx->hbSize; + memcpy(dstCCtx->headerBuffer , srcCCtx->headerBuffer, srcCCtx->hbSize); + + /* copy dictionary pointers */ + dstCCtx->nextToUpdate= srcCCtx->nextToUpdate; + dstCCtx->nextSrc = srcCCtx->nextSrc; + dstCCtx->base = srcCCtx->base; + dstCCtx->dictBase = srcCCtx->dictBase; + dstCCtx->dictLimit = srcCCtx->dictLimit; + dstCCtx->lowLimit = srcCCtx->lowLimit; return 0; } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 801ae75bb..2d1d45151 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -194,6 +194,65 @@ static int basicUnitTests(U32 seed, double compressibility) if (result != ERROR(srcSize_wrong)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); + /* Dictionary and Duplication tests */ + { + ZSTD_CCtx* ctxOrig = ZSTD_createCCtx(); + ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + const size_t dictSize = 500; + size_t cSizeOrig; + + DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++); + result = ZSTD_compressBegin(ctxOrig, 2); + if (ZSTD_isError(result)) goto _output_error; + result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize); + if (ZSTD_isError(result)) goto _output_error; + result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig); + if (ZSTD_isError(result)) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); + cSize = 0; + result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); + result = ZSTD_decompress_usingDict(dctx, + decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, + compressedBuffer, cSize, + CNBuffer, dictSize); + if (ZSTD_isError(result)) goto _output_error; + ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */ + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++); + cSizeOrig = cSize; + cSize = 0; + result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */ + ZSTD_freeCCtx(ctxDuplicated); + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); + result = ZSTD_decompress_usingDict(dctx, + decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, + compressedBuffer, cSize, + CNBuffer, dictSize); + if (ZSTD_isError(result)) goto _output_error; + ZSTD_freeDCtx(dctx); + DISPLAYLEVEL(4, "OK \n"); + } + /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); From c64c10065876f27c90c821d17224f3e0805eada8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jan 2016 17:54:02 +0100 Subject: [PATCH 24/27] fixed : unused variable --- lib/zstd_compress.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 6f94922f3..3cedeb797 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -67,7 +67,6 @@ ***************************************/ ZSTDLIB_API unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const U32 g_searchStrength = 8; -static const size_t g_hbSize = (((ZSTD_frameHeaderSize_max+15)/8)*8); /* ************************************* From bf42c8e5d8527bd28874fd09005cd61a140e07de Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 9 Jan 2016 01:08:23 +0100 Subject: [PATCH 25/27] block-level API --- NEWS | 3 ++- lib/zstd_compress.c | 37 ++++++++++++++++++++++++++++--------- lib/zstd_decompress.c | 3 +-- lib/zstd_static.h | 31 +++++++++++++++++++++++++++++-- programs/fuzzer.c | 28 ++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 14 deletions(-) diff --git a/NEWS b/NEWS index f99b0376b..fff7bf04e 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,8 @@ v0.4.6 fix : fast compression mode on Windows Improved : high compression mode on repetitive data -Added : ZSTD_duplicateCCtx() +New : block-level API +New : ZSTD_duplicateCCtx() v0.4.5 new : -m/--multiple : compress/decompress multiple files diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 3cedeb797..e66fedbe5 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1794,10 +1794,9 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int } -size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit); - if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */ return blockCompressor(zc, dst, maxDstSize, src, srcSize); } @@ -1827,7 +1826,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr, if (ctxPtr->dictLimit < ctxPtr->lowLimit) ctxPtr->dictLimit = ctxPtr->lowLimit; } - cSize = ZSTD_compressBlock(ctxPtr, op+3, maxDstSize-3, ip, blockSize); + cSize = ZSTD_compressBlock_internal(ctxPtr, op+3, maxDstSize-3, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) @@ -1853,14 +1852,15 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr, } -size_t ZSTD_compressContinue (ZSTD_CCtx* zc, - void* dst, size_t dstSize, - const void* src, size_t srcSize) +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc, + void* dst, size_t dstSize, + const void* src, size_t srcSize, + U32 frame) { const BYTE* const ip = (const BYTE*) src; size_t hbSize = 0; - if (zc->stage==0) + if (frame && (zc->stage==0)) { hbSize = zc->hbSize; if (dstSize <= hbSize) return ERROR(dstSize_tooSmall); @@ -1899,7 +1899,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, else zc->nextToUpdate -= correction; } - /* input-dictionary overlap */ + /* if input and dictionary overlap : reduce dictionary (presumed modified by input) */ if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) { zc->lowLimit = (U32)(ip + srcSize - zc->dictBase); @@ -1908,12 +1908,31 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, zc->nextSrc = ip + srcSize; { - size_t cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize); + size_t cSize; + if (frame) cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize); + else cSize = ZSTD_compressBlock_internal (zc, dst, dstSize, src, srcSize); if (ZSTD_isError(cSize)) return cSize; return cSize + hbSize; } } + +size_t ZSTD_compressContinue (ZSTD_CCtx* zc, + void* dst, size_t dstSize, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_internal(zc, dst, dstSize, src, srcSize, 1); +} + + +size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong); + if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */ + return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0); +} + + size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize) { const BYTE* const ip = (const BYTE*) src; diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 3431e327d..5dec85888 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -658,8 +658,7 @@ static size_t ZSTD_decompressSequences( } -static size_t ZSTD_decompressBlock( - ZSTD_DCtx* dctx, +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 62bd93577..f1e72e955 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz Note that dictionary presence is a "hidden" information, the decoder needs to be aware that it is required for proper decoding, or decoding will fail. - If you want to compress multiple messages using same dictionary, + If you want to compress a lot of messages using same dictionary, it can be beneficial to duplicate compression context rather than reloading dictionary each time. In such case, use ZSTD_duplicateCCtx(), which will need an already created ZSTD_CCtx, in order to duplicate compression context into it. @@ -157,7 +157,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz Finish a frame with ZSTD_compressEnd(), which will write the epilogue. Without it, the frame will be considered incomplete by decoders. - You can then reuse ZSTD_CCtx to compress new frames. + You can then reuse ZSTD_CCtx to compress some new frame. */ @@ -196,9 +196,36 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. */ +/* ************************************** +* Block functions +****************************************/ + +/*!Block functions produce and decode raw zstd blocks, without frame metadata. + It saves associated header sizes. + But user will have to save and regenerate fields required to regenerate data, such as block sizes. + + A few rules to respect : + - Uncompressed block size must be <= 128 KB + - Compressing or decompressing require a context structure + + Use ZSTD_createXCtx() to create them + - It is necessary to init context before starting + + compression : ZSTD_compressBegin(), which allows selection of compression level or parameters + + decompression : ZSTD_resetDCtx() + + If you compress multiple blocks without resetting, next blocks will create references to previous ones + - Dictionary can optionally be inserted, using ZSTD_de/compress_insertDictionary() + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. + + User must test for such outcome and be able to deal with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input +*/ + +size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); + + /* ************************************* * Pre-defined compression levels ***************************************/ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 2d1d45151..28f50c54a 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -227,6 +227,7 @@ static int basicUnitTests(U32 seed, double compressibility) compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; + if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */ DISPLAYLEVEL(4, "OK \n"); @@ -249,6 +250,7 @@ static int basicUnitTests(U32 seed, double compressibility) compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; + if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeDCtx(dctx); DISPLAYLEVEL(4, "OK \n"); } @@ -266,6 +268,32 @@ static int basicUnitTests(U32 seed, double compressibility) if (!ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); + /* block API tests */ + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + const size_t blockSize = 100 KB; + + /* basic block compression */ + DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); + result = ZSTD_compressBegin(cctx, 5); + if (ZSTD_isError(result)) goto _output_error; + cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); + if (ZSTD_isError(cSize)) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); + result = ZSTD_resetDCtx(dctx); + if (ZSTD_isError(result)) goto _output_error; + result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); + if (ZSTD_isError(result)) goto _output_error; + if (result != blockSize) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + /* long rle test */ { size_t sampleSize = 0; From b012510d9fe652b955c6b0589daffe5795f5b4de Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 9 Jan 2016 02:00:10 +0100 Subject: [PATCH 26/27] dictionary with block API --- lib/zstd_decompress.c | 43 +++++++++++++++++++++++++++++-------------- programs/fuzzer.c | 20 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 5dec85888..14918cb42 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -658,7 +658,19 @@ static size_t ZSTD_decompressSequences( } -size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, +static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) /* not contiguous */ + { + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dst; + dctx->previousDstEnd = dst; + } +} + + +static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { @@ -675,6 +687,15 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize) +{ + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize); +} + + size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, @@ -735,7 +756,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock(ctx, op, oend-op, ip, cBlockSize); + decodedSize = ZSTD_decompressBlock_internal(ctx, op, oend-op, ip, cBlockSize); break; case bt_raw : decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); @@ -786,13 +807,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con { /* Sanity check */ if (srcSize != ctx->expected) return ERROR(srcSize_wrong); - if (dst != ctx->previousDstEnd) /* not contiguous */ - { - ctx->dictEnd = ctx->previousDstEnd; - ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base)); - ctx->base = dst; - ctx->previousDstEnd = dst; - } + ZSTD_checkContinuity(ctx, dst); /* Decompress : frame header; part 1 */ switch (ctx->stage) @@ -849,7 +864,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con switch(ctx->bType) { case bt_compressed: - rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize); + rSize = ZSTD_decompressBlock_internal(ctx, dst, maxDstSize, src, srcSize); break; case bt_raw : rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize); @@ -874,10 +889,10 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con } -void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize) +void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, size_t dictSize) { ctx->dictEnd = ctx->previousDstEnd; - ctx->vBase = (const char*)src - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base)); - ctx->base = src; - ctx->previousDstEnd = (const char*)src + srcSize; + ctx->vBase = (const char*)dict - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base)); + ctx->base = dict; + ctx->previousDstEnd = (const char*)dict + dictSize; } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 28f50c54a..9c427c206 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -273,6 +273,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); const size_t blockSize = 100 KB; + const size_t dictSize = 16 KB; /* basic block compression */ DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); @@ -290,6 +291,25 @@ static int basicUnitTests(U32 seed, double compressibility) if (result != blockSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); + /* dictionary block compression */ + DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++); + result = ZSTD_compressBegin(cctx, 5); + if (ZSTD_isError(result)) goto _output_error; + result = ZSTD_compress_insertDictionary(cctx, CNBuffer, dictSize); + if (ZSTD_isError(result)) goto _output_error; + cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); + if (ZSTD_isError(cSize)) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); + result = ZSTD_resetDCtx(dctx); + if (ZSTD_isError(result)) goto _output_error; + ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize); + result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); + if (ZSTD_isError(result)) goto _output_error; + if (result != blockSize) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + ZSTD_freeCCtx(cctx); ZSTD_freeDCtx(dctx); } From 523b59423376b18e4157f7aeda200b32cf3bb158 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 9 Jan 2016 02:10:40 +0100 Subject: [PATCH 27/27] fixed read error --- lib/zstd_compress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index e66fedbe5..399f81169 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1797,6 +1797,7 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit); + if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */ return blockCompressor(zc, dst, maxDstSize, src, srcSize); } @@ -1832,6 +1833,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr, if (cSize == 0) { cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */ + if (ZSTD_isError(cSize)) return cSize; } else { @@ -1928,7 +1930,6 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong); - if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */ return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0); }