From 5203b8c68d1d92d7e46a3998ab6139a81825a93b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 9 Mar 2015 13:17:36 +0100 Subject: [PATCH 01/22] Updated datagen : sparse file generation --- programs/datagen.c | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index f2fc9404e..2a10b8113 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -127,12 +127,28 @@ static char RDG_genChar(U32* seed, const void* ltctx) #define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr) { - BYTE* buffPtr = ((BYTE*)buffer) - prefixSize; + BYTE* buffPtr = (BYTE*)buffer; const U32 matchProba32 = (U32)(32768 * matchProba); size_t pos = prefixSize; void* ldctx = litTable; U32* seed = seedPtr; + /* special case */ + while (matchProba >= 1.0) + { + size_t size0 = RDG_rand(seed) & 3; + size0 = 1U << (16 + size0 * 2); + size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ + if (buffSize < pos + size0) + { + memset(buffPtr+pos, 0, buffSize-pos); + return; + } + memset(buffPtr+pos, 0, size0); + pos += size0; + buffPtr[pos-1] = RDG_genChar(seed, ldctx); + } + /* init */ if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1; @@ -148,18 +164,18 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match int length = RDG_RANDLENGTH + 4; U32 offset = RDG_RAND15BITS + 1; if (offset > pos) offset = pos; - if (pos + length > buffSize) length = buffSize - pos; match = pos - offset; d = pos + length; + if (d > buffSize) d = buffSize; while (pos < d) buffPtr[pos++] = buffPtr[match++]; } else { /* Literal (noise) */ - U32 d; - int length = RDG_RANDLENGTH; - if (pos + length > buffSize) length = buffSize - pos; + size_t d; + size_t length = RDG_RANDLENGTH; d = pos + length; + if (d > buffSize) d = buffSize; while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx); } } @@ -169,7 +185,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) { void* ldctx; - if (litProba==0.0) litProba = matchProba / 3.8; + if (litProba==0.0) litProba = matchProba / 4.5; ldctx = RDG_createLiteralDistrib(litProba); RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed); free(ldctx); @@ -179,30 +195,28 @@ void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba #define RDG_BLOCKSIZE (128 KB) void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed) { - BYTE fullbuff[RDG_DICTSIZE + RDG_BLOCKSIZE + 1]; - BYTE* buff = fullbuff + RDG_DICTSIZE; + BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; U64 total = 0; - U32 genBlockSize = RDG_BLOCKSIZE; + size_t genBlockSize = RDG_BLOCKSIZE; void* ldctx; /* init */ - if (litProba==0.0) litProba = matchProba / 3.8; + if (litProba==0.0) litProba = matchProba / 4.5; ldctx = RDG_createLiteralDistrib(litProba); SET_BINARY_MODE(stdout); /* Generate dict */ - RDG_genBlock(fullbuff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed); + RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed); /* Generate compressible data */ while (total < size) { - RDG_genBlock(buff, RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed); - if (size-total < RDG_BLOCKSIZE) genBlockSize = (U32)(size-total); + RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed); + if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); total += genBlockSize; - buff[genBlockSize] = 0; fwrite(buff, 1, genBlockSize, stdout); /* update dict */ - memcpy(fullbuff, buff + (RDG_BLOCKSIZE - RDG_DICTSIZE), RDG_DICTSIZE); + memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); } free(ldctx); From 1d62914ea7d3f940657b083b06c38657923562f0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 9 Mar 2015 13:48:34 +0100 Subject: [PATCH 02/22] Added : valgrind test for datagen --- programs/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/Makefile b/programs/Makefile index bf44cc256..db3c710ae 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -142,6 +142,7 @@ test-fuzzer32: fuzzer32 ./fuzzer32 test-mem: zstd datagen fuzzer fullbench + valgrind --leak-check=yes ./datagen -g50M > /dev/null ./datagen -g16KB > tmp valgrind --leak-check=yes ./zstd -vf tmp /dev/null ./datagen -g128MB > tmp From bbfa7d77c9a802b983b456e1695f7cc82ba2fa37 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 23 Mar 2015 23:15:22 +0100 Subject: [PATCH 03/22] Fixed issue #28 --- lib/fse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fse.c b/lib/fse.c index e07b951cd..2861fd9f1 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -542,7 +542,7 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS U32 tableLog = maxTableLog; if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; if ((FSE_highbit32((U32)(srcSize - 1)) - 2) < tableLog) tableLog = FSE_highbit32((U32)(srcSize - 1)) - 2; /* Accuracy can be reduced */ - if ((FSE_highbit32(maxSymbolValue+1)+1) > tableLog) tableLog = FSE_highbit32(maxSymbolValue+1)+1; /* Need a minimum to safely represent all symbol values */ + if ((FSE_highbit32(maxSymbolValue)+2) > tableLog) tableLog = FSE_highbit32(maxSymbolValue)+2; /* Need a minimum to safely represent all symbol values */ if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; return tableLog; From 273d049a7c4e42074da9447e26944137f2aefc20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20F=C3=B6rberg?= Date: Tue, 24 Mar 2015 20:15:56 +0100 Subject: [PATCH 04/22] Add `unzstd` program for decompression. --- programs/Makefile | 4 ++++ programs/unzstd.1 | 31 +++++++++++++++++++++++++++++++ programs/zstdcli.c | 16 ++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 programs/unzstd.1 diff --git a/programs/Makefile b/programs/Makefile index db3c710ae..84a77894f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -99,16 +99,20 @@ install: zstd @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @install -m 755 zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat + @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd @echo Installing man pages @install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1 @install -m 644 zstdcat.1 $(DESTDIR)$(MANDIR)/zstdcat.1 + @install -m 644 unzstd.1 $(DESTDIR)$(MANDIR)/unzstd.1 @echo zstd installation completed uninstall: rm -f $(DESTDIR)$(BINDIR)/zstdcat + rm -f $(DESTDIR)$(BINDIR)/unzstd [ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT) [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 [ -f $(DESTDIR)$(MANDIR)/zstdcat.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstdcat.1 + [ -f $(DESTDIR)$(MANDIR)/unzstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/unzstd.1 @echo zstd programs successfully uninstalled test: test-zstd test-fullbench test-fuzzer test-mem diff --git a/programs/unzstd.1 b/programs/unzstd.1 new file mode 100644 index 000000000..157fab765 --- /dev/null +++ b/programs/unzstd.1 @@ -0,0 +1,31 @@ +\" +\" unzstd.1: This is a manual page for 'unzstd' program. This file is part of +\" the zstd project. +\" + +\" No hyphenation +.hy 0 +.nr HY 0 + +.TH unzstd "1" "2014-06-20" "unzstd" "User Commands" +.SH NAME +\fBunzstd\fR - Utility based on zstd + +.SH SYNOPSIS +.TP 5 +\fBunzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] + +.SH DESCRIPTION +.PP +\fBunzstd\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm. + +\fBunzstd\fR decompress input file, it is equivalent to \fBzstd -d\fR, + +Available options are the same as \fBzstd\fR ones (man zstd). + + +.SH BUGS +Report bugs at:- https://github.com/Cyan4973/zstd/ + +.SH AUTHOR +Yann Collet diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 5876f557c..37aaf8062 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -76,6 +76,7 @@ #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__ #define ZSTD_EXTENSION ".zst" #define ZSTD_CAT "zstdcat" +#define ZSTD_UNZSTD "unzstd" #define KB *(1 <<10) #define MB *(1 <<20) @@ -170,9 +171,24 @@ int main(int argc, char** argv) char* dynNameSpace = NULL; char extension[] = ZSTD_EXTENSION; + /* Pick out basename component. Don't rely on stdlib because of conflicting behaviour. */ + for (i = strlen(programName); i > 0; i--) + { + if (programName[i] == '/') + { + i++; + break; + } + } + programName += i; + /* zstdcat behavior */ if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } + /* unzstd behavior */ + if (!strcmp(programName, ZSTD_UNZSTD)) + decode=1; + // command switches for(i=1; i Date: Thu, 18 Jun 2015 07:43:16 -0800 Subject: [PATCH 05/22] Updated FSE Added clang and g++ tests --- .travis.yml | 2 ++ Makefile | 11 +++++++- lib/fse.c | 61 +++++++++++++++++++++++++------------------- lib/zstd.h | 6 ++--- programs/bench.c | 4 +-- programs/bench.h | 10 +------- programs/datagen.c | 15 ++++++----- programs/fileio.c | 14 +++++++--- programs/fullbench.c | 6 ++--- programs/fuzzer.c | 29 +++++++++++++++++---- programs/zstdcli.c | 2 +- 11 files changed, 99 insertions(+), 61 deletions(-) diff --git a/.travis.yml b/.travis.yml index f302fabe4..17846f160 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ before_install: env: - ZSTD_TRAVIS_CI_ENV=travis-install - ZSTD_TRAVIS_CI_ENV=test-all + - ZSTD_TRAVIS_CI_ENV=clangtest + - ZSTD_TRAVIS_CI_ENV=gpptest matrix: fast_finish: true diff --git a/Makefile b/Makefile index ecb5f17bc..adc6d5e8a 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION=0.0.1 +export VERSION=0.0.2 export RELEASE=r$(VERSION) DESTDIR?= @@ -51,6 +51,8 @@ TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV) endif +.PHONY: clean + default: zstdprograms all: @@ -90,4 +92,11 @@ test-travis: $(TRAVIS_TARGET) prg-travis: @cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV) +clangtest: clean + $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" + +gpptest: clean + $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + + endif diff --git a/lib/fse.c b/lib/fse.c index 2861fd9f1..a81e2f381 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -305,6 +305,8 @@ size_t FSE_headerBound(unsigned maxSymbolValue, unsigned tableLog) return maxSymbolValue ? maxHeaderSize : FSE_MAX_HEADERSIZE; } +#ifndef __clang_analyzer__ /* clang static analyzer has difficulties with this function : seems to believe normalizedCounter is uninitialized */ + static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, unsigned safeWrite) @@ -341,9 +343,9 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, while (charnum >= start+24) { start+=24; - bitStream += 0xFFFF< oend-2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */ - out[0] = (BYTE)bitStream; + out[0] = (BYTE) bitStream; out[1] = (BYTE)(bitStream>>8); out+=2; bitStream>>=16; @@ -370,7 +372,7 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, short count = normalizedCounter[charnum++]; const short max = (short)((2*threshold-1)-remaining); remaining -= FSE_abs(count); - if (remaining<0) return (size_t)-FSE_ERROR_GENERIC; + if (remaining<1) return (size_t)-FSE_ERROR_GENERIC; count++; /* +1 for extra accuracy */ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ bitStream += count << bitCount; @@ -400,6 +402,7 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, return (out-ostart); } +#endif // __clang_analyzer__ size_t FSE_writeHeader (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) @@ -690,7 +693,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, U32 weight = sEnd - sStart; if (weight < 1) return (size_t)-FSE_ERROR_GENERIC; - norm[s] = weight; + norm[s] = (short)weight; tmpTotal = end; } } @@ -852,7 +855,7 @@ void FSE_initCStream(FSE_CStream_t* bitC, void* start) void FSE_initCState(FSE_CState_t* statePtr, const void* CTable) { - const U32 tableLog = ( (U16*) CTable) [0]; + const U32 tableLog = ( (const U16*) CTable) [0]; statePtr->value = (ptrdiff_t)1<stateTable = ((const U16*) CTable) + 2; statePtr->symbolTT = (const U32*)CTable + 1 + (tableLog ? (1<<(tableLog-1)) : 1); @@ -1038,7 +1041,7 @@ size_t FSE_decompressRLE(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize) { if (cSrcSize != 1) return (size_t)-FSE_ERROR_srcSize_wrong; - memset(dst, *(BYTE*)cSrc, originalSize); + memset(dst, *(const BYTE*)cSrc, originalSize); return originalSize; } @@ -1100,30 +1103,30 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz if (srcSize >= sizeof(bitD_t)) { U32 contain32; - bitD->start = (char*)srcBuffer; - bitD->ptr = (char*)srcBuffer + srcSize - sizeof(bitD_t); + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD_t); bitD->bitContainer = FSE_readLEST(bitD->ptr); - contain32 = ((BYTE*)srcBuffer)[srcSize-1]; + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ bitD->bitsConsumed = 8 - FSE_highbit32(contain32); } else { U32 contain32; - bitD->start = (char*)srcBuffer; + bitD->start = (const char*)srcBuffer; bitD->ptr = bitD->start; - bitD->bitContainer = *(BYTE*)(bitD->start); + bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16); - case 6: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24); - case 5: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32); - case 4: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[3]) << 24; - case 3: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[2]) << 16; - case 2: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[1]) << 8; + case 7: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16); + case 6: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24); + case 5: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32); + case 4: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[3]) << 24; + case 3: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[2]) << 16; + case 2: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[1]) << 8; default:; } - contain32 = ((BYTE*)srcBuffer)[srcSize-1]; + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ bitD->bitsConsumed = 8 - FSE_highbit32(contain32); bitD->bitsConsumed += (U32)(sizeof(bitD_t) - srcSize)*8; @@ -1135,18 +1138,19 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz /* FSE_readBits * Read next n bits from the bitContainer. - * Use the fast variant *only* if n > 0. - * Note : for this function to work properly on 32-bits, don't read more than maxNbBits==25 + * On 32-bits, don't read more than maxNbBits==25 + * On 64-bits, don't read more than maxNbBits==57 + * Use the fast variant *only* if n >= 1. * return : value extracted. */ bitD_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits) { - bitD_t value = ((bitD->bitContainer << bitD->bitsConsumed) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits); + bitD_t value = ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD_t)*8)-1))) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits); bitD->bitsConsumed += nbBits; return value; } -bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 */ +bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ { bitD_t value = (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD_t)*8)-nbBits); bitD->bitsConsumed += nbBits; @@ -1215,7 +1219,7 @@ BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD) unsigned FSE_endOfDStream(const FSE_DStream_t* bitD) { - return FSE_reloadDStream((FSE_DStream_t*)bitD)==2; + return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD_t)*8)); } unsigned FSE_endOfDState(const FSE_DState_t* statePtr) @@ -1264,14 +1268,15 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( } /* tail */ + /* note : FSE_reloadDStream(&bitD) >= 1; Ends at exactly 2 */ while (1) { - if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state1) && FSE_endOfDStream(&bitD)) ) + if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) ) break; *op++ = fast ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD); - if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state2) && FSE_endOfDStream(&bitD)) ) + if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) ) break; *op++ = fast ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD); @@ -1442,6 +1447,8 @@ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) (unsigned* count, co } +#ifndef __clang_analyzer__ /* clang static analyzer doesn't understand that tableSymbol is necessarily entirely initialized */ + static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) @@ -1533,6 +1540,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) return 0; } +#endif // __clang_analyzer__ + #define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION) @@ -1559,7 +1568,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; U32 position = 0; U32 highThreshold = tableSize-1; - const S16 largeLimit= 1 << (tableLog-1); + const S16 largeLimit= (S16)(1 << (tableLog-1)); U32 noLarge = 1; U32 s; diff --git a/lib/zstd.h b/lib/zstd.h index cda16b88a..50ee72c43 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,7 +48,7 @@ extern "C" { #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */ #define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ -#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) unsigned ZSTD_versionNumber (void); @@ -70,7 +70,7 @@ ZSTD_compress() : or an error code if it fails (which can be tested using ZSTD_isError()) ZSTD_decompress() : - compressedSize : is obviously the source size + compressedSize : is the exact source size maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. It must be equal or larger than originalSize, otherwise decompression will fail. return : the number of bytes decompressed into destination buffer (originalSize) @@ -81,7 +81,7 @@ ZSTD_decompress() : /************************************** * Tool functions **************************************/ -size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size */ +size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size (worst case scenario) */ /* Error Management */ unsigned ZSTD_isError(size_t code); /* tells if a return value is an error code */ diff --git a/programs/bench.c b/programs/bench.c index 85f4993db..31a43f320 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -72,7 +72,7 @@ /************************************** -* Basic Types +* Basic Types **************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include @@ -471,7 +471,7 @@ static int BMK_syntheticTest(int cLevel, double compressibility) } -int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel) +int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel) { double compressibility = (double)g_compressibilityDefault / 100; diff --git a/programs/bench.h b/programs/bench.h index e07f37ff9..664c034e2 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -24,19 +24,11 @@ */ #pragma once -#if defined (__cplusplus) -extern "C" { -#endif - /* Main function */ -int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel); +int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel); /* Set Parameters */ void BMK_SetNbIterations(int nbLoops); - -#if defined (__cplusplus) -} -#endif diff --git a/programs/datagen.c b/programs/datagen.c index 2a10b8113..20eb30e99 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -75,7 +75,7 @@ * Local Functions *********************************************************/ #define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static unsigned int RDG_rand(U32* src) +static U32 RDG_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; @@ -88,20 +88,21 @@ static unsigned int RDG_rand(U32* src) #define LTSIZE 8192 #define LTMASK (LTSIZE-1) -static void* RDG_createLiteralDistrib(double ld) +static void* RDG_createLiteralDistrib(const double ld) { - BYTE* lt = malloc(LTSIZE); + BYTE* lt = (BYTE*)malloc(LTSIZE); U32 i = 0; BYTE character = '0'; BYTE firstChar = '('; BYTE lastChar = '}'; - if (ld==0.0) + if (ld<=0.02) { - character = 0; firstChar = 0; - lastChar =255; + lastChar = 254; + character = firstChar; } + if (ld==0.0) lastChar = 255; while (i inEnd) inSlot = inBuff; inSize = fread(inSlot, (size_t)1, blockSize, finput); + DISPLAY("Read block of size %u at pos %u \n", (U32)inSize, (U32)(inSlot-inBuff)); if (inSize==0) break; filesize += inSize; DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); @@ -340,9 +341,9 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* Allocate Memory */ inBuffSize = blockSize + FIO_blockHeaderSize; - inBuff = malloc(inBuffSize); + inBuff = (BYTE*)malloc(inBuffSize); outBuffSize = wNbBlocks * blockSize; - outBuff = malloc(outBuffSize); + outBuff = (BYTE*)malloc(outBuffSize); op = outBuff; oend = outBuff + outBuffSize; if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); @@ -352,6 +353,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha while (toRead) { size_t readSize, decodedSize; + static U32 nbReads = 0; /* Fill input buffer */ readSize = fread(inBuff, 1, toRead, finput); @@ -359,11 +361,15 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha EXM_THROW(34, "Read error"); /* Decode block */ + if (nbReads==55) + DISPLAY("!"); decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); + DISPLAY("nbReads : %u \n", nbReads++); if (decodedSize) /* not a header */ { /* Write block */ + DISPLAY("writing %u bytes from pos %u \n", (U32)decodedSize, (U32)(op-outBuff)); sizeCheck = fwrite(op, 1, decodedSize, foutput); if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file"); filesize += decodedSize; diff --git a/programs/fullbench.c b/programs/fullbench.c index b94918950..7afd12c88 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -267,7 +267,7 @@ size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* { U32 i; size_t total = 0; - BYTE* data = buff2; + BYTE* data = (BYTE*)buff2; (void)dst; (void)dstSize; (void)src; for (i=0; i < srcSize; i++) @@ -332,8 +332,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) /* Allocation */ dstBuffSize = ZSTD_compressBound(srcSize); - dstBuff = malloc(dstBuffSize); - buff2 = malloc(dstBuffSize); + dstBuff = (BYTE*)malloc(dstBuffSize); + buff2 = (BYTE*)malloc(dstBuffSize); if ((!dstBuff) || (!buff2)) { DISPLAY("\nError: not enough memory!\n"); diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 91218b9e7..b2dbcd3d9 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -256,6 +256,25 @@ static int basicUnitTests(U32 seed, double compressibility) if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error; DISPLAYLEVEL(4, "OK \n"); + /* long rle test */ + { + size_t sampleSize = 0; + DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); + FUZ_generateSynthetic(CNBuffer, sampleSize, compressibility, &randState); + memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); + sampleSize += 256 KB - 1; + FUZ_generateSynthetic((char*)CNBuffer+sampleSize, 96 KB, compressibility, &randState); + sampleSize += 96 KB; + cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize); + if (ZSTD_isError(cSize)) goto _output_error; + result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize); + if (ZSTD_isError(result)) goto _output_error; + if (result!=sampleSize) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + } + + + _end: free(CNBuffer); free(compressedBuffer); @@ -271,8 +290,8 @@ _output_error: static size_t findDiff(const void* buf1, const void* buf2, size_t max) { - const BYTE* b1 = buf1; - const BYTE* b2 = buf2; + const BYTE* b1 = (const BYTE*)buf1; + const BYTE* b2 = (const BYTE*)buf2; size_t i; for (i=0; i try to select one automatically (when possible) */ while (!outFileName) From fc774d31d12ad3f739afc510ca54ba2ccb81178a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 20 Jun 2015 15:33:08 -0800 Subject: [PATCH 06/22] Added clang & g++-multilib install for Travis CI --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 17846f160..b0265e4aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ compiler: gcc script: make test-travis before_install: - sudo apt-get update -qq + - sudo apt-get install -qq clang + - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq valgrind From 078a9a2804e87d32cdff1c0984a9ba92575793b7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 20 Jun 2015 19:37:53 -0800 Subject: [PATCH 07/22] Added Wcast-qual compilation flag Updated xxHash Removed log traces --- lib/zstd.c | 32 +++++++++++++++----------------- programs/Makefile | 15 ++++++++------- programs/fileio.c | 6 ------ 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index c8eed5a3e..cd5822053 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -178,11 +178,11 @@ static unsigned ZSTD_isLittleEndian(void) return one.c[0]; } -static U16 ZSTD_read16(const void* p) { return *(U16*)p; } +static U16 ZSTD_read16(const void* p) { return *(const U16*)p; } -static U32 ZSTD_read32(const void* p) { return *(U32*)p; } +static U32 ZSTD_read32(const void* p) { return *(const U32*)p; } -static size_t ZSTD_read_ARCH(const void* p) { return *(size_t*)p; } +static size_t ZSTD_read_ARCH(const void* p) { return *(const size_t*)p; } static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } @@ -503,7 +503,7 @@ static size_t ZSTD_compressRle (void* dst, size_t maxDstSize, const void* src, s /* at this stage : dstSize >= FSE_compressBound(srcSize) > (ZSTD_blockHeaderSize+1) (checked by ZSTD_compressLiterals()) */ (void)maxDstSize; - ostart[ZSTD_blockHeaderSize] = *(BYTE*)src; + ostart[ZSTD_blockHeaderSize] = *(const BYTE*)src; /* Build header */ ostart[0] = (BYTE)(srcSize>>16); @@ -621,9 +621,7 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize, errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue); if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC; if (errorCode == srcSize) return 1; - //if (errorCode < ((srcSize * 7) >> 10)) return 0; - //if (errorCode < (srcSize >> 7)) return 0; - if (errorCode < (srcSize >> 6)) return 0; /* heuristic : probably not compressible enough */ + if (errorCode < (srcSize >> 6)) return 0; /* cheap heuristic : probably not compressible enough */ tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); errorCode = (int)FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue); @@ -915,7 +913,7 @@ static const U64 prime7bytes = 58295818150454627ULL; //static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); } -static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; } +static U32 ZSTD_hashPtr(const void* p) { return ( (*(const U64*)p * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); } @@ -961,7 +959,7 @@ static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const /* Main Search Loop */ while (ip < ilimit) { - const BYTE* match = (BYTE*) ZSTD_updateMatch(HashTable, ip, base); + const BYTE* match = (const BYTE*) ZSTD_updateMatch(HashTable, ip, base); if (!ZSTD_checkMatch(match,ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; } @@ -1367,9 +1365,9 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx, } case bt_compressed: { - size_t cSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize); - if (ZSTD_isError(cSize)) return cSize; - *litPtr = oend - cSize; + size_t litSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize); + if (ZSTD_isError(litSize)) return litSize; + *litPtr = oend - litSize; ip += litcSize; break; } @@ -1508,7 +1506,7 @@ FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize ip, iend-ip); if (ZSTD_isError(errorCode)) return errorCode; /* end pos */ - if ((litPtr>=ostart) && (litPtr<=oend)) + if ((litPtr>=ostart) && (litPtr<=oend)) /* decoded literals are into dst buffer */ litEnd = oend - lastLLSize; else litEnd = ip - lastLLSize; @@ -1519,7 +1517,6 @@ FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize FSE_DStream_t DStream; FSE_DState_t stateLL, stateOffb, stateML; size_t prevOffset = 0, offset = 0; - size_t qutt=0; FSE_initDStream(&DStream, ip, iend-ip); FSE_initDState(&stateLL, &DStream, DTableLL); @@ -1545,7 +1542,6 @@ _another_round: if (add < 255) litLength += add; else { - //litLength = (*(U32*)dumps) & 0xFFFFFF; litLength = ZSTD_readLE32(dumps) & 0xFFFFFF; dumps += 3; } @@ -1578,7 +1574,7 @@ _another_round: if (add < 255) matchLength += add; else { - matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; + matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */ dumps += 3; } } @@ -1587,8 +1583,10 @@ _another_round: /* copy Match */ { BYTE* const endMatch = op + matchLength; + size_t qutt=0; U64 saved[2]; + /* save beginning of literal sequence, in case of write overlap */ if ((size_t)(litPtr - endMatch) < 12) { qutt = endMatch + 12 - litPtr; @@ -1624,7 +1622,7 @@ _another_round: op = endMatch; if ((size_t)(litPtr - endMatch) < 12) - memcpy((void*)litPtr, saved, qutt); + memcpy(endMatch + (litPtr - endMatch), saved, qutt); /* required as litPtr is const ptr */ } } diff --git a/programs/Makefile b/programs/Makefile index db3c710ae..1978f66fd 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,12 +30,12 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE?= r1 +RELEASE?= v0.0.2 DESTDIR?= PREFIX ?= /usr CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\" +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\" FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) BINDIR=$(PREFIX)/bin @@ -142,13 +142,14 @@ test-fuzzer32: fuzzer32 ./fuzzer32 test-mem: zstd datagen fuzzer fullbench - valgrind --leak-check=yes ./datagen -g50M > /dev/null + @echo "\n ---- valgrind tests : memory analyzer ----" + valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null ./datagen -g16KB > tmp - valgrind --leak-check=yes ./zstd -vf tmp /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null ./datagen -g128MB > tmp - valgrind --leak-check=yes ./zstd -vf tmp /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null @rm tmp - valgrind --leak-check=yes ./fuzzer -i128 -t1 - valgrind --leak-check=yes ./fullbench -i1 + valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1 + valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 endif diff --git a/programs/fileio.c b/programs/fileio.c index 9b62873f2..ac36fa3e2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -265,7 +265,6 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* Fill input Buffer */ if (inSlot + blockSize > inEnd) inSlot = inBuff; inSize = fread(inSlot, (size_t)1, blockSize, finput); - DISPLAY("Read block of size %u at pos %u \n", (U32)inSize, (U32)(inSlot-inBuff)); if (inSize==0) break; filesize += inSize; DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); @@ -353,7 +352,6 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha while (toRead) { size_t readSize, decodedSize; - static U32 nbReads = 0; /* Fill input buffer */ readSize = fread(inBuff, 1, toRead, finput); @@ -361,15 +359,11 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha EXM_THROW(34, "Read error"); /* Decode block */ - if (nbReads==55) - DISPLAY("!"); decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); - DISPLAY("nbReads : %u \n", nbReads++); if (decodedSize) /* not a header */ { /* Write block */ - DISPLAY("writing %u bytes from pos %u \n", (U32)decodedSize, (U32)(op-outBuff)); sizeCheck = fwrite(op, 1, decodedSize, foutput); if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file"); filesize += decodedSize; From 1efa31fad80e7640eb0f20c226645554b7f594e0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jul 2015 15:56:41 -0800 Subject: [PATCH 08/22] Updated FSE lib --- lib/fse.c | 278 +++++++++++++++++----------------- lib/fse.h | 346 ++++++++++++++++--------------------------- lib/fse_static.h | 199 ++++++++++++++++++++++--- lib/zstd.c | 64 ++++---- programs/fullbench.c | 2 +- 5 files changed, 472 insertions(+), 417 deletions(-) diff --git a/lib/fse.c b/lib/fse.c index a81e2f381..b0318f15f 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -241,14 +241,11 @@ typedef struct int deltaFindState; U16 maxState; BYTE minBitsOut; - /* one byte padding */ + /* one byte padding ; total 8 bytes */ } FSE_symbolCompressionTransform; -typedef struct -{ - U32 fakeTable[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; /* compatible with FSE_compressU16() */ -} CTable_max_t; - +typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; +typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; /**************************************************************** * Internal functions @@ -299,7 +296,7 @@ static short FSE_abs(short a) /**************************************************************** * Header bitstream management ****************************************************************/ -size_t FSE_headerBound(unsigned maxSymbolValue, unsigned tableLog) +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) { size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 1; return maxSymbolValue ? maxHeaderSize : FSE_MAX_HEADERSIZE; @@ -307,7 +304,7 @@ size_t FSE_headerBound(unsigned maxSymbolValue, unsigned tableLog) #ifndef __clang_analyzer__ /* clang static analyzer has difficulties with this function : seems to believe normalizedCounter is uninitialized */ -static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, +static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, unsigned safeWrite) { @@ -405,22 +402,23 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, #endif // __clang_analyzer__ -size_t FSE_writeHeader (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +size_t FSE_writeNCount (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */ if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */ - if (headerBufferSize < FSE_headerBound(maxSymbolValue, tableLog)) - return FSE_writeHeader_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + if (headerBufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - return FSE_writeHeader_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); + return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); } -size_t FSE_readHeader (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) { const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; const BYTE* ip = istart; int nbBits; int remaining; @@ -430,6 +428,7 @@ size_t FSE_readHeader (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t unsigned charnum = 0; int previous0 = 0; + if (hbSize < 4) return (size_t)-FSE_ERROR_srcSize_wrong; bitStream = FSE_readLE32(ip); nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return (size_t)-FSE_ERROR_tableLog_tooLarge; @@ -459,7 +458,7 @@ size_t FSE_readHeader (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t } n0 += bitStream & 3; bitCount += 2; - if (n0 > *maxSVPtr) return (size_t)-FSE_ERROR_GENERIC; + if (n0 > *maxSVPtr) return (size_t)-FSE_ERROR_maxSymbolValue_tooSmall; while (charnum < n0) normalizedCounter[charnum++] = 0; ip += bitCount>>3; bitCount &= 7; @@ -491,16 +490,27 @@ size_t FSE_readHeader (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t threshold >>= 1; } - ip += bitCount>>3; - bitCount &= 7; - bitStream = FSE_readLE32(ip) >> bitCount; + { + const BYTE* itarget = ip + (bitCount>>3); + if (itarget > iend - 4) + { + ip = iend - 4; + bitCount -= (int)(8 * (iend - 4 - ip)); + } + else + { + ip = itarget; + bitCount &= 7; + } + bitStream = FSE_readLE32(ip) >> (bitCount & 31); + } } } if (remaining != 1) return (size_t)-FSE_ERROR_GENERIC; *maxSVPtr = charnum-1; - ip += bitCount>0; - if ((size_t)(ip-istart) >= hbSize) return (size_t)-FSE_ERROR_srcSize_wrong; /* arguably a bit late , tbd */ + ip += (bitCount+7)>>3; + if ((size_t)(ip-istart) > hbSize) return (size_t)-FSE_ERROR_srcSize_wrong; return ip-istart; } @@ -509,7 +519,7 @@ size_t FSE_readHeader (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t * FSE Compression Code ****************************************************************/ /* -CTable is a variable size structure which contains : +FSE_CTable[0] is a variable size structure which contains : U16 tableLog; U16 maxSymbolValue; U16 nextStateNumber[1 << tableLog]; // This size is variable @@ -526,17 +536,17 @@ size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) return size; } -void* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) { size_t size; if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return malloc(size); + return (FSE_CTable*)malloc(size); } -void FSE_freeCTable (void* CTable) +void FSE_freeCTable (FSE_CTable* ct) { - free(CTable); + free(ct); } @@ -674,7 +684,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, U32 maxV = 0, maxC =0; for (s=0; s<=maxSymbolValue; s++) if (count[s] > maxC) maxV=s, maxC=count[s]; - norm[maxV] += ToDistribute; + norm[maxV] += (short)ToDistribute; return 0; } @@ -782,19 +792,18 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, } -/* fake CTable, for raw (uncompressed) input */ -size_t FSE_buildCTable_raw (void* CTable, unsigned nbBits) +/* fake FSE_CTable, for raw (uncompressed) input */ +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) { const unsigned tableSize = 1 << nbBits; const unsigned tableMask = tableSize - 1; const unsigned maxSymbolValue = tableMask; - U16* tableU16 = ( (U16*) CTable) + 2; - FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)CTable)+1) + (tableSize>>1)); + U16* tableU16 = ( (U16*) ct) + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)ct)+1) + (tableSize>>1)); unsigned s; /* Sanity checks */ if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */ - if (((size_t)CTable) & 3) return (size_t)-FSE_ERROR_GENERIC; /* Must be allocated of 4 bytes boundaries */ /* header */ tableU16[-2] = (U16) nbBits; @@ -816,15 +825,12 @@ size_t FSE_buildCTable_raw (void* CTable, unsigned nbBits) } -/* fake CTable, for rle (100% always same symbol) input */ -size_t FSE_buildCTable_rle (void* CTable, BYTE symbolValue) +/* fake FSE_CTable, for rle (100% always same symbol) input */ +size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) { const unsigned tableSize = 1; - U16* tableU16 = ( (U16*) CTable) + 2; - FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)CTable + 2); - - /* safety checks */ - if (((size_t)CTable) & 3) return (size_t)-FSE_ERROR_GENERIC; /* Must be 4 bytes aligned */ + U16* tableU16 = ( (U16*) ct) + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)ct + 2); /* header */ tableU16[-2] = (U16) 0; @@ -853,12 +859,12 @@ void FSE_initCStream(FSE_CStream_t* bitC, void* start) bitC->ptr = bitC->startPtr; } -void FSE_initCState(FSE_CState_t* statePtr, const void* CTable) +void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) { - const U32 tableLog = ( (const U16*) CTable) [0]; + const U32 tableLog = ( (const U16*) ct) [0]; statePtr->value = (ptrdiff_t)1<stateTable = ((const U16*) CTable) + 2; - statePtr->symbolTT = (const U32*)CTable + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateTable = ((const U16*) ct) + 2; + statePtr->symbolTT = (const FSE_symbolCompressionTransform*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); statePtr->stateLog = tableLog; } @@ -869,14 +875,14 @@ void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits) bitC->bitPos += nbBits; } -void FSE_encodeByte(FSE_CStream_t* bitC, FSE_CState_t* statePtr, BYTE symbol) +void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* statePtr, BYTE symbol) { - const FSE_symbolCompressionTransform* const symbolTT = (const FSE_symbolCompressionTransform*) statePtr->symbolTT; - const U16* const stateTable = (const U16*) statePtr->stateTable; - int nbBitsOut = symbolTT[symbol].minBitsOut; - nbBitsOut -= (int)((symbolTT[symbol].maxState - statePtr->value) >> 31); + const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + int nbBitsOut = symbolTT.minBitsOut; + nbBitsOut -= (int)((symbolTT.maxState - statePtr->value) >> 31); FSE_addBits(bitC, statePtr->value, nbBitsOut); - statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT[symbol].deltaFindState]; + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; } void FSE_flushBits(FSE_CStream_t* bitC) @@ -911,7 +917,7 @@ size_t FSE_closeCStream(FSE_CStream_t* bitC) size_t FSE_compress_usingCTable (void* dst, size_t dstSize, const void* src, size_t srcSize, - const void* CTable) + const FSE_CTable* ct) { const BYTE* const istart = (const BYTE*) src; const BYTE* ip; @@ -924,7 +930,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize, /* init */ (void)dstSize; /* objective : ensure it fits into dstBuffer (Todo) */ FSE_initCStream(&bitC, dst); - FSE_initCState(&CState1, CTable); + FSE_initCState(&CState1, ct); CState2 = CState1; ip=iend; @@ -932,32 +938,32 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize, /* join to even */ if (srcSize & 1) { - FSE_encodeByte(&bitC, &CState1, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); FSE_flushBits(&bitC); } /* join to mod 4 */ if ((sizeof(size_t)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) /* test bit 2 */ { - FSE_encodeByte(&bitC, &CState2, *--ip); - FSE_encodeByte(&bitC, &CState1, *--ip); + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); FSE_flushBits(&bitC); } /* 2 or 4 encoding per loop */ while (ip>istart) { - FSE_encodeByte(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState2, *--ip); if (sizeof(size_t)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ FSE_flushBits(&bitC); - FSE_encodeByte(&bitC, &CState1, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); if (sizeof(size_t)*8 > FSE_MAX_TABLELOG*4+7 ) /* this test must be static */ { - FSE_encodeByte(&bitC, &CState2, *--ip); - FSE_encodeByte(&bitC, &CState1, *--ip); + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); } FSE_flushBits(&bitC); @@ -983,7 +989,7 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize U32 count[FSE_MAX_SYMBOL_VALUE+1]; S16 norm[FSE_MAX_SYMBOL_VALUE+1]; - CTable_max_t CTable; + CTable_max_t ct; size_t errorCode; /* early out */ @@ -993,7 +999,7 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; /* Scan input and build symbol stats */ - errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue); + errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize); if (FSE_isError(errorCode)) return errorCode; if (errorCode == srcSize) return 1; if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ @@ -1003,14 +1009,14 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize if (FSE_isError(errorCode)) return errorCode; /* Write table description header */ - errorCode = FSE_writeHeader (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog); + errorCode = FSE_writeNCount (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog); if (FSE_isError(errorCode)) return errorCode; op += errorCode; /* Compress */ - errorCode = FSE_buildCTable (&CTable, norm, maxSymbolValue, tableLog); + errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog); if (FSE_isError(errorCode)) return errorCode; - op += FSE_compress_usingCTable(op, oend - op, ip, srcSize, &CTable); + op += FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct); /* check compressibility */ if ( (size_t)(op-ostart) >= srcSize-1 ) @@ -1036,24 +1042,12 @@ typedef struct BYTE nbBits; } FSE_decode_t; /* size == U32 */ -/* Specific corner case : RLE compression */ -size_t FSE_decompressRLE(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize) -{ - if (cSrcSize != 1) return (size_t)-FSE_ERROR_srcSize_wrong; - memset(dst, *(const BYTE*)cSrc, originalSize); - return originalSize; -} - -size_t FSE_buildDTable_rle (void* DTable, BYTE symbolValue) +size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) { - U32* const base32 = (U32*)DTable; + U32* const base32 = (U32*)dt; FSE_decode_t* const cell = (FSE_decode_t*)(base32 + 1); - /* Sanity check */ - if (((size_t)DTable) & 3) return (size_t)-FSE_ERROR_GENERIC; /* Must be allocated of 4 bytes boundaries */ - base32[0] = 0; cell->newState = 0; @@ -1064,9 +1058,9 @@ size_t FSE_buildDTable_rle (void* DTable, BYTE symbolValue) } -size_t FSE_buildDTable_raw (void* DTable, unsigned nbBits) +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) { - U32* const base32 = (U32*)DTable; + U32* const base32 = (U32*)dt; FSE_decode_t* dinfo = (FSE_decode_t*)(base32 + 1); const unsigned tableSize = 1 << nbBits; const unsigned tableMask = tableSize - 1; @@ -1075,7 +1069,6 @@ size_t FSE_buildDTable_raw (void* DTable, unsigned nbBits) /* Sanity checks */ if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */ - if (((size_t)DTable) & 3) return (size_t)-FSE_ERROR_GENERIC; /* Must be allocated of 4 bytes boundaries */ /* Build Decoding Table */ base32[0] = nbBits; @@ -1100,11 +1093,11 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz { if (srcSize < 1) return (size_t)-FSE_ERROR_srcSize_wrong; - if (srcSize >= sizeof(bitD_t)) + if (srcSize >= sizeof(size_t)) { U32 contain32; bitD->start = (const char*)srcBuffer; - bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD_t); + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t); bitD->bitContainer = FSE_readLEST(bitD->ptr); contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ @@ -1118,18 +1111,18 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16); - case 6: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24); - case 5: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32); - case 4: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[3]) << 24; - case 3: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[2]) << 16; - case 2: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[1]) << 8; + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; default:; } contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ bitD->bitsConsumed = 8 - FSE_highbit32(contain32); - bitD->bitsConsumed += (U32)(sizeof(bitD_t) - srcSize)*8; + bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8; } return srcSize; @@ -1143,23 +1136,23 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz * Use the fast variant *only* if n >= 1. * return : value extracted. */ -bitD_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits) +size_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits) { - bitD_t value = ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD_t)*8)-1))) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits); + size_t value = ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(size_t)*8)-1))) >> 1) >> (((sizeof(size_t)*8)-1)-nbBits); bitD->bitsConsumed += nbBits; return value; } -bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ +size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ { - bitD_t value = (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD_t)*8)-nbBits); + size_t value = (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(size_t)*8)-nbBits); bitD->bitsConsumed += nbBits; return value; } unsigned FSE_reloadDStream(FSE_DStream_t* bitD) { - if (bitD->ptr >= bitD->start + sizeof(bitD_t)) + if (bitD->ptr >= bitD->start + sizeof(size_t)) { bitD->ptr -= bitD->bitsConsumed >> 3; bitD->bitsConsumed &= 7; @@ -1168,8 +1161,8 @@ unsigned FSE_reloadDStream(FSE_DStream_t* bitD) } if (bitD->ptr == bitD->start) { - if (bitD->bitsConsumed < sizeof(bitD_t)*8) return 1; - if (bitD->bitsConsumed == sizeof(bitD_t)*8) return 2; + if (bitD->bitsConsumed < sizeof(size_t)*8) return 1; + if (bitD->bitsConsumed == sizeof(size_t)*8) return 2; return 3; } { @@ -1184,9 +1177,9 @@ unsigned FSE_reloadDStream(FSE_DStream_t* bitD) } -void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const void* DTable) +void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt) { - const U32* const base32 = (const U32*)DTable; + const U32* const base32 = (const U32*)dt; DStatePtr->state = FSE_readBits(bitD, base32[0]); FSE_reloadDStream(bitD); DStatePtr->table = base32 + 1; @@ -1197,7 +1190,7 @@ BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD) const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; const U32 nbBits = DInfo.nbBits; BYTE symbol = DInfo.symbol; - bitD_t lowBits = FSE_readBits(bitD, nbBits); + size_t lowBits = FSE_readBits(bitD, nbBits); DStatePtr->state = DInfo.newState + lowBits; return symbol; @@ -1208,7 +1201,7 @@ BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD) const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; const U32 nbBits = DInfo.nbBits; BYTE symbol = DInfo.symbol; - bitD_t lowBits = FSE_readBitsFast(bitD, nbBits); + size_t lowBits = FSE_readBitsFast(bitD, nbBits); DStatePtr->state = DInfo.newState + lowBits; return symbol; @@ -1219,19 +1212,19 @@ BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD) unsigned FSE_endOfDStream(const FSE_DStream_t* bitD) { - return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD_t)*8)); + return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(size_t)*8)); } -unsigned FSE_endOfDState(const FSE_DState_t* statePtr) +unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) { - return statePtr->state == 0; + return DStatePtr->state == 0; } FORCE_INLINE size_t FSE_decompress_usingDTable_generic( void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, - const void* DTable, unsigned fast) + const FSE_DTable* dt, unsigned fast) { BYTE* const ostart = (BYTE*) dst; BYTE* op = ostart; @@ -1239,15 +1232,16 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( BYTE* const olimit = omax-3; FSE_DStream_t bitD; - FSE_DState_t state1, state2; + FSE_DState_t state1; + FSE_DState_t state2; size_t errorCode; /* Init */ errorCode = FSE_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ if (FSE_isError(errorCode)) return errorCode; - FSE_initDState(&state1, &bitD, DTable); - FSE_initDState(&state2, &bitD, DTable); + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); /* 2 symbols per loop */ @@ -1255,12 +1249,12 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( { *op++ = fast ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD); - if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD_t)*8) /* This test must be static */ + if (FSE_MAX_TABLELOG*2+7 > sizeof(size_t)*8) /* This test must be static */ FSE_reloadDStream(&bitD); *op++ = fast ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD); - if (FSE_MAX_TABLELOG*4+7 < sizeof(bitD_t)*8) /* This test must be static */ + if (FSE_MAX_TABLELOG*4+7 < sizeof(size_t)*8) /* This test must be static */ { *op++ = fast ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD); *op++ = fast ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD); @@ -1294,11 +1288,11 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, - const void* DTable, size_t fastMode) + const FSE_DTable* dt, size_t fastMode) { /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, DTable, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, DTable, 0); + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); } @@ -1307,25 +1301,25 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; short counting[FSE_MAX_SYMBOL_VALUE+1]; - FSE_decode_t DTable[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; size_t errorCode, fastMode; if (cSrcSize<2) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */ /* normal FSE decoding mode */ - errorCode = FSE_readHeader (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); if (FSE_isError(errorCode)) return errorCode; if (errorCode >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */ ip += errorCode; cSrcSize -= errorCode; - fastMode = FSE_buildDTable (DTable, counting, maxSymbolValue, tableLog); + fastMode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); if (FSE_isError(fastMode)) return fastMode; /* always return, even if it is an error code */ - return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, DTable, fastMode); + return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt, fastMode); } @@ -1334,8 +1328,8 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr /* 2nd part of the file designed to be included - for type-specific functions (template equivalent in C) - Objective is to write such functions only once, for better maintenance + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance */ /* safety checks */ @@ -1353,7 +1347,8 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr /* Function templates */ -size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (unsigned* count, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned* maxSymbolValuePtr, unsigned safe) +size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) +(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe) { const FSE_FUNCTION_TYPE* ip = source; const FSE_FUNCTION_TYPE* const iend = ip+sourceSize; @@ -1427,48 +1422,45 @@ size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (unsigned* c while (!count[maxSymbolValue]) maxSymbolValue--; *maxSymbolValuePtr = maxSymbolValue; - return (int)max; + return (size_t)max; } /* hidden fast variant (unsafe) */ -size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION) (unsigned* count, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned* maxSymbolValuePtr) +size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION) +(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize) { - return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, source, sourceSize, maxSymbolValuePtr, 0); + return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0); } -size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) (unsigned* count, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned* maxSymbolValuePtr) +size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) +(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize) { if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255)) { *maxSymbolValuePtr = 255; - return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, source, sourceSize, maxSymbolValuePtr, 0); + return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0); } - return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, source, sourceSize, maxSymbolValuePtr, 1); + return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 1); } -#ifndef __clang_analyzer__ /* clang static analyzer doesn't understand that tableSymbol is necessarily entirely initialized */ - static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) -(void* CTable, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { const unsigned tableSize = 1 << tableLog; const unsigned tableMask = tableSize - 1; - U16* tableU16 = ( (U16*) CTable) + 2; - FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)CTable) + 1 + (tableLog ? tableSize>>1 : 1) ); + U16* tableU16 = ( (U16*) ct) + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)ct) + 1 + (tableLog ? tableSize>>1 : 1) ); const unsigned step = FSE_tableStep(tableSize); unsigned cumul[FSE_MAX_SYMBOL_VALUE+2]; U32 position = 0; - FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; + FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE] = {0}; /* should not be necessary, but analyzer complain without it, and performance loss is negligible with it */ U32 highThreshold = tableSize-1; unsigned symbol; unsigned i; - /* safety checks */ - if (((size_t)CTable) & 3) return (size_t)-FSE_ERROR_GENERIC; /* Must be allocated of 4 bytes boundaries */ - /* header */ tableU16[-2] = (U16) tableLog; tableU16[-1] = (U16) maxSymbolValue; @@ -1508,10 +1500,10 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) for (i=0; i FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return malloc( ((size_t)1< // size_t, ptrdiff_t +#include /* size_t, ptrdiff_t */ /****************************************** @@ -50,7 +50,7 @@ extern "C" { ******************************************/ size_t FSE_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize); -size_t FSE_decompress(void* dst, size_t maxDstSize, +size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize); /* FSE_compress(): @@ -58,29 +58,19 @@ FSE_compress(): 'dst' buffer must be already allocated, and sized to handle worst case situations. Worst case size evaluation is provided by FSE_compressBound(). return : size of compressed data - Special values : if result == 0, data is uncompressible => Nothing is stored within cSrc !! - if result == 1, data is one constant element x srcSize times. Use RLE compression. - if FSE_isError(result), it's an error code. + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. FSE_decompress(): Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', into already allocated destination buffer 'dst', of size 'maxDstSize'. - ** Important ** : This function doesn't decompress uncompressed nor RLE data ! return : size of regenerated data (<= maxDstSize) or an error code, which can be tested using FSE_isError() -*/ - -size_t FSE_decompressRLE(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); -/* -FSE_decompressRLE(): - Decompress specific RLE corner case (equivalent to memset()). - cSrcSize must be == 1. originalSize must be exact. - return : size of regenerated data (==originalSize) - or an error code, which can be tested using FSE_isError() - -Note : there is no function provided for uncompressed data, as it's just a simple memcpy() + ** Important ** : FSE_decompress() doesn't decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + FSE library doesn't manage headers, which are intentionally left to the user layer. */ @@ -102,51 +92,103 @@ FSE_compress2(): Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' Both parameters can be defined as '0' to mean : use default value return : size of compressed data - or -1 if there is an error + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. */ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); /****************************************** - FSE detailed API +* FSE detailed API ******************************************/ /* -int FSE_compress(char* dest, const char* source, int inputSize) does the following: +FSE_compress() does the following: 1. count symbol occurrence from source[] into table count[] 2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) 3. save normalized counters to memory buffer using writeHeader() 4. build encoding table 'CTable' from normalized counters -5. encode the data stream using encoding table +5. encode the data stream using encoding table 'CTable' -int FSE_decompress(char* dest, int originalSize, const char* compressed) performs: +FSE_decompress() does the following: 1. read normalized counters with readHeader() 2. build decoding table 'DTable' from normalized counters -3. decode the data stream using decoding table +3. decode the data stream using decoding table 'DTable' -The following API allows triggering specific sub-functions. +The following API allows to trigger specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using one's own method. */ /* *** COMPRESSION *** */ -size_t FSE_count(unsigned* count, const unsigned char* src, size_t srcSize, unsigned* maxSymbolValuePtr); - -unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue); -size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t total, unsigned maxSymbolValue); - -size_t FSE_headerBound(unsigned maxSymbolValue, unsigned tableLog); -size_t FSE_writeHeader (void* headerBuffer, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -void* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); -void FSE_freeCTable (void* CTable); -size_t FSE_buildCTable(void* CTable, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -size_t FSE_compress_usingCTable (void* dst, size_t dstSize, const void* src, size_t srcSize, const void* CTable); +/* +FSE_count(): + Provides the precise count of each symbol within a table 'count' + 'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1). + maxSymbolValuePtr[0] will be updated if detected smaller than initially expected + return : the count of the most frequent symbol (which is not identified) + if return == srcSize, there is only one symbol. + if FSE_isError(return), it's an error code. */ +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize); /* +FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + return : recommended tableLog (necessarily <= initial 'tableLog') */ +unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue); + +/* +FSE_normalizeCount(): + normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); + +/* +FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog' + Typically useful for allocation purpose. */ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/* +FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + return : size of the compressed table + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + + +/* +Constructor and Destructor of type FSE_CTable +Not that its size depends on parameters 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void */ +FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); +void FSE_freeCTable (FSE_CTable* ct); + +/* +FSE_buildCTable(): + Builds 'ct', which must be already allocated, using FSE_createCTable() + return : 0 + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/* +FSE_compress_usingCTable(): + Compress 'src' using 'ct' into 'dst' which must be already allocated + return : size of compressed data + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, const void* src, size_t srcSize, const FSE_CTable* ct); + +/* +Tutorial : +---------- The first step is to count all symbols. FSE_count() provides one quick way to do this job. -Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have '*maxSymbolValuePtr+1' cells. -'source' is a table of char of size 'sourceSize'. All values within 'src' MUST be <= *maxSymbolValuePtr -*maxSymbolValuePtr will be updated, with its real value (necessarily <= original value) +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) FSE_count() will return the number of occurrence of the most frequent symbol. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). @@ -170,211 +212,83 @@ For guaranteed success, buffer size must be at least FSE_headerBound(). The result of the function is the number of bytes written into 'header'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()) (for example, buffer size too small). -'normalizedCounter' can then be used to create the compression tables 'CTable'. +'normalizedCounter' can then be used to create the compression table 'CTable'. The space required by 'CTable' must be already allocated. Its size is provided by FSE_sizeof_CTable(). 'CTable' must be aligned of 4 bytes boundaries. You can then use FSE_buildCTable() to fill 'CTable'. In both cases, if there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). -'CTable' can then be used to compress 'source', with FSE_compress_usingCTable(). -Similar to FSE_count(), the convention is that 'source' is assumed to be a table of char of size 'sourceSize' -The function returns the size of compressed data (without header), or -1 if failed. +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header). +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). */ /* *** DECOMPRESSION *** */ -size_t FSE_readHeader (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize); - -void* FSE_createDTable(unsigned tableLog); -void FSE_freeDTable(void* DTable); -size_t FSE_buildDTable (void* DTable, const short* const normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const void* DTable, size_t fastMode); +/* +FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + return : size read from 'rBuffer' + or an errorCode, which can be tested using FSE_isError() + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); /* -If the block is RLE compressed, or uncompressed, use the relevant specific functions. +Constructor and Destructor of type FSE_DTable +Note that its size depends on parameters 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void */ +FSE_DTable* FSE_createDTable(unsigned tableLog); +void FSE_freeDTable(FSE_DTable* dt); + +/* +FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable() + return : 1 if 'dt' is compatible with fast mode, 0 otherwise, + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/* +FSE_decompress_usingDTable(): + Decompress compressed source 'cSrc' of size 'cSrcSize' + using 'dt' into 'dst' which must be already allocated. + Use fastMode==1 only if authorized by result of FSE_buildDTable(). + return : size of regenerated data (necessarily <= maxDstSize) + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt, size_t fastMode); + +/* +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) The first step is to obtain the normalized frequencies of symbols. This can be performed by reading a header with FSE_readHeader(). -'normalizedCounter' must be already allocated, and have at least '*maxSymbolValuePtr+1' cells of short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of short. In practice, that means it's necessary to know 'maxSymbolValue' beforehand, or size the table to handle worst case situations (typically 256). FSE_readHeader will provide 'tableLog' and 'maxSymbolValue' stored into the header. The result of FSE_readHeader() is the number of bytes read from 'header'. -The following values have special meaning : -return 2 : there is only a single symbol value. The value is provided into the second byte of header. -return 1 : data is uncompressed +Note that 'headerSize' must be at least 4 bytes, even if useful information is less than that. If there is an error, the function will return an error code, which can be tested using FSE_isError(). -The next step is to create the decompression tables 'DTable' from 'normalizedCounter'. +The next step is to create the decompression tables 'FSE_DTable' from 'normalizedCounter'. This is performed by the function FSE_buildDTable(). -The space required by 'DTable' must be already allocated and properly aligned. -One can create a DTable using FSE_createDTable(). -The function will return 1 if DTable is compatible with fastMode, 0 otherwise. +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +The function will return 1 if FSE_DTable is compatible with fastMode, 0 otherwise. If there is an error, the function will return an error code, which can be tested using FSE_isError(). -'DTable' can then be used to decompress 'compressed', with FSE_decompress_usingDTable(). -Only trigger fastMode if it was authorized by result of FSE_buildDTable(), otherwise decompression will fail. +'FSE_DTable' can then be used to decompress 'cSrc', with FSE_decompress_usingDTable(). +Only trigger fastMode if it was authorized by the result of FSE_buildDTable(), otherwise decompression will fail. cSrcSize must be correct, otherwise decompression will fail. FSE_decompress_usingDTable() result will tell how many bytes were regenerated. If there is an error, the function will return an error code, which can be tested using FSE_isError(). */ -/****************************************** -* FSE streaming compression API -******************************************/ -typedef struct -{ - size_t bitContainer; - int bitPos; - char* startPtr; - char* ptr; -} FSE_CStream_t; - -typedef struct -{ - ptrdiff_t value; - const void* stateTable; - const void* symbolTT; - unsigned stateLog; -} FSE_CState_t; - -void FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer); -void FSE_initCState(FSE_CState_t* CStatePtr, const void* CTable); - -void FSE_encodeByte(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned char symbol); -void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits); -void FSE_flushBits(FSE_CStream_t* bitC); - -void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* CStatePtr); -size_t FSE_closeCStream(FSE_CStream_t* bitC); - -/* -These functions are inner components of FSE_compress_usingCTable(). -They allow creation of custom streams, mixing multiple tables and bit sources. - -A key property to keep in mind is that encoding and decoding are done **in reverse direction**. -So the first symbol you will encode is the last you will decode, like a lifo stack. - -You will need a few variables to track your CStream. They are : - -void* CTable; // Provided by FSE_buildCTable() -FSE_CStream_t bitC; // bitStream tracking structure -FSE_CState_t state; // State tracking structure - - -The first thing to do is to init the bitStream, and the state. - FSE_initCStream(&bitC, dstBuffer); - FSE_initState(&state, CTable); - -You can then encode your input data, byte after byte. -FSE_encodeByte() outputs a maximum of 'tableLog' bits at a time. -Remember decoding will be done in reverse direction. - FSE_encodeByte(&bitStream, &state, symbol); - -At any time, you can add any bit sequence. -Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders - FSE_addBits(&bitStream, bitField, nbBits); - -The above methods don't commit data to memory, they just store it into local register, for speed. -Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -Writing data to memory is a manual operation, performed by the flushBits function. - FSE_flushBits(&bitStream); - -Your last FSE encoding operation shall be to flush your last state value(s). - FSE_flushState(&bitStream, &state); - -You must then close the bitStream if you opened it with FSE_initCStream(). -It's possible to embed some user-info into the header, as an optionalId [0-31]. -The function returns the size in bytes of CStream. -If there is an error, it returns an errorCode (which can be tested using FSE_isError()). - size_t size = FSE_closeCStream(&bitStream, optionalId); -*/ - - -/****************************************** -* FSE streaming decompression API -******************************************/ -//typedef unsigned int bitD_t; -typedef size_t bitD_t; - -typedef struct -{ - bitD_t bitContainer; - unsigned bitsConsumed; - const char* ptr; - const char* start; -} FSE_DStream_t; - -typedef struct -{ - bitD_t state; - const void* table; -} FSE_DState_t; - - -size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const void* DTable); - -unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD); -bitD_t FSE_readBits(FSE_DStream_t* bitD, unsigned nbBits); -unsigned int FSE_reloadDStream(FSE_DStream_t* bitD); - -unsigned FSE_endOfDStream(const FSE_DStream_t* bitD); -unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); - -/* -Let's now decompose FSE_decompress_usingDTable() into its unitary elements. -You will decode FSE-encoded symbols from the bitStream, -and also any other bitFields you put in, **in reverse order**. - -You will need a few variables to track your bitStream. They are : - -FSE_DStream_t DStream; // Stream context -FSE_DState_t DState; // State context. Multiple ones are possible -const void* DTable; // Decoding table, provided by FSE_buildDTable() -U32 tableLog; // Provided by FSE_readHeader() - -The first thing to do is to init the bitStream. - errorCode = FSE_initDStream(&DStream, &optionalId, srcBuffer, srcSize); - -You should then retrieve your initial state(s) (multiple ones are possible) : - errorCode = FSE_initDState(&DState, &DStream, DTable, tableLog); - -You can then decode your data, symbol after symbol. -For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. -Keep in mind that symbols are decoded in reverse order, like a lifo stack (last in, first out). - unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); - -You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) -Note : maximum allowed nbBits is 25 - unsigned int bitField = FSE_readBits(&DStream, nbBits); - -All above operations only read from local register (which size is controlled by bitD_t==32 bits). -Reading data from memory is manually performed by the reload method. - endSignal = FSE_reloadDStream(&DStream); - -FSE_reloadDStream() result tells if there is still some more data to read from DStream. -0 : there is still some data left into the DStream. -1 Dstream reached end of buffer, but is not yet fully extracted. It will not load data from memory any more. -2 Dstream reached its exact end, corresponding in general to decompression completed. -3 Dstream went too far. Decompression result is corrupted. - -When reaching end of buffer(1), progress slowly if you decode multiple symbols per loop, -to properly detect the exact end of stream. -After each decoded symbol, check if DStream is fully consumed using this simple test : - FSE_reloadDStream(&DStream) >= 2 - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - FSE_endOfDStream(&DStream); -Check also the states. There might be some entropy left there, still able to decode some high probability symbol. - FSE_endOfDState(&DState); -*/ - - #if defined (__cplusplus) } #endif diff --git a/lib/fse_static.h b/lib/fse_static.h index 7d400a5c1..f085d1f5c 100644 --- a/lib/fse_static.h +++ b/lib/fse_static.h @@ -40,24 +40,19 @@ extern "C" { /****************************************** -* Tool functions +* FSE API compatible with DLL ******************************************/ -#define FSE_MAX_HEADERSIZE 512 -#define FSE_COMPRESSBOUND(size) (size + (size>>7) + FSE_MAX_HEADERSIZE) /* Macro can be useful for static allocation */ +#include "fse.h" /****************************************** * Static allocation ******************************************/ -/* You can statically allocate a CTable as a table of U32 using below macro */ +#define FSE_MAX_HEADERSIZE 512 +#define FSE_COMPRESSBOUND(size) (size + (size>>7) + FSE_MAX_HEADERSIZE) /* Macro can be useful for static allocation */ +/* You can statically allocate a CTable as a table of unsigned using below macro */ #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) -#define FSE_DTABLE_SIZE_U32(maxTableLog) ((1<= 2 + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + FSE_endOfDStream(&DStream); +Check also the states. There might be some entropy left there, able to decode some high probability (>50%) symbol. + FSE_endOfDState(&DState); +*/ + + +/****************************************** +* FSE unsafe symbol API +******************************************/ +size_t FSE_readBitsFast(FSE_DStream_t* bitD, unsigned nbBits); /* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */ unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD); diff --git a/lib/zstd.c b/lib/zstd.c index cd5822053..a4e77fd3e 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -535,7 +535,7 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* sr /* return : size of CStream in bits */ static size_t ZSTD_compressLiterals_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, - const void* CTable) + const FSE_CTable* CTable) { const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; @@ -553,32 +553,32 @@ static size_t ZSTD_compressLiterals_usingCTable(void* dst, size_t dstSize, // join to mod 2 if (srcSize & 1) { - FSE_encodeByte(&bitC, &CState1, *ip++); + FSE_encodeSymbol(&bitC, &CState1, *ip++); FSE_flushBits(&bitC); } // join to mod 4 if ((sizeof(size_t)*8 > LitFSELog*4+7 ) && (srcSize & 2)) // test bit 2 { - FSE_encodeByte(&bitC, &CState2, *ip++); - FSE_encodeByte(&bitC, &CState1, *ip++); + FSE_encodeSymbol(&bitC, &CState2, *ip++); + FSE_encodeSymbol(&bitC, &CState1, *ip++); FSE_flushBits(&bitC); } // 2 or 4 encoding per loop while (ip LitFSELog*4+7 ) // this test must be static { - FSE_encodeByte(&bitC, &CState2, *ip++); - FSE_encodeByte(&bitC, &CState1, *ip++); + FSE_encodeSymbol(&bitC, &CState2, *ip++); + FSE_encodeSymbol(&bitC, &CState1, *ip++); } FSE_flushBits(&bitC); @@ -618,7 +618,7 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize, if (dstSize < FSE_compressBound(srcSize)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* Scan input and build symbol stats */ - errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue); + errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize); if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC; if (errorCode == srcSize) return 1; if (errorCode < (srcSize >> 6)) return 0; /* cheap heuristic : probably not compressible enough */ @@ -628,14 +628,14 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize, if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC; /* Write table description header */ - errorCode = FSE_writeHeader (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog); + errorCode = FSE_writeNCount (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog); if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC; op += errorCode; /* Compress */ - errorCode = FSE_buildCTable (&CTable, norm, maxSymbolValue, tableLog); + errorCode = FSE_buildCTable (CTable, norm, maxSymbolValue, tableLog); if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC; - errorCode = ZSTD_compressLiterals_usingCTable(op, oend - op, ip, srcSize, &CTable); + errorCode = ZSTD_compressLiterals_usingCTable(op, oend - op, ip, srcSize, CTable); if (ZSTD_isError(errorCode)) return errorCode; op += errorCode; @@ -738,7 +738,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, /* Encoding table of Literal Lengths */ max = MaxLL; - mostFrequent = FSE_countFast(count, seqStorePtr->litLengthStart, nbSeq, &max); + mostFrequent = FSE_countFast(count, &max, seqStorePtr->litLengthStart, nbSeq); if (mostFrequent == nbSeq) { *op++ = *(seqStorePtr->litLengthStart); @@ -754,7 +754,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, { tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max); - op += FSE_writeHeader(op, maxDstSize, norm, max, tableLog); + op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog); FSE_buildCTable(CTable_LitLength, norm, max, tableLog); LLtype = bt_compressed; } @@ -771,7 +771,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, if (op_offset_start[i]==0) offsetBits_start[i]=0; } offsetBitsPtr += nbSeq; - mostFrequent = FSE_countFast(count, offsetBits_start, nbSeq, &max); + mostFrequent = FSE_countFast(count, &max, offsetBits_start, nbSeq); } if (mostFrequent == nbSeq) { @@ -788,14 +788,14 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, { tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max); - op += FSE_writeHeader(op, maxDstSize, norm, max, tableLog); + op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog); FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog); Offtype = bt_compressed; } /* Encoding Table of MatchLengths */ max = MaxML; - mostFrequent = FSE_countFast(count, seqStorePtr->matchLengthStart, nbSeq, &max); + mostFrequent = FSE_countFast(count, &max, seqStorePtr->matchLengthStart, nbSeq); if (mostFrequent == nbSeq) { *op++ = *seqStorePtr->matchLengthStart; @@ -811,7 +811,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, { tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max); - op += FSE_writeHeader(op, maxDstSize, norm, max, tableLog); + op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog); FSE_buildCTable(CTable_MatchLength, norm, max, tableLog); MLtype = bt_compressed; } @@ -836,12 +836,12 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, BYTE offCode = *(--offsetBitsPtr); /* 32b*/ /* 64b*/ U32 nbBits = (offCode-1) * (!!offCode); BYTE litLength = *(--op_litLength); /* (7)*/ /* (7)*/ - FSE_encodeByte(&blockStream, &stateMatchLength, matchLength); /* 17 */ /* 17 */ + FSE_encodeSymbol(&blockStream, &stateMatchLength, matchLength); /* 17 */ /* 17 */ if (ZSTD_32bits()) FSE_flushBits(&blockStream); /* 7 */ FSE_addBits(&blockStream, offset, nbBits); /* 32 */ /* 42 */ if (ZSTD_32bits()) FSE_flushBits(&blockStream); /* 7 */ - FSE_encodeByte(&blockStream, &stateOffsetBits, offCode); /* 16 */ /* 51 */ - FSE_encodeByte(&blockStream, &stateLitLength, litLength); /* 26 */ /* 61 */ + FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 16 */ /* 51 */ + FSE_encodeSymbol(&blockStream, &stateLitLength, litLength); /* 26 */ /* 61 */ FSE_flushBits(&blockStream); /* 7 */ /* 7 */ } @@ -1243,7 +1243,7 @@ static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const voi FORCE_INLINE size_t ZSTD_decompressLiterals_usingDTable_generic( void* const dst, size_t maxDstSize, const void* src, size_t srcSize, - const void* DTable, U32 fast) + const FSE_DTable* DTable, U32 fast) { BYTE* op = (BYTE*) dst; BYTE* const olimit = op; @@ -1303,7 +1303,7 @@ FORCE_INLINE size_t ZSTD_decompressLiterals_usingDTable_generic( static size_t ZSTD_decompressLiterals_usingDTable( void* const dst, size_t maxDstSize, const void* src, size_t srcSize, - const void* DTable, U32 fast) + const FSE_DTable* DTable, U32 fast) { if (fast) return ZSTD_decompressLiterals_usingDTable_generic(dst, maxDstSize, src, srcSize, DTable, 1); return ZSTD_decompressLiterals_usingDTable_generic(dst, maxDstSize, src, srcSize, DTable, 0); @@ -1315,7 +1315,7 @@ static size_t ZSTD_decompressLiterals(void* ctx, void* dst, size_t maxDstSize, /* assumed : blockType == blockCompressed */ const BYTE* ip = (const BYTE*)src; short norm[256]; - void* DTable = ctx; + FSE_DTable* DTable = (FSE_DTable*)ctx; U32 maxSymbolValue = 255; U32 tableLog; U32 fastMode; @@ -1323,7 +1323,7 @@ static size_t ZSTD_decompressLiterals(void* ctx, void* dst, size_t maxDstSize, if (srcSize < 2) return (size_t)-ZSTD_ERROR_wrongLBlockSize; /* too small input size */ - errorCode = FSE_readHeader (norm, &maxSymbolValue, &tableLog, ip, srcSize); + errorCode = FSE_readNCount (norm, &maxSymbolValue, &tableLog, ip, srcSize); if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC; ip += errorCode; srcSize -= errorCode; @@ -1380,7 +1380,7 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx, size_t ZSTD_decodeSeqHeaders(size_t* lastLLPtr, const BYTE** dumpsPtr, - void* DTableLL, void* DTableML, void* DTableOffb, + FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize) { const BYTE* const istart = (const BYTE* const)src; @@ -1427,7 +1427,7 @@ size_t ZSTD_decodeSeqHeaders(size_t* lastLLPtr, const BYTE** dumpsPtr, FSE_buildDTable_raw(DTableLL, LLbits); break; default : max = MaxLL; - headerSize = FSE_readHeader(norm, &max, &LLlog, ip, iend-ip); + headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; ip += headerSize; FSE_buildDTable(DTableLL, norm, max, LLlog); @@ -1444,7 +1444,7 @@ size_t ZSTD_decodeSeqHeaders(size_t* lastLLPtr, const BYTE** dumpsPtr, FSE_buildDTable_raw(DTableOffb, Offbits); break; default : max = MaxOff; - headerSize = FSE_readHeader(norm, &max, &Offlog, ip, iend-ip); + headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; ip += headerSize; FSE_buildDTable(DTableOffb, norm, max, Offlog); @@ -1461,7 +1461,7 @@ size_t ZSTD_decodeSeqHeaders(size_t* lastLLPtr, const BYTE** dumpsPtr, FSE_buildDTable_raw(DTableML, MLbits); break; default : max = MaxML; - headerSize = FSE_readHeader(norm, &max, &MLlog, ip, iend-ip); + headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; ip += headerSize; FSE_buildDTable(DTableML, norm, max, MLlog); @@ -1489,9 +1489,9 @@ FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize const BYTE* litEnd; const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */ const size_t dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */ - void* DTableML = ctx; - void* DTableLL = ((U32*)ctx) + FSE_DTABLE_SIZE_U32(MLFSELog); - void* DTableOffb = ((U32*)DTableLL) + FSE_DTABLE_SIZE_U32(LLFSELog); + FSE_DTable* DTableML = (FSE_DTable*)ctx; + FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog); + FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog); /* blockType == blockCompressed, srcSize is trusted */ diff --git a/programs/fullbench.c b/programs/fullbench.c index 7afd12c88..11d9b62d1 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -230,7 +230,7 @@ static size_t g_cSize = 0; extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); extern size_t ZSTD_decodeLiteralsBlock(void* ctx, void* dst, size_t maxDstSize, const BYTE** litPtr, const void* src, size_t srcSize); -extern size_t ZSTD_decodeSeqHeaders(size_t* lastLLPtr, const BYTE** dumpsPtr, void* DTableLL, void* DTableML, void* DTableOffb, const void* src, size_t srcSize); +extern size_t ZSTD_decodeSeqHeaders(size_t* lastLLPtr, const BYTE** dumpsPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize); size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) From b6a0072699e5817ea510a73afecdab22a7ae71ec Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jul 2015 16:23:52 -0800 Subject: [PATCH 09/22] Updated Visual projects --- programs/datagen.c | 67 ++++++++++--------- visual/2012/fullbench/fullbench.vcxproj | 14 ++++ .../2012/fullbench/fullbench.vcxproj.filters | 6 ++ visual/2012/fuzzer/fuzzer.vcxproj | 12 ++++ visual/2012/zstd.sln | 0 visual/2012/zstd/zstd.vcxproj | 12 ++++ 6 files changed, 79 insertions(+), 32 deletions(-) mode change 100755 => 100644 visual/2012/zstd.sln diff --git a/programs/datagen.c b/programs/datagen.c index 20eb30e99..9df9da841 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -71,11 +71,22 @@ #define PRIME2 2246822519U +/************************************** +* Local types +**************************************/ +#define LTLOG 13 +#define LTSIZE (1<> (32 - r))) -static U32 RDG_rand(U32* src) +static unsigned int RDG_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; @@ -86,23 +97,19 @@ static U32 RDG_rand(U32* src) } -#define LTSIZE 8192 -#define LTMASK (LTSIZE-1) -static void* RDG_createLiteralDistrib(const double ld) +static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) { - BYTE* lt = (BYTE*)malloc(LTSIZE); U32 i = 0; BYTE character = '0'; BYTE firstChar = '('; BYTE lastChar = '}'; - if (ld<=0.02) + if (ld==0.0) { + character = 0; firstChar = 0; - lastChar = 254; - character = firstChar; + lastChar =255; } - if (ld==0.0) lastChar = 255; while (i lastChar) character = firstChar; } - return lt; } -static char RDG_genChar(U32* seed, const void* ltctx) + +static BYTE RDG_genChar(U32* seed, const litDistribTable lt) { - const BYTE* lt = (const BYTE*)ltctx; U32 id = RDG_rand(seed) & LTMASK; - return lt[id]; + return (lt[id]); } + #define RDG_DICTSIZE (32 KB) #define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) #define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) -void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr) +void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) { BYTE* buffPtr = (BYTE*)buffer; const U32 matchProba32 = (U32)(32768 * matchProba); size_t pos = prefixSize; - void* ldctx = litTable; U32* seed = seedPtr; /* special case */ while (matchProba >= 1.0) { size_t size0 = RDG_rand(seed) & 3; - size0 = 1U << (16 + size0 * 2); + size0 = (size_t)1 << (16 + size0 * 2); size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ if (buffSize < pos + size0) { @@ -147,11 +153,11 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match } memset(buffPtr+pos, 0, size0); pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, ldctx); + buffPtr[pos-1] = RDG_genChar(seed, lt); } /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1; + if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; /* Generate compressible data */ while (pos < buffSize) @@ -160,11 +166,11 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match if (RDG_RAND15BITS < matchProba32) { /* Copy (within 32K) */ - int match; - U32 d; + size_t match; + size_t d; int length = RDG_RANDLENGTH + 4; U32 offset = RDG_RAND15BITS + 1; - if (offset > pos) offset = pos; + if (offset > pos) offset = (U32)pos; match = pos - offset; d = pos + length; if (d > buffSize) d = buffSize; @@ -177,7 +183,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match size_t length = RDG_RANDLENGTH; d = pos + length; if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx); + while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); } } } @@ -185,11 +191,10 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) { - void* ldctx; + litDistribTable lt; if (litProba==0.0) litProba = matchProba / 4.5; - ldctx = RDG_createLiteralDistrib(litProba); - RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed); - free(ldctx); + RDG_fillLiteralDistrib(lt, litProba); + RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); } @@ -199,26 +204,24 @@ void RDG_genOut(unsigned long long size, double matchProba, double litProba, uns BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; U64 total = 0; size_t genBlockSize = RDG_BLOCKSIZE; - void* ldctx; + litDistribTable lt; /* init */ if (litProba==0.0) litProba = matchProba / 4.5; - ldctx = RDG_createLiteralDistrib(litProba); + RDG_fillLiteralDistrib(lt, litProba); SET_BINARY_MODE(stdout); /* Generate dict */ - RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed); + RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed); /* Generate compressible data */ while (total < size) { - RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed); + RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed); if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); total += genBlockSize; fwrite(buff, 1, genBlockSize, stdout); /* update dict */ memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); } - - free(ldctx); } diff --git a/visual/2012/fullbench/fullbench.vcxproj b/visual/2012/fullbench/fullbench.vcxproj index 4b6e0c35c..5d47329f8 100644 --- a/visual/2012/fullbench/fullbench.vcxproj +++ b/visual/2012/fullbench/fullbench.vcxproj @@ -69,18 +69,22 @@ true $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true true $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true false $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true false $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true @@ -89,6 +93,8 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -102,6 +108,8 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -117,6 +125,8 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -134,6 +144,8 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -145,6 +157,7 @@ + @@ -152,6 +165,7 @@ + diff --git a/visual/2012/fullbench/fullbench.vcxproj.filters b/visual/2012/fullbench/fullbench.vcxproj.filters index 35a890010..5890bd638 100644 --- a/visual/2012/fullbench/fullbench.vcxproj.filters +++ b/visual/2012/fullbench/fullbench.vcxproj.filters @@ -24,6 +24,9 @@ Fichiers sources + + Fichiers sources + @@ -38,5 +41,8 @@ Fichiers d%27en-tête + + Fichiers d%27en-tête + \ No newline at end of file diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2012/fuzzer/fuzzer.vcxproj index a0dc550c2..68d5273fd 100644 --- a/visual/2012/fuzzer/fuzzer.vcxproj +++ b/visual/2012/fuzzer/fuzzer.vcxproj @@ -69,18 +69,22 @@ true $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true true $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true false $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true false $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true @@ -89,6 +93,8 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -102,6 +108,8 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -117,6 +125,8 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -134,6 +144,8 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console diff --git a/visual/2012/zstd.sln b/visual/2012/zstd.sln old mode 100755 new mode 100644 diff --git a/visual/2012/zstd/zstd.vcxproj b/visual/2012/zstd/zstd.vcxproj index 45e4bc154..eb8289b44 100644 --- a/visual/2012/zstd/zstd.vcxproj +++ b/visual/2012/zstd/zstd.vcxproj @@ -86,18 +86,22 @@ true $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true true $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true false $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true false $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true @@ -106,6 +110,8 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -119,6 +125,8 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -134,6 +142,8 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console @@ -151,6 +161,8 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true Console From ae67293e23e63bc843f7b628de6cddc082c0286b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jul 2015 18:00:05 -0800 Subject: [PATCH 10/22] Updated Visual analyzer settings --- visual/2012/fullbench/fullbench.vcxproj | 4 ++++ visual/2012/fuzzer/fuzzer.vcxproj | 4 ++++ visual/2012/zstd/zstd.vcxproj | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/visual/2012/fullbench/fullbench.vcxproj b/visual/2012/fullbench/fullbench.vcxproj index 5d47329f8..ac5c68134 100644 --- a/visual/2012/fullbench/fullbench.vcxproj +++ b/visual/2012/fullbench/fullbench.vcxproj @@ -95,6 +95,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -110,6 +111,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -127,6 +129,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -146,6 +149,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2012/fuzzer/fuzzer.vcxproj index 68d5273fd..8a114aefc 100644 --- a/visual/2012/fuzzer/fuzzer.vcxproj +++ b/visual/2012/fuzzer/fuzzer.vcxproj @@ -95,6 +95,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -110,6 +111,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -127,6 +129,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -146,6 +149,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console diff --git a/visual/2012/zstd/zstd.vcxproj b/visual/2012/zstd/zstd.vcxproj index eb8289b44..18e9824ca 100644 --- a/visual/2012/zstd/zstd.vcxproj +++ b/visual/2012/zstd/zstd.vcxproj @@ -112,6 +112,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -127,6 +128,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -144,6 +146,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console @@ -163,6 +166,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true + /analyze:stacksize19000 %(AdditionalOptions) Console From 14c6d0db9f0f4fcec44311806ff4cf138af188d1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jul 2015 18:14:14 -0800 Subject: [PATCH 11/22] updated xxhash version --- programs/xxhash.c | 321 ++++++++++++++++++++++------------------------ programs/xxhash.h | 56 ++++++-- 2 files changed, 200 insertions(+), 177 deletions(-) diff --git a/programs/xxhash.c b/programs/xxhash.c index e2216262e..e6fb8f149 100644 --- a/programs/xxhash.c +++ b/programs/xxhash.c @@ -1,6 +1,7 @@ /* xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. +Copyright (C) 2012-2015, Yann Collet + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,128 +28,113 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- public discussion board : https://groups.google.com/forum/#!forum/lz4c +- xxHash source repository : https://github.com/Cyan4973/xxHash */ -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +/************************************** +* Tuning parameters +**************************************/ +/* Unaligned memory access is automatically enabled for "common" CPU, such as x86. + * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. + * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. + * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). + */ #if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) # define XXH_USE_UNALIGNED_ACCESS 1 #endif -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 +/* XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. +/* XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1. + * It will improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ #define XXH_FORCE_NATIVE_FORMAT 0 -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif -#ifdef _MSC_VER // Visual Studio +/************************************** +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # define FORCE_INLINE static __forceinline #else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif -//************************************** -// Includes & Memory related functions -//************************************** + +/************************************** +* Includes & Memory related functions +***************************************/ #include "xxhash.h" -// Modify the local functions below should you wish to use some other memory routines -// for malloc(), free() +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ #include static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } -// for memcpy() +/* for memcpy() */ #include -static void* XXH_memcpy(void* dest, const void* src, size_t size) +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + + +/************************************** +* Basic Types +***************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +static U32 XXH_read32(const void* memPtr) { - return memcpy(dest,src,size); + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; } -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S -{ - U32 v; -} _PACKED U32_S; -typedef struct _U64_S -{ - U64 v; -} _PACKED U64_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) - - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** +/****************************************** +* Compiler-specific Functions and Macros +******************************************/ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ #if defined(_MSC_VER) # define XXH_rotl32(x,r) _rotl(x,r) # define XXH_rotl64(x,r) _rotl64(x,r) @@ -157,21 +143,21 @@ typedef struct _U64_S # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) #endif -#if defined(_MSC_VER) // Visual Studio +#if defined(_MSC_VER) /* Visual Studio */ # define XXH_swap32 _byteswap_ulong # define XXH_swap64 _byteswap_uint64 #elif GCC_VERSION >= 403 # define XXH_swap32 __builtin_bswap32 # define XXH_swap64 __builtin_bswap64 #else -static inline U32 XXH_swap32 (U32 x) +static U32 XXH_swap32 (U32 x) { return ((x << 24) & 0xff000000 ) | ((x << 8) & 0x00ff0000 ) | ((x >> 8) & 0x0000ff00 ) | ((x >> 24) & 0x000000ff ); } -static inline U64 XXH_swap64 (U64 x) +static U64 XXH_swap64 (U64 x) { return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | @@ -185,9 +171,57 @@ static inline U64 XXH_swap64 (U64 x) #endif -//************************************** -// Constants -//************************************** +/*************************************** +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ +static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) +#endif + + +/***************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + + +/*************************************** +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ + + +/*************************************** +* Constants +***************************************/ #define PRIME32_1 2654435761U #define PRIME32_2 2246822519U #define PRIME32_3 3266489917U @@ -200,57 +234,10 @@ static inline U64 XXH_swap64 (U64 x) #define PRIME64_4 9650029242287828579ULL #define PRIME64_5 2870177450012600261ULL -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch -static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) -#endif - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); - else - return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); -} - -FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - - -//**************************** -// Simple Hash Functions -//**************************** +/***************************** +* Simple Hash Functions +*****************************/ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; @@ -328,10 +315,10 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH } -unsigned int XXH32 (const void* input, size_t len, unsigned seed) +unsigned XXH32 (const void* input, size_t len, unsigned seed) { #if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH32_state_t state; XXH32_reset(&state, seed); XXH32_update(&state, input, len); @@ -340,7 +327,7 @@ unsigned int XXH32 (const void* input, size_t len, unsigned seed) XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage + if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -471,7 +458,7 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) { #if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, input, len); @@ -480,7 +467,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -497,7 +484,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed } /**************************************************** - * Advanced Hash Functions +* Advanced Hash Functions ****************************************************/ /*** Allocation ***/ @@ -528,7 +515,7 @@ typedef struct XXH32_state_t* XXH32_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) @@ -539,7 +526,7 @@ XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) XXH64_state_t* XXH64_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) @@ -590,14 +577,14 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v state->total_len += len; - if (state->memsize + len < 16) // fill in tmp buffer + if (state->memsize + len < 16) /* fill in tmp buffer */ { XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) // some data left from previous update + if (state->memsize) /* some data left from previous update */ { XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); { @@ -681,9 +668,9 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) { - XXH_istate32_t* state = (XXH_istate32_t*) state_in; + const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; const BYTE * p = (const BYTE*)state->mem32; - BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; + const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->total_len >= 16) @@ -744,14 +731,14 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v state->total_len += len; - if (state->memsize + len < 32) // fill in tmp buffer + if (state->memsize + len < 32) /* fill in tmp buffer */ { XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) // some data left from previous update + if (state->memsize) /* some data left from previous update */ { XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); { @@ -835,9 +822,9 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) { - XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; const BYTE * p = (const BYTE*)state->mem64; - BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; + const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) diff --git a/programs/xxhash.h b/programs/xxhash.h index 55b45015a..c60aa6157 100644 --- a/programs/xxhash.h +++ b/programs/xxhash.h @@ -1,7 +1,8 @@ /* xxHash - Extremely Fast Hash algorithm Header File - Copyright (C) 2012-2014, Yann Collet. + Copyright (C) 2012-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ + - xxHash source repository : https://github.com/Cyan4973/xxHash */ /* Notice extracted from xxHash homepage : @@ -55,6 +56,12 @@ SHA1-32 0.28 GB/s 10 Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s */ #pragma once @@ -65,20 +72,48 @@ extern "C" { /***************************** - Includes +* Definitions *****************************/ #include /* size_t */ - - -/***************************** - Type -*****************************/ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; +/***************************** +* Namespace Emulation +*****************************/ +/* Motivations : + +If you need to include xxHash into your library, +but wish to avoid xxHash symbols to be present on your library interface +in an effort to avoid potential name collision if another library also includes xxHash, + +you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash +with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values). + +Note that no change is required within the calling program : +it can still call xxHash functions using their regular name. +They will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +#endif + /***************************** - Simple Hash Functions +* Simple Hash Functions *****************************/ unsigned int XXH32 (const void* input, size_t length, unsigned seed); @@ -93,12 +128,13 @@ XXH32() : Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s XXH64() : Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Faster on 64-bits systems. Slower on 32-bits systems. */ /***************************** - Advanced Hash Functions +* Advanced Hash Functions *****************************/ typedef struct { long long ll[ 6]; } XXH32_state_t; typedef struct { long long ll[11]; } XXH64_state_t; From 7393c5a51deff759355f21c743165a4adbcf7aac Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jul 2015 18:20:42 -0800 Subject: [PATCH 12/22] Advanced API change : Cctx & DCtx are now incomplete types for stricter type checking --- lib/zstd.c | 33 +++++++++++++++++---------------- lib/zstd_static.h | 23 ++++++++++++----------- programs/fileio.c | 4 ++-- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index a4e77fd3e..b97e27fdd 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -310,7 +310,7 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) } -typedef struct +typedef struct ZSTD_Cctx_s { const BYTE* base; U32 current; @@ -324,7 +324,7 @@ typedef struct } cctxi_t; -ZSTD_cctx_t ZSTD_createCCtx(void) +ZSTD_Cctx* ZSTD_createCCtx(void) { cctxi_t* ctx = (cctxi_t*) malloc( sizeof(cctxi_t) ); ctx->seqStore.buffer = malloc(WORKPLACESIZE); @@ -333,17 +333,17 @@ ZSTD_cctx_t ZSTD_createCCtx(void) ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE; ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2); ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2); - return (ZSTD_cctx_t)ctx; + return (ZSTD_Cctx* )ctx; } -void ZSTD_resetCCtx(ZSTD_cctx_t cctx) +void ZSTD_resetCCtx(ZSTD_Cctx* cctx) { cctxi_t* ctx = (cctxi_t*)cctx; ctx->base = NULL; memset(ctx->hashTable, 0, HASH_TABLESIZE*4); } -size_t ZSTD_freeCCtx(ZSTD_cctx_t cctx) +size_t ZSTD_freeCCtx(ZSTD_Cctx* cctx) { cctxi_t* ctx = (cctxi_t*) (cctx); free(ctx->seqStore.buffer); @@ -996,7 +996,7 @@ static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const } -size_t ZSTD_compressBegin(ZSTD_cctx_t ctx, void* dst, size_t maxDstSize) +size_t ZSTD_compressBegin(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize) { /* Sanity check */ if (maxDstSize < ZSTD_frameHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; @@ -1080,7 +1080,7 @@ static void ZSTD_limitCtx(void* cctx, const U32 limit) } -size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { cctxi_t* ctx = (cctxi_t*) cctx; const BYTE* const istart = (const BYTE* const)src; @@ -1146,7 +1146,7 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con } -size_t ZSTD_compressEnd(ZSTD_cctx_t ctx, void* dst, size_t maxDstSize) +size_t ZSTD_compressEnd(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize) { BYTE* op = (BYTE*)dst; @@ -1163,7 +1163,7 @@ size_t ZSTD_compressEnd(ZSTD_cctx_t ctx, void* dst, size_t maxDstSize) } -static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; BYTE* op = ostart; @@ -1197,7 +1197,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - void* ctx; + ZSTD_Cctx* ctx; size_t r; ctx = ZSTD_createCCtx(); @@ -1207,6 +1207,7 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi } + /************************************************************** * Decompression code **************************************************************/ @@ -1710,7 +1711,7 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src * Streaming Decompression API *******************************/ -typedef struct +typedef struct ZTSD_Dctx_s { U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)]; size_t expected; @@ -1719,27 +1720,27 @@ typedef struct } dctx_t; -ZSTD_dctx_t ZSTD_createDCtx(void) +ZSTD_Dctx* ZSTD_createDCtx(void) { dctx_t* dctx = (dctx_t*)malloc(sizeof(dctx_t)); dctx->expected = ZSTD_frameHeaderSize; dctx->phase = 0; - return (ZSTD_dctx_t)dctx; + return (ZSTD_Dctx*)dctx; } -size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx) +size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx) { free(dctx); return 0; } -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx) +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx) { return ((dctx_t*)dctx)->expected; } -size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { dctx_t* ctx = (dctx_t*)dctx; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index b63fbba06..a059288fb 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -45,20 +45,21 @@ extern "C" { /************************************** * Streaming functions **************************************/ -typedef void* ZSTD_cctx_t; -ZSTD_cctx_t ZSTD_createCCtx(void); -size_t ZSTD_freeCCtx(ZSTD_cctx_t cctx); +typedef struct ZSTD_Cctx_s ZSTD_Cctx; +ZSTD_Cctx* ZSTD_createCCtx(void); +size_t ZSTD_freeCCtx(ZSTD_Cctx* cctx); -size_t ZSTD_compressBegin(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize); -size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -size_t ZSTD_compressEnd(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize); +size_t ZSTD_compressBegin(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize); +size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize); -typedef void* ZSTD_dctx_t; -ZSTD_dctx_t ZSTD_createDCtx(void); -size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx); -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx); -size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +typedef struct ZSTD_Dctx_s ZSTD_Dctx; +ZSTD_Dctx* ZSTD_createDCtx(void); +size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx); + +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx); +size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); /* Use above functions alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as input to ZSTD_decompressContinue(). diff --git a/programs/fileio.c b/programs/fileio.c index ac36fa3e2..04d938fa4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -235,7 +235,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* FILE* finput; FILE* foutput; size_t sizeCheck, cSize; - ZSTD_cctx_t ctx; + ZSTD_Cctx* ctx; /* Init */ @@ -319,7 +319,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha U32 wNbBlocks = 4; U64 filesize = 0; BYTE* header[MAXHEADERSIZE]; - ZSTD_cctx_t dctx; + ZSTD_Dctx* dctx; size_t toRead; size_t sizeCheck; From 94f998b1fc1034c8cdc58850755279b3e5288eaa Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jul 2015 23:10:40 -0800 Subject: [PATCH 13/22] Fixed a few visual analyzer warnings --- lib/zstd.c | 19 +++++++++---------- programs/fuzzer.c | 6 ++++++ programs/zstdcli.c | 2 ++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index b97e27fdd..0eb0c2e78 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -326,26 +326,25 @@ typedef struct ZSTD_Cctx_s ZSTD_Cctx* ZSTD_createCCtx(void) { - cctxi_t* ctx = (cctxi_t*) malloc( sizeof(cctxi_t) ); + ZSTD_Cctx* ctx = (ZSTD_Cctx*) malloc( sizeof(ZSTD_Cctx) ); + if (ctx==NULL) return NULL; ctx->seqStore.buffer = malloc(WORKPLACESIZE); ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer); ctx->seqStore.litStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2)); ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE; ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2); ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2); - return (ZSTD_Cctx* )ctx; + return ctx; } -void ZSTD_resetCCtx(ZSTD_Cctx* cctx) +void ZSTD_resetCCtx(ZSTD_Cctx* ctx) { - cctxi_t* ctx = (cctxi_t*)cctx; ctx->base = NULL; memset(ctx->hashTable, 0, HASH_TABLESIZE*4); } -size_t ZSTD_freeCCtx(ZSTD_Cctx* cctx) +size_t ZSTD_freeCCtx(ZSTD_Cctx* ctx) { - cctxi_t* ctx = (cctxi_t*) (cctx); free(ctx->seqStore.buffer); free(ctx); return 0; @@ -687,7 +686,6 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const size_t maxLSize = maxCSize > minSeqSize ? maxCSize - minSeqSize : 0; BYTE* seqHead; - /* init */ op = dst; @@ -1711,7 +1709,7 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src * Streaming Decompression API *******************************/ -typedef struct ZTSD_Dctx_s +typedef struct ZSTD_Dctx_s { U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)]; size_t expected; @@ -1722,10 +1720,11 @@ typedef struct ZTSD_Dctx_s ZSTD_Dctx* ZSTD_createDCtx(void) { - dctx_t* dctx = (dctx_t*)malloc(sizeof(dctx_t)); + ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx)); + if (dctx==NULL) return NULL; dctx->expected = ZSTD_frameHeaderSize; dctx->phase = 0; - return (ZSTD_Dctx*)dctx; + return dctx; } size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index b2dbcd3d9..8d48dcd2d 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -206,6 +206,12 @@ static int basicUnitTests(U32 seed, double compressibility) CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + if (!CNBuffer || !compressedBuffer || !decodedBuffer) + { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } FUZ_generateSynthetic(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); // Basic tests diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7e7c56754..e39266d45 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -281,6 +281,7 @@ int main(int argc, char** argv) { size_t l = strlen(inFileName); dynNameSpace = (char*)calloc(1,l+5); + if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); } strcpy(dynNameSpace, inFileName); strcpy(dynNameSpace+l, ZSTD_EXTENSION); outFileName = dynNameSpace; @@ -292,6 +293,7 @@ int main(int argc, char** argv) size_t outl; size_t inl = strlen(inFileName); dynNameSpace = (char*)calloc(1,inl+1); + if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); } outFileName = dynNameSpace; strcpy(dynNameSpace, inFileName); outl = inl; From b5e06dc7c31070c30777843444e28078dfc9a9a2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jul 2015 23:20:56 -0800 Subject: [PATCH 14/22] Fixed a few minor visual analyzer warnings --- programs/datagen.c | 2 +- programs/fileio.c | 2 +- programs/fullbench.c | 2 +- programs/fuzzer.c | 4 +++- programs/zstdcli.c | 4 +++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index 9df9da841..50daaa077 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -56,7 +56,7 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ # include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define SET_BINARY_MODE(file) {int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } #else # define SET_BINARY_MODE(file) #endif diff --git a/programs/fileio.c b/programs/fileio.c index 04d938fa4..ee23dc01c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -65,7 +65,7 @@ # ifdef __MINGW32__ /* int _fileno(FILE *stream); // seems no longer useful // MINGW somehow forgets to include this windows declaration into */ # endif -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else # include /* isatty */ diff --git a/programs/fullbench.c b/programs/fullbench.c index 11d9b62d1..b7ecc8db1 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -653,7 +653,7 @@ int main(int argc, char** argv) result = benchSample(benchNb); else result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb); - if (main_pause) { printf("press enter...\n"); getchar(); } + if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } return result; } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 8d48dcd2d..db4bc65fa 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -512,8 +512,10 @@ int main(int argc, char** argv) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); if (mainPause) { + int unused; DISPLAY("Press Enter \n"); - getchar(); + unused = getchar(); + (void)unused; } return result; } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e39266d45..7687218c4 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -150,8 +150,10 @@ static int badusage(const char* programName) static void waitEnter(void) { + int unused; DISPLAY("Press enter to continue...\n"); - getchar(); + unused = getchar(); + (void)unused; } From ee28a0def40ba0dac1ca957e543d8ca73951d0bd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jul 2015 23:45:35 -0800 Subject: [PATCH 15/22] Fixed issue #34 and #31 --- lib/zstd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index 0eb0c2e78..277307fbc 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -737,7 +737,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, /* Encoding table of Literal Lengths */ max = MaxLL; mostFrequent = FSE_countFast(count, &max, seqStorePtr->litLengthStart, nbSeq); - if (mostFrequent == nbSeq) + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { *op++ = *(seqStorePtr->litLengthStart); FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); @@ -771,7 +771,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, offsetBitsPtr += nbSeq; mostFrequent = FSE_countFast(count, &max, offsetBits_start, nbSeq); } - if (mostFrequent == nbSeq) + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { *op++ = *offsetBits_start; FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); @@ -794,7 +794,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, /* Encoding Table of MatchLengths */ max = MaxML; mostFrequent = FSE_countFast(count, &max, seqStorePtr->matchLengthStart, nbSeq); - if (mostFrequent == nbSeq) + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { *op++ = *seqStorePtr->matchLengthStart; FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); From 7083b8a54aa68ada01ebb7b0aec0857d01b809c4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jul 2015 23:58:59 -0800 Subject: [PATCH 16/22] Travis CI : added ARM compilation tests --- .travis.yml | 2 ++ Makefile | 6 ++++++ lib/Makefile | 8 +++++--- programs/Makefile | 12 ++++++------ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b0265e4aa..62e818929 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ compiler: gcc script: make test-travis before_install: - sudo apt-get update -qq + - sudo apt-get install -qq gcc-arm-linux-gnueabi - sudo apt-get install -qq clang - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib @@ -13,6 +14,7 @@ env: - ZSTD_TRAVIS_CI_ENV=test-all - ZSTD_TRAVIS_CI_ENV=clangtest - ZSTD_TRAVIS_CI_ENV=gpptest + - ZSTD_TRAVIS_CI_ENV=armtest matrix: fast_finish: true diff --git a/Makefile b/Makefile index adc6d5e8a..47d727cc9 100644 --- a/Makefile +++ b/Makefile @@ -98,5 +98,11 @@ clangtest: clean gpptest: clean $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" +armtest: clean + cd $(ZSTDDIR); $(MAKE) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" + cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror" + +sanitize: clean + $(MAKE) test CC=clang CPPFLAGS="-g -fsanitize=undefined" FUZZER_TIME="-T1mn" NB_LOOPS=-i1 endif diff --git a/lib/Makefile b/lib/Makefile index 4d838167b..46000f7f9 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -41,7 +41,9 @@ LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) DESTDIR?= PREFIX ?= /usr CFLAGS ?= -O3 -CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes +LDFLAGS = -I. +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include @@ -67,10 +69,10 @@ all: libzstd libzstd: zstd.c @echo compiling static library - @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ + @$(CC) $(FLAGS) -c $^ @$(AR) rcs libzstd.a zstd.o @echo compiling dynamic library $(LIBVER) - @$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) + @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) @echo creating versioned links @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) diff --git a/programs/Makefile b/programs/Makefile index c621b6861..0526c36c8 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -122,14 +122,14 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-all: test test32 test-zstd: zstd datagen - ./datagen | ./zstd -v | ./zstd -d > $(VOID) - ./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID) - ./datagen -g6GB | ./zstd -vq | ./zstd -d > $(VOID) + ./datagen | ./zstd -v | ./zstd -d > $(VOID) + ./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID) + ./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID) test-zstd32: zstd32 datagen - ./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID) - ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) - ./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID) + ./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID) + ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) + ./datagen -g6GB -P99 | ./zstd32 -vq | ./zstd32 -d > $(VOID) test-fullbench: fullbench datagen ./fullbench -i1 From 17867ced036e4f184a99abcd2dc0e510fe764af9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jul 2015 00:14:27 -0800 Subject: [PATCH 17/22] Fixed minor clang warnings --- lib/zstd.c | 2 +- programs/Makefile | 3 ++- programs/zstdcli.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index 277307fbc..880bd8eac 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -456,7 +456,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); + return (__builtin_clz((U32)val) >> 3); # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } diff --git a/programs/Makefile b/programs/Makefile index 0526c36c8..a6304db55 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -36,7 +36,8 @@ DESTDIR?= PREFIX ?= /usr CFLAGS ?= -O3 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\" -FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +LDFLAGS = -I../lib +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) BINDIR=$(PREFIX)/bin MANDIR=$(PREFIX)/share/man/man1 diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 11a5ec89c..7fa85c855 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -174,7 +174,7 @@ int main(int argc, char** argv) char extension[] = ZSTD_EXTENSION; /* Pick out basename component. Don't rely on stdlib because of conflicting behaviour. */ - for (i = strlen(programName); i > 0; i--) + for (i = (int)strlen(programName); i > 0; i--) { if (programName[i] == '/') { From 674d91b3da7f98bbcd23a8c8b88b220663e6d1a6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jul 2015 00:36:49 -0800 Subject: [PATCH 18/22] Added sanitize=undefine test --- .travis.yml | 1 + Makefile | 4 ++-- lib/zstd.c | 10 ++++++---- programs/Makefile | 6 +++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 62e818929..463cb6efc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ env: - ZSTD_TRAVIS_CI_ENV=clangtest - ZSTD_TRAVIS_CI_ENV=gpptest - ZSTD_TRAVIS_CI_ENV=armtest + - ZSTD_TRAVIS_CI_ENV=sanitize matrix: fast_finish: true diff --git a/Makefile b/Makefile index 47d727cc9..8049649d6 100644 --- a/Makefile +++ b/Makefile @@ -100,9 +100,9 @@ gpptest: clean armtest: clean cd $(ZSTDDIR); $(MAKE) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" - cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror" + cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" sanitize: clean - $(MAKE) test CC=clang CPPFLAGS="-g -fsanitize=undefined" FUZZER_TIME="-T1mn" NB_LOOPS=-i1 + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" endif diff --git a/lib/zstd.c b/lib/zstd.c index 880bd8eac..32cb89298 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -178,11 +178,13 @@ static unsigned ZSTD_isLittleEndian(void) return one.c[0]; } -static U16 ZSTD_read16(const void* p) { return *(const U16*)p; } +static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; } -static U32 ZSTD_read32(const void* p) { return *(const U32*)p; } +static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; } -static size_t ZSTD_read_ARCH(const void* p) { return *(const size_t*)p; } +static U32 ZSTD_read64(const void* p) { U64 r; memcpy(&r, p, sizeof(r)); return r; } + +static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; } static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } @@ -911,7 +913,7 @@ static const U64 prime7bytes = 58295818150454627ULL; //static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); } -static U32 ZSTD_hashPtr(const void* p) { return ( (*(const U64*)p * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; } +static U32 ZSTD_hashPtr(const void* p) { return ( (ZSTD_read64(p) * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); } diff --git a/programs/Makefile b/programs/Makefile index a6304db55..640446225 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -116,11 +116,11 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/unzstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/unzstd.1 @echo zstd programs successfully uninstalled -test: test-zstd test-fullbench test-fuzzer test-mem +test: test-zstd test-fullbench test-fuzzer test32: test-zstd32 test-fullbench32 test-fuzzer32 -test-all: test test32 +test-all: test test32 test-mem test-zstd: zstd datagen ./datagen | ./zstd -v | ./zstd -d > $(VOID) @@ -151,7 +151,7 @@ test-mem: zstd datagen fuzzer fullbench valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null ./datagen -g16KB > tmp valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null - ./datagen -g128MB > tmp + ./datagen -g64MB > tmp valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null @rm tmp valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1 From dc6c1d920a1f48f5c307fccc4ca8899315e3c0e2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jul 2015 00:44:22 -0800 Subject: [PATCH 19/22] Fixed clang warning --- lib/zstd.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index 32cb89298..267681410 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -182,7 +182,7 @@ static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); retu static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; } -static U32 ZSTD_read64(const void* p) { U64 r; memcpy(&r, p, sizeof(r)); return r; } +static U64 ZSTD_read64(const void* p) { U64 r; memcpy(&r, p, sizeof(r)); return r; } static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; } @@ -925,7 +925,6 @@ static const BYTE* ZSTD_updateMatch(U32* table, const BYTE* p, const BYTE* start U32 h = ZSTD_hashPtr(p); const BYTE* r; r = table[h] + start; - //table[h] = (U32)(p - start); ZSTD_addPtr(table, p, start); return r; } From 858db1291ff1d769287ee2e703e69109b0e1e36e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jul 2015 01:02:04 -0800 Subject: [PATCH 20/22] Fixed : ZSTD_compress() corrently prevents buffer overflow --- lib/zstd.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index 267681410..f6140f750 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1089,6 +1089,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, con const U32 updateRate = 2 * BLOCKSIZE; /* Init */ + if (maxDstSize < ZSTD_compressBound(srcSize) - 4 /*header size*/) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; if (ctx->base==NULL) ctx->base = (const BYTE*)src, ctx->current=0, ctx->nextUpdate = g_maxDistance; if (src != ctx->base + ctx->current) /* not contiguous */ @@ -1119,7 +1120,6 @@ size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, con } /* compress */ - if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize); if (cSize == 0) { @@ -1165,6 +1165,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize) static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; BYTE* op = ostart; /* Header */ @@ -1177,7 +1178,7 @@ static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, co /* Compression */ { - size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize); + size_t cSize = ZSTD_compressContinue(ctx, op, oend-op, src, srcSize); if (ZSTD_isError(cSize)) return cSize; op += cSize; maxDstSize -= cSize; @@ -1185,7 +1186,7 @@ static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, co /* Close frame */ { - size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize); + size_t endSize = ZSTD_compressEnd(ctx, op, oend-op); if(ZSTD_isError(endSize)) return endSize; op += endSize; } From 968f275981448de16810073d3d7687dff332462d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jul 2015 16:54:25 -0800 Subject: [PATCH 21/22] Changed allocation strategy to reduce stack usage of ZSTD_compressSequences() --- lib/zstd.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index f6140f750..5e4b2fa52 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -146,7 +146,7 @@ static const U32 g_maxDistance = 4 * BLOCKSIZE; static const U32 g_maxLimit = 1 GB; static const U32 g_searchStrength = 8; -#define WORKPLACESIZE (BLOCKSIZE*11/4) +#define WORKPLACESIZE (BLOCKSIZE*3) #define MINMATCH 4 #define MLbits 7 #define LLbits 6 @@ -292,6 +292,8 @@ typedef struct { void* buffer; U32* offsetStart; U32* offset; + BYTE* offCodeStart; + BYTE* offCode; BYTE* litStart; BYTE* lit; BYTE* litLengthStart; @@ -332,7 +334,8 @@ ZSTD_Cctx* ZSTD_createCCtx(void) if (ctx==NULL) return NULL; ctx->seqStore.buffer = malloc(WORKPLACESIZE); ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer); - ctx->seqStore.litStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2)); + ctx->seqStore.offCodeStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2)); + ctx->seqStore.litStart = ctx->seqStore.offCodeStart + (BLOCKSIZE>>2); ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE; ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2); ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2); @@ -680,7 +683,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const BYTE* op_matchLength = seqStorePtr->matchLength; const size_t nbSeq = op_litLength - op_litLength_start; BYTE* op; - BYTE offsetBits_start[BLOCKSIZE / 4]; + BYTE* offsetBits_start = seqStorePtr->offCodeStart; BYTE* offsetBitsPtr = offsetBits_start; const size_t minGain = ZSTD_minGain(srcSize); const size_t maxCSize = srcSize - minGain; @@ -1010,13 +1013,12 @@ size_t ZSTD_compressBegin(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize) } -/* this should be auto-vectorized by compiler */ static void ZSTD_scaleDownCtx(void* cctx, const U32 limit) { cctxi_t* ctx = (cctxi_t*) cctx; int i; -#if defined(__AVX2__) /* */ +#if defined(__AVX2__) /* AVX2 version */ __m256i* h = ctx->hashTable; const __m256i limit8 = _mm256_set1_epi32(limit); @@ -1028,6 +1030,7 @@ static void ZSTD_scaleDownCtx(void* cctx, const U32 limit) _mm256_storeu_si256((__m256i*)(h+i), src); } #else + /* this should be auto-vectorized by compiler */ U32* h = ctx->hashTable; for (i=0; i */ +#if defined(__AVX2__) /* AVX2 version */ { __m256i* h = ctx->hashTable; @@ -1068,6 +1070,7 @@ static void ZSTD_limitCtx(void* cctx, const U32 limit) } } #else + /* this should be auto-vectorized by compiler */ { U32* h = (U32*)(ctx->hashTable); for (i=0; ibase==NULL) ctx->base = (const BYTE*)src, ctx->current=0, ctx->nextUpdate = g_maxDistance; if (src != ctx->base + ctx->current) /* not contiguous */ From 8b48b248217ae64e125a7c667d8e758143a9a6a1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jul 2015 17:26:17 -0800 Subject: [PATCH 22/22] Fixed minor visual analyzer warning --- programs/Makefile | 2 +- programs/datagen.c | 13 +++++++++---- programs/datagen.h | 12 ++++++------ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 640446225..20205b90a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -59,7 +59,7 @@ endif default: zstd -all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 +all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen zstd: $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c $(CC) $(FLAGS) $^ -o $@$(EXT) diff --git a/programs/datagen.c b/programs/datagen.c index 50daaa077..fa0e62a1f 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -130,7 +130,6 @@ static BYTE RDG_genChar(U32* seed, const litDistribTable lt) } -#define RDG_DICTSIZE (32 KB) #define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) #define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) @@ -140,7 +139,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match size_t pos = prefixSize; U32* seed = seedPtr; - /* special case */ + /* special case : sparse content */ while (matchProba >= 1.0) { size_t size0 = RDG_rand(seed) & 3; @@ -154,6 +153,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match memset(buffPtr+pos, 0, size0); pos += size0; buffPtr[pos-1] = RDG_genChar(seed, lt); + return; } /* init */ @@ -198,20 +198,22 @@ void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba } +#define RDG_DICTSIZE (32 KB) #define RDG_BLOCKSIZE (128 KB) void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed) { - BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; + BYTE* buff = (BYTE*)malloc(RDG_DICTSIZE + RDG_BLOCKSIZE); U64 total = 0; size_t genBlockSize = RDG_BLOCKSIZE; litDistribTable lt; /* init */ + if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); } if (litProba==0.0) litProba = matchProba / 4.5; RDG_fillLiteralDistrib(lt, litProba); SET_BINARY_MODE(stdout); - /* Generate dict */ + /* Generate initial dict */ RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed); /* Generate compressible data */ @@ -224,4 +226,7 @@ void RDG_genOut(unsigned long long size, double matchProba, double litProba, uns /* update dict */ memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); } + + // cleanup + free(buff); } diff --git a/programs/datagen.h b/programs/datagen.h index 631d1463d..89482dc2c 100644 --- a/programs/datagen.h +++ b/programs/datagen.h @@ -28,13 +28,13 @@ void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); -/* RDG_genOut - Generate 'size' bytes of compressible data into stdout. +/* RDG_genBuffer + Generate 'size' bytes of compressible data into 'buffer'. Compressibility can be controlled using 'matchProba'. - 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. - Generated data can be selected using 'seed'. + 'LitProba' is optional, and affect variability of individual bytes. If litProba==0.0, default value is used. + Generated data pattern can be modified using different 'seed'. If (matchProba, litProba and seed) are equal, the function always generate the same content. - RDG_genBuffer - Same as RDG_genOut, but generate data into provided buffer + RDG_genOut + Same as RDG_genBuffer, but generate data towards stdout */