Merge pull request #295 from Cyan4973/dev

v0.8.1

.gitignore

@@ -18,29 +18,15 @@ zstd
 *.out
 *.app
 
-# Visual C++
-projects/VS2008
-projects/VS2010
-projects/VS2012
-projects/VS2013
-projects/VS2015
-build/bin
-
-# IDEA solution files
-*.idea
+# Test artefacts
+tmp*
+dictionary
 
 # Other files
 .directory
 _codelite/
 _zstdbench/
 .clang_complete
-
-# CMake
-projects/cmake/
-
-# Test artefacts
-tmp*
-dictionary
-
-# tmp files
+*.idea
 *.swp
+.DS_Store

.travis.yml

@@ -6,80 +6,81 @@ matrix:
     # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
-    - os: linux
-      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
-    - os: linux
-      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest"
     - os: linux
       sudo: false
       env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
     - os: linux
       sudo: false
       env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
-    - os: linux
-      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
     # Standard Ubuntu 12.04 LTS Server Edition 64 bit
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+            - g++-multilib 
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make armtest"
+      addons:
+        apt:
+          packages:
+            - gcc-arm-linux-gnueabi
+            - libc6-dev-armel-cross
+            - linux-libc-dev-armel-cross
+            - binfmt-support
+            - qemu
+            - qemu-user-static
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
-    - os: linux
-      sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
-    - os: linux
-      sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
-    - os: linux
-      sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
-    - os: linux
-      sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C programs test32"
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+            - gcc-multilib
     - os: linux
       sudo: required
       env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make asan32"
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+            - gcc-multilib
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C programs valgrindTest"
+      addons:
+        apt:
+          packages:
+            - valgrind
     # Ubuntu 14.04 LTS Server Edition 64 bit
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
-    - os: linux
-      dist: trusty
-      sudo: required
-      env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
-    - os: linux
-      dist: trusty
-      sudo: required
-      env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
-    - os: linux
-      dist: trusty
-      sudo: required
-      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
+      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper && make clean && make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - gcc-multilib
+            - gcc-5
+            - gcc-5-multilib
+            - gcc-6
+            - gcc-6-multilib
     # OS X Mavericks
     - os: osx
-      env: PLATFORM="OS X Mavericks" CMD="make travis-install"
-    - os: osx
-      env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
-    - os: osx
-      env: PLATFORM="OS X Mavericks" CMD="make test"
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install"
   exclude:
     - compiler: gcc
 

Makefile

@@ -120,7 +120,7 @@ asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
 
 msan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory"   # datagen.c used to fail this test for no obvious reason
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer"   # datagen.c fails this test for no obvious reason
 
 asan32: clean
 	$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
@@ -174,37 +174,3 @@ bmi32test: clean
 staticAnalyze: clean
 	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
 endif
-
-
-#------------------------------------------------------------------------
-# for Travis CI
-#------------------------------------------------------------------------
-libc6install:
-	sudo apt-get install -y -qq libc6-dev-i386
-
-gppinstall:
-	sudo apt-get install -y -qq g++-multilib
-
-gcc5install:
-	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-	sudo apt-get update -y -qq
-	sudo apt-get install -y -qq gcc-5-multilib
-
-gcc6install:
-	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-	sudo apt-get update -y -qq
-	sudo apt-get install -y -qq gcc-6-multilib
-
-arminstall: clean
-	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
-
-ppcinstall: clean
-	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
-
-ppc64install: clean   # compilation works but cannot be executed
-	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
-	update-binfmts --displ
-
-valgrindinstall:
-	echo sudo apt-get install -y -qq valgrind
-	sudo apt-get install -y -qq valgrind

NEWS

@@ -1,3 +1,12 @@
+v0.8.1
+New streaming API
+Changed : --ultra now enables levels beyond 19
+Changed : -i# now selects benchmark time in second
+Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
+Fixed : speed regression on specific patterns (#272)
+Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
+Fixed : ICC compilation, by Przemyslaw Skibinski
+
 v0.8.0
 Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
 New : Build on FreeBSD and DragonFly, thanks to JrMarino

appveyor.yml

@@ -59,29 +59,51 @@ build_script:
     )
   - if [%COMPILER%]==[visual] (
       ECHO *** &&
-      ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
       ECHO *** &&
-      msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
       COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
       COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
       COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
       COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
       COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
       COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\
     )

examples/.gitignore

@@ -3,6 +3,8 @@ simple_compression
 simple_decompression
 dictionary_compression
 dictionary_decompression
+streaming_compression
+streaming_decompression
 
 #test artefact
 tmp*

examples/Makefile

@@ -31,7 +31,8 @@ LDFLAGS+= -lzstd
 default: all
 
 all: simple_compression simple_decompression \
-	dictionary_compression dictionary_decompression
+	dictionary_compression dictionary_decompression \
+	streaming_compression streaming_decompression
 
 simple_compression : simple_compression.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -45,15 +46,29 @@ dictionary_compression : dictionary_compression.c
 dictionary_decompression : dictionary_decompression.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+streaming_compression : streaming_compression.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+streaming_decompression : streaming_decompression.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
 clean:
 	@rm -f core *.o tmp* result* *.zst \
         simple_compression simple_decompression \
-		dictionary_compression dictionary_decompression
+		dictionary_compression dictionary_decompression \
+		streaming_compression streaming_decompression
 	@echo Cleaning completed
 
 test: all
 	cp README.md tmp
+	@echo starting simple compression
 	./simple_compression tmp
-	@echo starting simple_decompression
 	./simple_decompression tmp.zst
+	./streaming_decompression tmp.zst
+	@echo starting streaming compression
+	./streaming_compression tmp
+	./streaming_decompression tmp.zst
+	@echo starting dictionary compression
+	./dictionary_compression tmp README.md
+	./dictionary_decompression tmp.zst README.md
 	@echo tests completed

examples/dictionary_compression.c

@@ -31,7 +31,7 @@
 #include <zstd.h>      // presumes zstd library is installed
 
 
-static off_t fsize_X(const char *filename)
+static off_t fsize_orDie(const char *filename)
 {
     struct stat st;
     if (stat(filename, &st) == 0) return st.st_size;
@@ -40,7 +40,7 @@ static off_t fsize_X(const char *filename)
     exit(1);
 }
 
-static FILE* fopen_X(const char *filename, const char *instruction)
+static FILE* fopen_orDie(const char *filename, const char *instruction)
 {
     FILE* const inFile = fopen(filename, instruction);
     if (inFile) return inFile;
@@ -49,20 +49,20 @@ static FILE* fopen_X(const char *filename, const char *instruction)
     exit(2);
 }
 
-static void* malloc_X(size_t size)
+static void* malloc_orDie(size_t size)
 {
     void* const buff = malloc(size);
     if (buff) return buff;
     /* error */
-    perror(NULL);
+    perror("malloc");
     exit(3);
 }
 
-static void* loadFile_X(const char* fileName, size_t* size)
+static void* loadFile_orDie(const char* fileName, size_t* size)
 {
-    off_t const buffSize = fsize_X(fileName);
-    FILE* const inFile = fopen_X(fileName, "rb");
-    void* const buffer = malloc_X(buffSize);
+    off_t const buffSize = fsize_orDie(fileName);
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
     size_t const readSize = fread(buffer, 1, buffSize, inFile);
     if (readSize != (size_t)buffSize) {
         fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
@@ -73,9 +73,9 @@ static void* loadFile_X(const char* fileName, size_t* size)
     return buffer;
 }
 
-static void saveFile_X(const char* fileName, const void* buff, size_t buffSize)
+static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
 {
-    FILE* const oFile = fopen_X(fileName, "wb");
+    FILE* const oFile = fopen_orDie(fileName, "wb");
     size_t const wSize = fwrite(buff, 1, buffSize, oFile);
     if (wSize != (size_t)buffSize) {
         fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
@@ -89,23 +89,27 @@ static void saveFile_X(const char* fileName, const void* buff, size_t buffSize)
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
-static const ZSTD_CDict* createDict(const char* dictFileName)
+static ZSTD_CDict* createCDict_orDie(const char* dictFileName)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_X(dictFileName, &dictSize);
-    const ZSTD_CDict* const ddict = ZSTD_createCDict(dictBuffer, dictSize, 3);
+    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 3);
+    if (!cdict) {
+        fprintf(stderr, "ZSTD_createCDict error \n");
+        exit(7);
+    }
     free(dictBuffer);
-    return ddict;
+    return cdict;
 }
 
 
 static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
 {
     size_t fSize;
-    void* const fBuff = loadFile_X(fname, &fSize);
+    void* const fBuff = loadFile_orDie(fname, &fSize);
     size_t const cBuffSize = ZSTD_compressBound(fSize);
-    void* const cBuff = malloc_X(cBuffSize);
+    void* const cBuff = malloc_orDie(cBuffSize);
 
     ZSTD_CCtx* const cctx = ZSTD_createCCtx();
     size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
@@ -114,7 +118,7 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
         exit(7);
     }
 
-    saveFile_X(oname, cBuff, cSize);
+    saveFile_orDie(oname, cBuff, cSize);
 
     /* success */
     printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
@@ -125,11 +129,11 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
 }
 
 
-static char* createOutFilename(const char* filename)
+static char* createOutFilename_orDie(const char* filename)
 {
     size_t const inL = strlen(filename);
     size_t const outL = inL + 5;
-    void* outSpace = malloc_X(outL);
+    void* outSpace = malloc_orDie(outL);
     memset(outSpace, 0, outL);
     strcat(outSpace, filename);
     strcat(outSpace, ".zst");
@@ -149,15 +153,17 @@ int main(int argc, const char** argv)
 
     /* load dictionary only once */
     const char* const dictName = argv[argc-1];
-    const ZSTD_CDict* const dictPtr = createDict(dictName);
+    ZSTD_CDict* const dictPtr = createCDict_orDie(dictName);
 
     int u;
     for (u=1; u<argc-1; u++) {
         const char* inFilename = argv[u];
-        char* const outFilename = createOutFilename(inFilename);
+        char* const outFilename = createOutFilename_orDie(inFilename);
         compress(inFilename, outFilename, dictPtr);
         free(outFilename);
     }
 
+    ZSTD_freeCDict(dictPtr);
     printf("All %u files compressed. \n", argc-2);
+    return 0;
 }

examples/dictionary_decompression.c

@@ -31,41 +31,41 @@
 #include <zstd.h>      // presumes zstd library is installed
 
 
-static off_t fsize_X(const char *filename)
+static off_t fsize_orDie(const char *filename)
 {
     struct stat st;
     if (stat(filename, &st) == 0) return st.st_size;
     /* error */
-    printf("stat: %s : %s \n", filename, strerror(errno));
+    perror(filename);
     exit(1);
 }
 
-static FILE* fopen_X(const char *filename, const char *instruction)
+static FILE* fopen_orDie(const char *filename, const char *instruction)
 {
     FILE* const inFile = fopen(filename, instruction);
     if (inFile) return inFile;
     /* error */
-    printf("fopen: %s : %s \n", filename, strerror(errno));
+    perror(filename);
     exit(2);
 }
 
-static void* malloc_X(size_t size)
+static void* malloc_orDie(size_t size)
 {
     void* const buff = malloc(size);
     if (buff) return buff;
     /* error */
-    printf("malloc: %s \n", strerror(errno));
+    perror("malloc");
     exit(3);
 }
 
-static void* loadFile_X(const char* fileName, size_t* size)
+static void* loadFile_orDie(const char* fileName, size_t* size)
 {
-    off_t const buffSize = fsize_X(fileName);
-    FILE* const inFile = fopen_X(fileName, "rb");
-    void* const buffer = malloc_X(buffSize);
+    off_t const buffSize = fsize_orDie(fileName);
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
     size_t const readSize = fread(buffer, 1, buffSize, inFile);
     if (readSize != (size_t)buffSize) {
-        printf("fread: %s : %s \n", fileName, strerror(errno));
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
         exit(4);
     }
     fclose(inFile);
@@ -75,12 +75,13 @@ static void* loadFile_X(const char* fileName, size_t* size)
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
-static const ZSTD_DDict* createDict(const char* dictFileName)
+static ZSTD_DDict* createDict_orDie(const char* dictFileName)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_X(dictFileName, &dictSize);
-    const ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
+    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
+    if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
     free(dictBuffer);
     return ddict;
 }
@@ -89,19 +90,19 @@ static const ZSTD_DDict* createDict(const char* dictFileName)
 static void decompress(const char* fname, const ZSTD_DDict* ddict)
 {
     size_t cSize;
-    void* const cBuff = loadFile_X(fname, &cSize);
+    void* const cBuff = loadFile_orDie(fname, &cSize);
     unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
     if (rSize==0) {
-        printf("%s : original size unknown \n", fname);
-        exit(5);
+        fprintf(stderr, "%s : original size unknown \n", fname);
+        exit(6);
     }
-    void* const rBuff = malloc_X(rSize);
+    void* const rBuff = malloc_orDie(rSize);
 
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
     size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
 
     if (dSize != rSize) {
-        printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
+        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
         exit(7);
     }
 
@@ -127,10 +128,12 @@ int main(int argc, const char** argv)
 
     /* load dictionary only once */
     const char* const dictName = argv[argc-1];
-    const ZSTD_DDict* const dictPtr = createDict(dictName);
+    ZSTD_DDict* const dictPtr = createDict_orDie(dictName);
 
     int u;
     for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
 
+    ZSTD_freeDDict(dictPtr);
     printf("All %u files correctly decoded (in memory) \n", argc-2);
+    return 0;
 }

examples/streaming_compression.c

@@ -0,0 +1,143 @@
+/*
+  Streaming compression
+  Educational program using zstd library
+  Copyright (C) Yann Collet 2016
+
+  GPL v2 License
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+  You can contact the author at :
+  - zstd homepage : http://www.zstd.net/
+*/
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // fprintf, perror, feof
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc:");
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) return 0;
+    /* error */
+    perror("fclose");
+    exit(6);
+}
+
+
+static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+    size_t const buffInSize = ZSTD_CStreamInSize();;
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    size_t const buffOutSize = ZSTD_CStreamOutSize();;
+    void*  const buffOut = malloc_orDie(buffOutSize);
+    size_t read, toRead = buffInSize;
+
+    ZSTD_CStream* const cstream = ZSTD_createCStream();
+    if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
+    size_t const initResult = ZSTD_initCStream(cstream, cLevel);
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); }
+
+    while( (read = fread_orDie(buffIn, toRead, fin)) ) {
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        while (input.pos < input.size) {
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            toRead = ZSTD_compressStream(cstream, &output , &input);
+            fwrite_orDie(buffOut, output.pos, fout);
+        }
+    }
+
+    ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+    size_t const remainingToFlush = ZSTD_endStream(cstream, &output);
+    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); }
+    fwrite_orDie(buffOut, output.pos, fout);
+
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+static const char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (const char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+    const char* const inFilename = argv[1];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const outFilename = createOutFilename_orDie(inFilename);
+    compressFile_orDie(inFilename, outFilename, 1);
+
+    return 0;
+}

examples/streaming_decompression.c

@@ -0,0 +1,117 @@
+/*
+  Streaming compression
+  Educational program using zstd library
+  Copyright (C) Yann Collet 2016
+
+  GPL v2 License
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+  You can contact the author at :
+  - zstd homepage : http://www.zstd.net/
+*/
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // fprintf, perror, feof
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc:");
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(4);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) return 0;
+    /* error */
+    perror("fclose");
+    exit(6);
+}
+
+
+static void decompressFile_orDie(const char* fname)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    size_t const buffInSize = ZSTD_DStreamInSize();;
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    size_t const buffOutSize = ZSTD_DStreamOutSize();;
+    void*  const buffOut = malloc_orDie(buffOutSize);
+    size_t read, toRead = buffInSize;
+
+    ZSTD_DStream* const dstream = ZSTD_createDStream();
+    if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
+    size_t const initResult = ZSTD_initDStream(dstream);
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error \n"); exit(11); }
+
+    while( (read = fread_orDie(buffIn, toRead, fin)) ) {
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        while (input.pos < input.size) {
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            toRead = ZSTD_decompressStream(dstream, &output , &input);
+            /* note : data is just "sinked" into buffOut
+               a more complete example would write it to disk or stdout */
+        }
+    }
+
+    fclose_orDie(fin);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+    const char* const inFilename = argv[1];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    decompressFile_orDie(inFilename);
+    printf("%s correctly decoded (in memory). \n", inFilename);
+
+    return 0;
+}

lib/Makefile

@@ -46,7 +46,7 @@ PREFIX ?= /usr/local
 LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR=$(PREFIX)/include
 
-CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=ZSTD_
+CPPFLAGS= -I. -I./common
 CFLAGS ?= -O3
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef

lib/README.md

@@ -37,15 +37,18 @@ Other optional functionalities provided are :
 
 - `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`.
               This module also depends on `common/` and `decompress/` .
-              Note that it's required to compile the library with `ZSTD_LEGACY_SUPPORT = 1` .
+              Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` .
               The main API can be consulted in `legacy/zstd_legacy.h`.
-              Advanced API from each version can be found in its relevant header file.
-              For example, advanced API for version `v0.4` is in `zstd_v04.h` .
+              Advanced API from each version can be found in their relevant header file.
+              For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` .
 
 
-#### Streaming API
+#### Obsolete streaming API
 
-Streaming is currently provided by `common/zbuff.h`.
+Streaming is now provided within `zstd.h`.
+Older streaming API is still provided within `common/zbuff.h`.
+It is considered obsolete, and will be removed in a future version.
+Consider migrating towards newer streaming API.
 
 
 #### Miscellaneous
@@ -55,3 +58,4 @@ The other files are not source code. There are :
  - LICENSE : contains the BSD license text
  - Makefile : script to compile or install zstd library (static and dynamic)
  - libzstd.pc.in : for pkg-config (`make install`)
+ - README.md : this file

lib/common/mem.h

@@ -110,7 +110,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -142,7 +142,13 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
 /* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
+	__pragma( pack(push, 1) )
+    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
+    __pragma( pack(pop) )
+#else
+    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+#endif
 
 MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
 MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }

lib/common/xxhash.c

@@ -52,7 +52,7 @@
 #ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define XXH_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define XXH_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -132,7 +132,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
 ***************************************/
 #ifndef MEM_MODULE
 # define MEM_MODULE
-# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 #   include <stdint.h>
     typedef uint8_t  BYTE;
     typedef uint16_t U16;
@@ -144,7 +144,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
     typedef unsigned short     U16;
     typedef unsigned int       U32;
     typedef   signed int       S32;
-    typedef unsigned long long U64;
+    typedef unsigned long long U64;   /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
 #  endif
 #endif
 
@@ -307,6 +307,20 @@ static const U64 PRIME64_5 =  2870177450012600261ULL;
 XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
 
 
+/* **************************
+*  Utils
+****************************/
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+
 /* ***************************
 *  Simple Hash Functions
 *****************************/
@@ -545,8 +559,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
 XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
 {
     XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
-    memset(&state, 0, sizeof(state));
-    state.seed = seed;
+    memset(&state, 0, sizeof(state)-4);   /* do not write into reserved, for future removal */
     state.v1 = seed + PRIME32_1 + PRIME32_2;
     state.v2 = seed + PRIME32_2;
     state.v3 = seed + 0;
@@ -559,8 +572,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int s
 XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
 {
     XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
-    memset(&state, 0, sizeof(state));
-    state.seed = seed;
+    memset(&state, 0, sizeof(state)-8);   /* do not write into reserved, for future removal */
     state.v1 = seed + PRIME64_1 + PRIME64_2;
     state.v2 = seed + PRIME64_2;
     state.v3 = seed + 0;
@@ -579,11 +591,12 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void
     if (input==NULL) return XXH_ERROR;
 #endif
 
-    state->total_len += len;
+    state->total_len_32 += (unsigned)len;
+    state->large_len |= (len>=16) | (state->total_len_32>=16);
 
     if (state->memsize + len < 16)  {   /* fill in tmp buffer */
         XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
-        state->memsize += (U32)len;
+        state->memsize += (unsigned)len;
         return XXH_OK;
     }
 
@@ -620,8 +633,8 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void
     }
 
     if (p < bEnd) {
-        XXH_memcpy(state->mem32, p, bEnd-p);
-        state->memsize = (int)(bEnd-p);
+        XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
     }
 
     return XXH_OK;
@@ -645,13 +658,13 @@ FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess
     const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
     U32 h32;
 
-    if (state->total_len >= 16) {
+    if (state->large_len) {
         h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
     } else {
-        h32 = state->seed + PRIME32_5;
+        h32 = state->v3 /* == seed */ + PRIME32_5;
     }
 
-    h32 += (U32) state->total_len;
+    h32 += state->total_len_32;
 
     while (p+4<=bEnd) {
         h32 += XXH_readLE32(p, endian) * PRIME32_3;
@@ -737,8 +750,8 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void
     }
 
     if (p < bEnd) {
-        XXH_memcpy(state->mem64, p, bEnd-p);
-        state->memsize = (int)(bEnd-p);
+        XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
     }
 
     return XXH_OK;
@@ -774,7 +787,7 @@ FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess
         h64 = XXH64_mergeRound(h64, v3);
         h64 = XXH64_mergeRound(h64, v4);
     } else {
-        h64  = state->seed + PRIME64_5;
+        h64  = state->v3 + PRIME64_5;
     }
 
     h64 += (U64) state->total_len;

lib/common/xxhash.h

@@ -71,6 +71,10 @@ XXH32        6.8 GB/s            6.0 GB/s
 extern "C" {
 #endif
 
+#ifndef XXH_NAMESPACE
+#  define XXH_NAMESPACE ZSTD_  /* Zstandard specific */
+#endif
+
 
 /* ****************************
 *  Definitions
@@ -82,18 +86,21 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 /* ****************************
 *  API modifier
 ******************************/
-/*!XXH_PRIVATE_API
-*  Transforms all publics symbols within `xxhash.c` into private ones.
-*  Methodology :
-*  instead of : #include "xxhash.h"
-*  do :
+/** XXH_PRIVATE_API
+*   This is useful if you want to include xxhash functions in `static` mode
+*   in order to inline them, and remove their symbol from the public list.
+*   Methodology :
 *     #define XXH_PRIVATE_API
-*     #include "xxhash.c"   // note the .c , instead of .h
-*  also : don't compile and link xxhash.c separately
+*     #include "xxhash.h"
+*   `xxhash.c` is automatically included.
+*   It's not useful to compile and link it as a separate module anymore.
 */
 #ifdef XXH_PRIVATE_API
+#  ifndef XXH_STATIC_LINKING_ONLY
+#    define XXH_STATIC_LINKING_ONLY
+#  endif
 #  if defined(__GNUC__)
-#    define XXH_PUBLIC_API static __attribute__((unused))
+#    define XXH_PUBLIC_API static __inline __attribute__((unused))
 #  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #    define XXH_PUBLIC_API static inline
 #  elif defined(_MSC_VER)
@@ -103,17 +110,17 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 #  endif
 #else
 #  define XXH_PUBLIC_API   /* do nothing */
-#endif
+#endif /* XXH_PRIVATE_API */
 
 /*!XXH_NAMESPACE, aka Namespace Emulation :
 
 If you want to include _and expose_ xxHash functions from within your own library,
 but also want to avoid symbol collisions with another library which also includes xxHash,
 
-you can use XXH_NAMESPACE, to automatically prefix any public symbol from `xxhash.c`
+you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
 with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values).
 
-Note that no change is required within the calling program as long as it also includes `xxhash.h` :
+Note that no change is required within the calling program as long as it includes `xxhash.h` :
 regular symbol name will be automatically translated by this header.
 */
 #ifdef XXH_NAMESPACE
@@ -132,6 +139,12 @@ regular symbol name will be automatically translated by this header.
 #  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
 #  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
 #  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
 #endif
 
 
@@ -140,7 +153,7 @@ regular symbol name will be automatically translated by this header.
 ***************************************/
 #define XXH_VERSION_MAJOR    0
 #define XXH_VERSION_MINOR    6
-#define XXH_VERSION_RELEASE  0
+#define XXH_VERSION_RELEASE  2
 #define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
 XXH_PUBLIC_API unsigned XXH_versionNumber (void);
 
@@ -163,7 +176,7 @@ XXH32() :
 XXH64() :
     Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
     "seed" can be used to alter the result predictably.
-    This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
+    This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
 */
 
 
@@ -173,8 +186,7 @@ XXH64() :
 typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
 typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
 
-/*! Dynamic allocation of states
-    Compatible with dynamic libraries */
+/*! State allocation, compatible with dynamic libraries */
 
 XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
 XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
@@ -193,28 +205,40 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned lon
 XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
 XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
 
-/*!
-These functions generate the xxHash of an input provided in multiple segments,
-as opposed to provided as a single block.
+/*
+These functions generate the xxHash of an input provided in multiple segments.
+Note that, for small input, they are slower than single-call functions, due to state management.
+For small input, prefer `XXH32()` and `XXH64()` .
 
-XXH state must first be allocated, using either static or dynamic method provided above.
+XXH state must first be allocated, using XXH*_createState() .
 
-Start a new hash by initializing state with a seed, using XXHnn_reset().
+Start a new hash by initializing state with a seed, using XXH*_reset().
 
-Then, feed the hash state by calling XXHnn_update() as many times as necessary.
-Obviously, input must be valid, hence allocated and read accessible.
+Then, feed the hash state by calling XXH*_update() as many times as necessary.
+Obviously, input must be allocated and read accessible.
 The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
 
-Finally, a hash value can be produced anytime, by using XXHnn_digest().
+Finally, a hash value can be produced anytime, by using XXH*_digest().
 This function returns the nn-bits hash as an int or long long.
 
 It's still possible to continue inserting input into the hash state after a digest,
-and later on generate some new hashes, by calling again XXHnn_digest().
+and generate some new hashes later on, by calling again XXH*_digest().
 
 When done, free XXH state space if it was allocated dynamically.
 */
 
 
+/* **************************
+*  Utils
+****************************/
+#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))   /* ! C99 */
+#  define restrict   /* disable restrict */
+#endif
+
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state);
+
+
 /* **************************
 *  Canonical representation
 ****************************/
@@ -227,43 +251,55 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t
 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
 
-/*! Default result type for XXH functions are primitive unsigned 32 and 64 bits.
-*   The canonical representation uses human-readable write convention, aka big-endian (large digits first).
-*   These functions allow transformation of hash result into and from its canonical format.
-*   This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+*  These functions allow transformation of hash result into and from its canonical format.
+*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
 */
 
 
 #ifdef XXH_STATIC_LINKING_ONLY
 
-/* This part contains definition which shall only be used with static linking.
-   The prototypes / types defined here are not guaranteed to remain stable.
-   They could change in a future version, becoming incompatible with a different version of the library */
+/* ================================================================================================
+   This section contains definitions which are not guaranteed to remain stable.
+   They may change in future versions, becoming incompatible with a different version of the library.
+   They shall only be used with static linking.
+   Never use these definitions in association with dynamic linking !
+=================================================================================================== */
+
+/* These definitions are only meant to allow allocation of XXH state
+   statically, on stack, or in a struct for example.
+   Do not use members directly. */
 
    struct XXH32_state_s {
-       unsigned long long total_len;
-       unsigned seed;
+       unsigned total_len_32;
+       unsigned large_len;
        unsigned v1;
        unsigned v2;
        unsigned v3;
        unsigned v4;
        unsigned mem32[4];   /* buffer defined as U32 for alignment */
        unsigned memsize;
+       unsigned reserved;   /* never read nor write, will be removed in a future version */
    };   /* typedef'd to XXH32_state_t */
 
    struct XXH64_state_s {
        unsigned long long total_len;
-       unsigned long long seed;
        unsigned long long v1;
        unsigned long long v2;
        unsigned long long v3;
        unsigned long long v4;
        unsigned long long mem64[4];   /* buffer defined as U64 for alignment */
        unsigned memsize;
+       unsigned reserved[2];          /* never read nor write, will be removed in a future version */
    };   /* typedef'd to XXH64_state_t */
 
 
-#endif
+#  ifdef XXH_PRIVATE_API
+#    include "xxhash.c"   /* include xxhash functions as `static`, for inlining */
+#  endif
+
+#endif /* XXH_STATIC_LINKING_ONLY */
 
 
 #if defined (__cplusplus)

lib/common/zstd_internal.h

@@ -67,9 +67,10 @@
 #define ZSTD_OPT_NUM    (1<<12)
 #define ZSTD_DICT_MAGIC  0xEC30A437   /* v0.7+ */
 
-#define ZSTD_REP_NUM    3                 /* number of repcodes */
-#define ZSTD_REP_CHECK  (ZSTD_REP_NUM-0)  /* number of repcodes to check by the optimal parser */
-#define ZSTD_REP_MOVE   (ZSTD_REP_NUM-1)
+#define ZSTD_REP_NUM      3                 /* number of repcodes */
+#define ZSTD_REP_CHECK    (ZSTD_REP_NUM)    /* number of repcodes to check by the optimal parser */
+#define ZSTD_REP_MOVE     (ZSTD_REP_NUM-1)
+#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
 static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 
 #define KB *(1 <<10)
@@ -154,6 +155,16 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
     while (op < oend);
 }
 
+MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd)   /* should be faster for decoding, but strangely, not verified on all platform */
+{
+    const BYTE* ip = (const BYTE*)src;
+    BYTE* op = (BYTE*)dst;
+    BYTE* const oend = (BYTE*)dstEnd;
+    do
+        COPY8(op, ip)
+    while (op < oend);
+}
+
 
 /*-*******************************************
 *  Private interfaces

lib/compress/zstd_compress.c

@@ -32,7 +32,7 @@
 */
 
 
-/* *******************************************************
+/*-*******************************************************
 *  Compiler specifics
 *********************************************************/
 #ifdef _MSC_VER    /* Visual Studio */
@@ -55,7 +55,7 @@
 #include "mem.h"
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"         /* XXH_reset, update, digest */
-#define FSE_STATIC_LINKING_ONLY
+#define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
@@ -940,8 +940,8 @@ static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
 
 FORCE_INLINE
 void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
-                                 const void* src, size_t srcSize,
-                                 const U32 mls)
+                               const void* src, size_t srcSize,
+                               const U32 mls)
 {
     U32* const hashTable = cctx->hashTable;
     U32  const hBits = cctx->params.cParams.hashLog;
@@ -973,7 +973,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
         const BYTE* match = base + matchIndex;
         hashTable[h] = current;   /* update hash table */
 
-        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
+        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
             mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
             ip++;
             ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
@@ -2205,8 +2205,13 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
 static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
+    const BYTE* const base = zc->base;
+    const BYTE* const istart = (const BYTE*)src;
+    const U32 current = (U32)(istart-base);
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0;   /* don't even attempt compression below a certain srcSize */
     ZSTD_resetSeqStore(&(zc->seqStore));
+    if (current > zc->nextToUpdate + 384)
+        zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));   /* update tree not updated after finding very long rep matches */
     blockCompressor(zc, src, srcSize);
     return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
 }
@@ -2244,6 +2249,21 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
         if (remaining < blockSize) blockSize = remaining;
 
+        /* preemptive overflow correction */
+        if (cctx->lowLimit > (1<<30)) {
+            U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt);
+            U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
+            U32 const newLowLimit = cctx->lowLimit & chainMask;   /* preserve position % chainSize */
+            U32 const correction = cctx->lowLimit - newLowLimit;
+            ZSTD_reduceIndex(cctx, correction);
+            cctx->base += correction;
+            cctx->dictBase += correction;
+            cctx->lowLimit = newLowLimit;
+            cctx->dictLimit -= correction;
+            if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
+            else cctx->nextToUpdate -= correction;
+        }
+
         if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
             /* enforce maxDist */
             U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
@@ -2317,7 +2337,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
 }
 
 
-static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
+static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
                               void* dst, size_t dstCapacity,
                         const void* src, size_t srcSize,
                                U32 frame, U32 lastFrameChunk)
@@ -2325,53 +2345,40 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
     const BYTE* const ip = (const BYTE*) src;
     size_t fhSize = 0;
 
-    if (zc->stage==ZSTDcs_created) return ERROR(stage_wrong);   /* missing init (ZSTD_compressBegin) */
+    if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong);   /* missing init (ZSTD_compressBegin) */
 
-    if (frame && (zc->stage==ZSTDcs_init)) {
-        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID);
+    if (frame && (cctx->stage==ZSTDcs_init)) {
+        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
         if (ZSTD_isError(fhSize)) return fhSize;
         dstCapacity -= fhSize;
         dst = (char*)dst + fhSize;
-        zc->stage = ZSTDcs_ongoing;
+        cctx->stage = ZSTDcs_ongoing;
     }
 
     /* Check if blocks follow each other */
-    if (src != zc->nextSrc) {
+    if (src != cctx->nextSrc) {
         /* not contiguous */
-        ptrdiff_t const delta = zc->nextSrc - ip;
-        zc->lowLimit = zc->dictLimit;
-        zc->dictLimit = (U32)(zc->nextSrc - zc->base);
-        zc->dictBase = zc->base;
-        zc->base -= delta;
-        zc->nextToUpdate = zc->dictLimit;
-        if (zc->dictLimit - zc->lowLimit < HASH_READ_SIZE) zc->lowLimit = zc->dictLimit;   /* too small extDict */
+        ptrdiff_t const delta = cctx->nextSrc - ip;
+        cctx->lowLimit = cctx->dictLimit;
+        cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
+        cctx->dictBase = cctx->base;
+        cctx->base -= delta;
+        cctx->nextToUpdate = cctx->dictLimit;
+        if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit;   /* too small extDict */
     }
 
-    /* preemptive overflow correction */
-    if (zc->lowLimit > (1<<30)) {
-        U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) | (zc->params.cParams.strategy == ZSTD_btopt);
-        U32 const chainMask = (1 << (zc->params.cParams.chainLog - btplus)) - 1;
-        U32 const newLowLimit = zc->lowLimit & chainMask;   /* preserve position % chainSize */
-        U32 const correction = zc->lowLimit - newLowLimit;
-        ZSTD_reduceIndex(zc, correction);
-        zc->base += correction;
-        zc->dictBase += correction;
-        zc->lowLimit = newLowLimit;
-        zc->dictLimit -= correction;
-        if (zc->nextToUpdate < correction) zc->nextToUpdate = 0;
-        else zc->nextToUpdate -= correction;
+    /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
+    if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
+        ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
+        U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
+        cctx->lowLimit = lowLimitMax;
     }
 
-    /* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
-    if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) {
-        zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
-        if (zc->lowLimit > zc->dictLimit) zc->lowLimit = zc->dictLimit;
-    }
+    cctx->nextSrc = ip + srcSize;
 
-    zc->nextSrc = ip + srcSize;
     {   size_t const cSize = frame ?
-                             ZSTD_compress_generic (zc, dst, dstCapacity, src, srcSize, lastFrameChunk) :
-                             ZSTD_compressBlock_internal (zc, dst, dstCapacity, src, srcSize);
+                             ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
+                             ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
         if (ZSTD_isError(cSize)) return cSize;
         return cSize + fhSize;
     }
@@ -2619,22 +2626,6 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
 }
 
 
-/*! ZSTD_compress_usingPreparedCCtx() :
-*   Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
-*   It avoids reloading the dictionary each time.
-*   `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
-*   Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
-static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
-                                       void* dst, size_t dstCapacity,
-                                 const void* src, size_t srcSize)
-{
-    size_t const errorCode = ZSTD_copyCCtx(cctx, preparedCCtx);
-    if (ZSTD_isError(errorCode)) return errorCode;
-
-    return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
-}
-
-
 static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
                                void* dst, size_t dstCapacity,
                          const void* src, size_t srcSize,
@@ -2730,9 +2721,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
 ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_customMem const allocator = { NULL, NULL, NULL };
-    ZSTD_parameters params;
-    memset(&params, 0, sizeof(params));
-    params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
     params.fParams.contentSizeFlag = 1;
     return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
 }
@@ -2747,14 +2736,293 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
     return 0;
 }
 
+/*! ZSTD_compress_usingCDict() :
+*   Compression using a digested Dictionary.
+*   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+*   Note that compression level is decided during dictionary creation */
 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
                                            void* dst, size_t dstCapacity,
                                      const void* src, size_t srcSize,
                                      const ZSTD_CDict* cdict)
 {
-    return ZSTD_compress_usingPreparedCCtx(cctx, cdict->refContext,
-                                           dst, dstCapacity,
-                                           src, srcSize);
+    size_t const errorCode = ZSTD_copyCCtx(cctx, cdict->refContext);
+    if (ZSTD_isError(errorCode)) return errorCode;
+
+    if (cdict->refContext->params.fParams.contentSizeFlag==1) {
+        cctx->params.fParams.contentSizeFlag = 1;
+        cctx->frameContentSize = srcSize;
+    }
+
+    return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+}
+
+
+
+/* ******************************************************************
+*  Streaming
+********************************************************************/
+
+typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
+
+struct ZSTD_CStream_s {
+    ZSTD_CCtx* zc;
+    char*  inBuff;
+    size_t inBuffSize;
+    size_t inToCompress;
+    size_t inBuffPos;
+    size_t inBuffTarget;
+    size_t blockSize;
+    char*  outBuff;
+    size_t outBuffSize;
+    size_t outBuffContentSize;
+    size_t outBuffFlushedSize;
+    ZSTD_cStreamStage stage;
+    U32    checksum;
+    U32    frameEnded;
+    ZSTD_customMem customMem;
+};   /* typedef'd to ZSTD_CStream within "zstd.h" */
+
+ZSTD_CStream* ZSTD_createCStream(void)
+{
+    return ZSTD_createCStream_advanced(defaultCustomMem);
+}
+
+ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
+{
+    ZSTD_CStream* zcs;
+
+    if (!customMem.customAlloc && !customMem.customFree)
+        customMem = defaultCustomMem;
+
+    if (!customMem.customAlloc || !customMem.customFree)
+        return NULL;
+
+    zcs = (ZSTD_CStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CStream));
+    if (zcs==NULL) return NULL;
+    memset(zcs, 0, sizeof(ZSTD_CStream));
+    memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
+    zcs->zc = ZSTD_createCCtx_advanced(customMem);
+    if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; }
+    return zcs;
+}
+
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
+{
+    if (zcs==NULL) return 0;   /* support free on NULL */
+    ZSTD_freeCCtx(zcs->zc);
+    if (zcs->inBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff);
+    if (zcs->outBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff);
+    zcs->customMem.customFree(zcs->customMem.opaque, zcs);
+    return 0;
+}
+
+
+/*======   Initialization   ======*/
+
+size_t ZSTD_CStreamInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
+
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+                                 const void* dict, size_t dictSize,
+                                 ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+    /* allocate buffers */
+    {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
+        if (zcs->inBuffSize < neededInBuffSize) {
+            zcs->inBuffSize = neededInBuffSize;
+            zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff);   /* should not be necessary */
+            zcs->inBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, neededInBuffSize);
+            if (zcs->inBuff == NULL) return ERROR(memory_allocation);
+        }
+        zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
+    }
+    if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
+        zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
+        zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff);   /* should not be necessary */
+        zcs->outBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, zcs->outBuffSize);
+        if (zcs->outBuff == NULL) return ERROR(memory_allocation);
+    }
+
+    { size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize);
+      if (ZSTD_isError(errorCode)) return errorCode; }
+
+    zcs->inToCompress = 0;
+    zcs->inBuffPos = 0;
+    zcs->inBuffTarget = zcs->blockSize;
+    zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+    zcs->stage = zcss_load;
+    zcs->checksum = params.fParams.checksumFlag > 0;
+    zcs->frameEnded = 0;
+    return 0;   /* ready to go */
+}
+
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
+{
+    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
+    return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
+}
+
+size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
+{
+    return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
+}
+
+
+/*======   Compression   ======*/
+
+typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
+
+MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+    size_t const length = MIN(dstCapacity, srcSize);
+    memcpy(dst, src, length);
+    return length;
+}
+
+static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
+                              void* dst, size_t* dstCapacityPtr,
+                        const void* src, size_t* srcSizePtr,
+                              ZSTD_flush_e const flush)
+{
+    U32 someMoreWork = 1;
+    const char* const istart = (const char*)src;
+    const char* const iend = istart + *srcSizePtr;
+    const char* ip = istart;
+    char* const ostart = (char*)dst;
+    char* const oend = ostart + *dstCapacityPtr;
+    char* op = ostart;
+
+    while (someMoreWork) {
+        switch(zcs->stage)
+        {
+        case zcss_init: return ERROR(init_missing);   /* call ZBUFF_compressInit() first ! */
+
+        case zcss_load:
+            /* complete inBuffer */
+            {   size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
+                size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip);
+                zcs->inBuffPos += loaded;
+                ip += loaded;
+                if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) {
+                    someMoreWork = 0; break;  /* not enough input to get a full block : stop there, wait for more */
+            }   }
+            /* compress current block (note : this stage cannot be stopped in the middle) */
+            {   void* cDst;
+                size_t cSize;
+                size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
+                size_t oSize = oend-op;
+                if (oSize >= ZSTD_compressBound(iSize))
+                    cDst = op;   /* compress directly into output buffer (avoid flush stage) */
+                else
+                    cDst = zcs->outBuff, oSize = zcs->outBuffSize;
+                cSize = (flush == zsf_end) ?
+                        ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
+                        ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
+                if (ZSTD_isError(cSize)) return cSize;
+                if (flush == zsf_end) zcs->frameEnded = 1;
+                /* prepare next block */
+                zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
+                if (zcs->inBuffTarget > zcs->inBuffSize)
+                    zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;   /* note : inBuffSize >= blockSize */
+                zcs->inToCompress = zcs->inBuffPos;
+                if (cDst == op) { op += cSize; break; }   /* no need to flush */
+                zcs->outBuffContentSize = cSize;
+                zcs->outBuffFlushedSize = 0;
+                zcs->stage = zcss_flush;   /* pass-through to flush stage */
+            }
+
+        case zcss_flush:
+            {   size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+                size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+                op += flushed;
+                zcs->outBuffFlushedSize += flushed;
+                if (toFlush!=flushed) { someMoreWork = 0; break; }  /* dst too small to store flushed data : stop there */
+                zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+                zcs->stage = zcss_load;
+                break;
+            }
+
+        case zcss_final:
+            someMoreWork = 0;   /* do nothing */
+            break;
+
+        default:
+            return ERROR(GENERIC);   /* impossible */
+        }
+    }
+
+    *srcSizePtr = ip - istart;
+    *dstCapacityPtr = op - ostart;
+    if (zcs->frameEnded) return 0;
+    {   size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
+        if (hintInSize==0) hintInSize = zcs->blockSize;
+        return hintInSize;
+    }
+}
+
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+    size_t sizeRead = input->size - input->pos;
+    size_t sizeWritten = output->size - output->pos;
+    size_t const result = ZSTD_compressStream_generic(zcs,
+                                                      (char*)(output->dst) + output->pos, &sizeWritten,
+                                                      (const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
+    input->pos += sizeRead;
+    output->pos += sizeWritten;
+    return result;
+}
+
+
+/*======   Finalize   ======*/
+
+/*! ZSTD_flushStream() :
+*   @return : amount of data remaining to flush */
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
+{
+    size_t srcSize = 0;
+    size_t sizeWritten = output->size - output->pos;
+    size_t const result = ZSTD_compressStream_generic(zcs,
+                                                      (char*)(output->dst) + output->pos, &sizeWritten,
+                                                      &srcSize, &srcSize, /* use a valid src address instead of NULL */
+                                                      zsf_flush);
+    output->pos += sizeWritten;
+    if (ZSTD_isError(result)) return result;
+    return zcs->outBuffContentSize - zcs->outBuffFlushedSize;   /* remaining to flush */
+}
+
+
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
+{
+    BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
+    BYTE* const oend = (BYTE*)(output->dst) + output->size;
+    BYTE* op = ostart;
+
+    if (zcs->stage != zcss_final) {
+        /* flush whatever remains */
+        size_t srcSize = 0;
+        size_t sizeWritten = output->size - output->pos;
+        size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end);  /* use a valid src address instead of NULL */
+        size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+        op += sizeWritten;
+        if (remainingToFlush) {
+            output->pos += sizeWritten;
+            return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
+        }
+        /* create epilogue */
+        zcs->stage = zcss_final;
+        zcs->outBuffContentSize = !notEnded ? 0 :
+            ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0);  /* write epilogue, including final empty block, into outBuff */
+    }
+
+    /* flush epilogue */
+    {   size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+        size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+        op += flushed;
+        zcs->outBuffFlushedSize += flushed;
+        output->pos += op-ostart;
+        if (toFlush==flushed) zcs->stage = zcss_init;  /* end reached */
+        return toFlush - flushed;
+    }
 }
 
 

lib/compress/zstd_opt.h

@@ -276,7 +276,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
             /* save best solution */
             if (currentMl > bestLength) {
                 bestLength = currentMl;
-                matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex3;
+                matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3;
                 matches[mnum].len = (U32)currentMl;
                 mnum++;
                 if (currentMl > ZSTD_OPT_NUM) goto update;
@@ -321,7 +321,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
         if (matchLength > bestLength) {
             if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength;
             bestLength = matchLength;
-            matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex;
+            matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex;
             matches[mnum].len = (U32)matchLength;
             mnum++;
             if (matchLength > ZSTD_OPT_NUM) break;
@@ -452,11 +452,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
         litlen = (U32)(ip - anchor);
 
         /* check repCode */
-        {   U32 i;
-            for (i=(ip == anchor); i<ZSTD_REP_CHECK; i++) {
-                if ((rep[i]<(U32)(ip-prefixStart))
-                    && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch))) {
-                    mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch;
+        {   U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
+            for (i=(ip == anchor); i<last_i; i++) {
+                const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i];
+                if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart))
+                    && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) {
+                    mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch;
                     ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
                     if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
                         best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
@@ -491,7 +492,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
             best_mlen = matches[u].len;
             ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
             while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
+                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
                 if (mlen > last_pos || price < opt[mlen].price)
                     SET_PRICE(mlen, mlen, matches[u].off, litlen, price);   /* note : macro modifies last_pos */
                 mlen++;
@@ -528,26 +529,27 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                continue;
 
            mlen = opt[cur].mlen;
-           if (opt[cur].off >= ZSTD_REP_NUM) {
+            if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
                 opt[cur].rep[2] = opt[cur-mlen].rep[1];
                 opt[cur].rep[1] = opt[cur-mlen].rep[0];
-                opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
+                opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
                 ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
            } else {
                 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
                 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
-                opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off];
+                opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
                 ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
            }
 
            ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
 
            best_mlen = minMatch;
-           {   U32 i;
-               for (i=(opt[cur].mlen != 1); i<ZSTD_REP_CHECK; i++) {  /* check rep */
-                   if ((opt[cur].rep[i]<(U32)(inr-prefixStart))
-                       && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch))) {
-                       mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch;
+            {   U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
+                for (i=(opt[cur].mlen != 1); i<last_i; i++) {  /* check rep */
+                    const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
+                    if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart))
+                       && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) {
+                       mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch;
                        ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
 
                        if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
@@ -600,12 +602,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
                         if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH);
+                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
                         else
-                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
+                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
                     } else {
                         litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
+                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
                     }
 
                   //  ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen);
@@ -652,27 +654,28 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
             litLength = (U32)(ip - anchor);
            // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
 
-            if (offset >= ZSTD_REP_NUM) {
+            if (offset > ZSTD_REP_MOVE_OPT) {
                 rep[2] = rep[1];
                 rep[1] = rep[0];
-                rep[0] = offset - ZSTD_REP_MOVE;
+                rep[0] = offset - ZSTD_REP_MOVE_OPT;
+                offset--;
             } else {
                 if (offset != 0) {
-                    best_off = rep[offset];
+                    best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]);
                     if (offset != 1) rep[2] = rep[1];
                     rep[1] = rep[0];
                     rep[0] = best_off;
                 }
-                if ((litLength == 0) & (offset==0)) offset = rep[1];  /* protection, but should never happen */
-                if ((litLength == 0) & (offset<=2)) offset--;
+                if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); };
+                if (litLength==0) offset--;
             }
 
             ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
 
 #if ZSTD_OPT_DEBUG >= 5
             U32 ml2;
-            if (offset >= ZSTD_REP_NUM)
-                ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE), iend);
+            if (offset+1 > ZSTD_REP_MOVE_OPT)
+                ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend);
             else
                 ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend);
             if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) {
@@ -748,12 +751,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
         opt[0].litlen = (U32)(ip - anchor);
 
         /* check repCode */
-        {   U32 i;
-            for (i = (ip==anchor); i<ZSTD_REP_CHECK; i++) {
-                const U32 repIndex = (U32)(current - rep[i]);
+        {   U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
+            for (i = (ip==anchor); i<last_i; i++) {
+                const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i];
+                const U32 repIndex = (U32)(current - repCur);
                 const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
                 const BYTE* const repMatch = repBase + repIndex;
-                if ( (rep[i] <= current)
+                if ( (repCur > 0 && repCur <= (S32)current)
                    && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex))  /* intentional overflow */
                    && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
                     /* repcode detected we should take it */
@@ -801,7 +805,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
             ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
             litlen = opt[0].litlen;
             while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
+                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
                 if (mlen > last_pos || price < opt[mlen].price)
                     SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
                 mlen++;
@@ -836,27 +840,28 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                 continue;
 
             mlen = opt[cur].mlen;
-            if (opt[cur].off >= ZSTD_REP_NUM) {
+            if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
                 opt[cur].rep[2] = opt[cur-mlen].rep[1];
                 opt[cur].rep[1] = opt[cur-mlen].rep[0];
-                opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
+                opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
                 ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
             } else {
                 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
                 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
-                opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off];
+                opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
                 ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
             }
 
             ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
             best_mlen = 0;
 
-            {   U32 i;
-                for (i = (opt[cur].mlen != 1); i<ZSTD_REP_CHECK; i++) {
-                    const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]);
+            {   U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
+                for (i = (mlen != 1); i<last_i; i++) {
+                    const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
+                    const U32 repIndex = (U32)(current+cur - repCur);
                     const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
                     const BYTE* const repMatch = repBase + repIndex;
-                    if ( (opt[cur].rep[i] <= current+cur)
+                    if ( (repCur > 0 && repCur <= (S32)(current+cur))
                       && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex))  /* intentional overflow */
                       && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
                         /* repcode detected */
@@ -914,12 +919,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
                         if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH);
+                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
                         else
-                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
+                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
                     } else {
                         litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
+                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
                     }
 
                 //    ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen);
@@ -966,27 +971,29 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
             litLength = (U32)(ip - anchor);
          //   ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
 
-            if (offset >= ZSTD_REP_NUM) {
+            if (offset > ZSTD_REP_MOVE_OPT) {
                 rep[2] = rep[1];
                 rep[1] = rep[0];
-                rep[0] = offset - ZSTD_REP_MOVE;
+                rep[0] = offset - ZSTD_REP_MOVE_OPT;
+                offset--;
             } else {
                 if (offset != 0) {
-                    best_off = rep[offset];
+                    best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]);
                     if (offset != 1) rep[2] = rep[1];
                     rep[1] = rep[0];
                     rep[0] = best_off;
                 }
-                if ((litLength==0) & (offset==0)) offset = rep[1];  /* protection, but should never happen */
-                if ((litLength==0) & (offset<=2)) offset --;
+
+                if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); };
+                if (litLength==0) offset--;
             }
 
             ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
 
 #if ZSTD_OPT_DEBUG >= 5
             U32 ml2;
-            if (offset >= ZSTD_REP_NUM) {
-                best_off = offset - ZSTD_REP_MOVE;
+            if (offset+1 > ZSTD_REP_MOVE_OPT) {
+                best_off = offset+1 - ZSTD_REP_MOVE_OPT;
                 if (best_off > (size_t)(ip - prefixStart))  {
                     const BYTE* match = dictEnd - (best_off - (ip - prefixStart));
                     ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart);

lib/decompress/zstd_decompress.c

@@ -77,6 +77,7 @@
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
+#  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
 #else
 #  ifdef __GNUC__
 #    define FORCE_INLINE static inline __attribute__((always_inline))
@@ -152,7 +153,8 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);
     dctx->litEntropy = dctx->fseEntropy = 0;
     dctx->dictID = 0;
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; }
+    MEM_STATIC_ASSERT(sizeof(dctx->rep)==sizeof(repStartValue));
+    memcpy(dctx->rep, repStartValue, sizeof(repStartValue));
     return 0;
 }
 
@@ -160,15 +162,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
 {
     ZSTD_DCtx* dctx;
 
-    if (!customMem.customAlloc && !customMem.customFree)
-        customMem = defaultCustomMem;
-
-    if (!customMem.customAlloc || !customMem.customFree)
-        return NULL;
+    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
+    if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
     dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
     if (!dctx) return NULL;
-    memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem));
+    memcpy(&dctx->customMem, &customMem, sizeof(customMem));
     ZSTD_decompressBegin(dctx);
     return dctx;
 }
@@ -187,8 +186,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 
 void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 {
-    memcpy(dstDCtx, srcDCtx,
-           sizeof(ZSTD_DCtx) - (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max));  /* no need to copy workspace */
+    size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
+    memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize);  /* no need to copy workspace */
 }
 
 
@@ -209,7 +208,7 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
         U32 const singleSegment = (fhd >> 5) & 1;
         U32 const fcsId = fhd >> 6;
         return ZSTD_frameHeaderSize_min + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
-                + (singleSegment && !ZSTD_fcs_fieldSize[fcsId]);
+                + (singleSegment && !fcsId);
     }
 }
 
@@ -1300,3 +1299,239 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                            dst, dstCapacity,
                                            src, srcSize);
 }
+
+
+/*=====================================
+*   Streaming decompression
+*====================================*/
+
+typedef enum { zdss_init, zdss_loadHeader,
+               zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
+
+/* *** Resource management *** */
+struct ZSTD_DStream_s {
+    ZSTD_DCtx* zd;
+    ZSTD_frameParams fParams;
+    ZSTD_dStreamStage stage;
+    char*  inBuff;
+    size_t inBuffSize;
+    size_t inPos;
+    char*  outBuff;
+    size_t outBuffSize;
+    size_t outStart;
+    size_t outEnd;
+    size_t blockSize;
+    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+    size_t lhSize;
+    ZSTD_customMem customMem;
+};   /* typedef'd to ZSTD_DStream within "zstd.h" */
+
+
+ZSTD_DStream* ZSTD_createDStream(void)
+{
+    return ZSTD_createDStream_advanced(defaultCustomMem);
+}
+
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
+{
+    ZSTD_DStream* zds;
+
+    if (!customMem.customAlloc && !customMem.customFree)
+        customMem = defaultCustomMem;
+
+    if (!customMem.customAlloc || !customMem.customFree)
+        return NULL;
+
+    zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream));
+    if (zds==NULL) return NULL;
+    memset(zds, 0, sizeof(ZSTD_DStream));
+    memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
+    zds->zd = ZSTD_createDCtx_advanced(customMem);
+    if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
+    zds->stage = zdss_init;
+    return zds;
+}
+
+size_t ZSTD_freeDStream(ZSTD_DStream* zds)
+{
+    if (zds==NULL) return 0;   /* support free on null */
+    ZSTD_freeDCtx(zds->zd);
+    if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
+    if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
+    zds->customMem.customFree(zds->customMem.opaque, zds);
+    return 0;
+}
+
+
+/* *** Initialization *** */
+
+size_t ZSTD_DStreamInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
+{
+    zds->stage = zdss_loadHeader;
+    zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+    return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize);
+}
+
+size_t ZSTD_initDStream(ZSTD_DStream* zds)
+{
+    return ZSTD_initDStream_usingDict(zds, NULL, 0);
+}
+
+
+/* *** Decompression *** */
+
+MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+    size_t const length = MIN(dstCapacity, srcSize);
+    memcpy(dst, src, length);
+    return length;
+}
+
+
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+    const char* const istart = (const char*)(input->src) + input->pos;
+    const char* const iend = (const char*)(input->src) + input->size;
+    const char* ip = istart;
+    char* const ostart = (char*)(output->dst) + output->pos;
+    char* const oend = (char*)(output->dst) + output->size;
+    char* op = ostart;
+    U32 someMoreWork = 1;
+
+    while (someMoreWork) {
+        switch(zds->stage)
+        {
+        case zdss_init :
+            return ERROR(init_missing);
+
+        case zdss_loadHeader :
+            {   size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize);
+                if (ZSTD_isError(hSize)) return hSize;
+                if (hSize != 0) {   /* need more input */
+                    size_t const toLoad = hSize - zds->lhSize;   /* if hSize!=0, hSize > zds->lhSize */
+                    if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
+                        memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
+                        zds->lhSize += iend-ip;
+                        input->pos = input->size;
+                        return (hSize - zds->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
+                    }
+                    memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
+                    break;
+            }   }
+
+            /* Consume header */
+            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd);  /* == ZSTD_frameHeaderSize_min */
+                size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size);
+                if (ZSTD_isError(h1Result)) return h1Result;   /* should not happen : already checked */
+                if (h1Size < zds->lhSize) {   /* long header */
+                    size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
+                    size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size);
+                    if (ZSTD_isError(h2Result)) return h2Result;
+            }   }
+
+            zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
+
+            /* Frame header instruct buffer sizes */
+            {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+                size_t const neededOutSize = zds->fParams.windowSize + blockSize;
+                zds->blockSize = blockSize;
+                if (zds->inBuffSize < blockSize) {
+                    zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
+                    zds->inBuffSize = blockSize;
+                    zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize);
+                    if (zds->inBuff == NULL) return ERROR(memory_allocation);
+                }
+                if (zds->outBuffSize < neededOutSize) {
+                    zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
+                    zds->outBuffSize = neededOutSize;
+                    zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize);
+                    if (zds->outBuff == NULL) return ERROR(memory_allocation);
+            }   }
+            zds->stage = zdss_read;
+            /* pass-through */
+
+        case zdss_read:
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
+                if (neededInSize==0) {  /* end of frame */
+                    zds->stage = zdss_init;
+                    someMoreWork = 0;
+                    break;
+                }
+                if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
+                    const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
+                    size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
+                        zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
+                        ip, neededInSize);
+                    if (ZSTD_isError(decodedSize)) return decodedSize;
+                    ip += neededInSize;
+                    if (!decodedSize && !isSkipFrame) break;   /* this was just a header */
+                    zds->outEnd = zds->outStart +  decodedSize;
+                    zds->stage = zdss_flush;
+                    break;
+                }
+                if (ip==iend) { someMoreWork = 0; break; }   /* no more input */
+                zds->stage = zdss_load;
+                /* pass-through */
+            }
+
+        case zdss_load:
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
+                size_t const toLoad = neededInSize - zds->inPos;   /* should always be <= remaining space within inBuff */
+                size_t loadedSize;
+                if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
+                loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
+                ip += loadedSize;
+                zds->inPos += loadedSize;
+                if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
+
+                /* decode loaded input */
+                {  const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
+                   size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
+                        zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
+                        zds->inBuff, neededInSize);
+                    if (ZSTD_isError(decodedSize)) return decodedSize;
+                    zds->inPos = 0;   /* input is consumed */
+                    if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; }   /* this was just a header */
+                    zds->outEnd = zds->outStart +  decodedSize;
+                    zds->stage = zdss_flush;
+                    /* pass-through */
+            }   }
+
+        case zdss_flush:
+            {   size_t const toFlushSize = zds->outEnd - zds->outStart;
+                size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
+                op += flushedSize;
+                zds->outStart += flushedSize;
+                if (flushedSize == toFlushSize) {  /* flush completed */
+                    zds->stage = zdss_read;
+                    if (zds->outStart + zds->blockSize > zds->outBuffSize)
+                        zds->outStart = zds->outEnd = 0;
+                    break;
+                }
+                /* cannot flush everything */
+                someMoreWork = 0;
+                break;
+            }
+        default: return ERROR(GENERIC);   /* impossible */
+    }   }
+
+    /* result */
+    input->pos += (size_t)(ip-istart);
+    output->pos += (size_t)(op-ostart);
+    {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
+        if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart);   /* return 0 only if fully flushed too */
+        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
+        if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC);   /* should never happen */
+        nextSrcSizeHint -= zds->inPos;   /* already loaded*/
+        return nextSrcSizeHint;
+    }
+}
+
+
+
+
+
+

lib/dictBuilder/zdict.c

@@ -332,12 +332,13 @@ static dictItem ZDICT_analyzePos(
         } while (length >=MINMATCHLENGTH);
 
         /* look backward */
-        do {
-            length = ZDICT_count(b + pos, b + suffix[start-1]);
-            if (length >= LLIMIT) length = LLIMIT-1;
-            lengthList[length]++;
-            if (length >=MINMATCHLENGTH) start--;
-        } while(length >= MINMATCHLENGTH);
+		length = MINMATCHLENGTH;
+		while ((length >= MINMATCHLENGTH) & (start > 0)) {
+			length = ZDICT_count(b + pos, b + suffix[start - 1]);
+			if (length >= LLIMIT) length = LLIMIT - 1;
+			lengthList[length]++;
+			if (length >= MINMATCHLENGTH) start--;
+		}
 
         /* largest useful length */
         memset(cumulLength, 0, sizeof(cumulLength));
@@ -610,18 +611,17 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
             {   const BYTE* codePtr = seqStorePtr->llCode;
                 U32 u;
                 for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
-        }   }
+            }
 
-        /* rep offsets */
-        {   const seqDef* const seq = seqStorePtr->sequences;
-            U32 offset1 = seq[0].offset - 3;
-            U32 offset2 = seq[1].offset - 3;
-            if (offset1 >= MAXREPOFFSET) offset1 = 0;
-            if (offset2 >= MAXREPOFFSET) offset2 = 0;
-            repOffsets[offset1] += 3;
-            repOffsets[offset2] += 1;
-        }
-    }
+            if (nbSeq >= 2) { /* rep offsets */
+                const seqDef* const seq = seqStorePtr->sequencesStart;
+                U32 offset1 = seq[0].offset - 3;
+                U32 offset2 = seq[1].offset - 3;
+                if (offset1 >= MAXREPOFFSET) offset1 = 0;
+                if (offset2 >= MAXREPOFFSET) offset2 = 0;
+                repOffsets[offset1] += 3;
+                repOffsets[offset2] += 1;
+    }   }   }
 }
 
 /*
@@ -675,7 +675,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     short matchLengthNCount[MaxML+1];
     U32 litLengthCount[MaxLL+1];
     short litLengthNCount[MaxLL+1];
-    U32 repOffset[MAXREPOFFSET] = { 0 };
+    U32 repOffset[MAXREPOFFSET];
     offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
     EStats_ress_t esr;
     ZSTD_parameters params;
@@ -683,31 +683,32 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     size_t pos = 0, errorCode;
     size_t eSize = 0;
     size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
-    size_t const averageSampleSize = totalSrcSize / nbFiles;
+    size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles);
     BYTE* dstPtr = (BYTE*)dstBuffer;
 
     /* init */
+    esr.ref = ZSTD_createCCtx();
+    esr.zc = ZSTD_createCCtx();
+    esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+    if (!esr.ref || !esr.zc || !esr.workPlace) {
+        eSize = ERROR(memory_allocation);
+        DISPLAYLEVEL(1, "Not enough memory \n");
+        goto _cleanup;
+    }
     if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; }   /* too large dictionary */
     for (u=0; u<256; u++) countLit[u]=1;   /* any character must be described */
     for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1;
     for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
     for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
+    memset(repOffset, 0, sizeof(repOffset));
     repOffset[1] = repOffset[4] = repOffset[8] = 1;
     memset(bestRepOffset, 0, sizeof(bestRepOffset));
-    esr.ref = ZSTD_createCCtx();
-    esr.zc = ZSTD_createCCtx();
-    esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
-    if (!esr.ref || !esr.zc || !esr.workPlace) {
-            eSize = ERROR(memory_allocation);
-            DISPLAYLEVEL(1, "Not enough memory");
-            goto _cleanup;
-    }
     if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
     params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
 	{	size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
 		if (ZSTD_isError(beginResult)) {
 			eSize = ERROR(GENERIC);
-			DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed ");
+			DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
 			goto _cleanup;
 	}	}
 
@@ -723,7 +724,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     errorCode = HUF_buildCTable (hufTable, countLit, 255, huffLog);
     if (HUF_isError(errorCode)) {
         eSize = ERROR(GENERIC);
-        DISPLAYLEVEL(1, "HUF_buildCTable error");
+        DISPLAYLEVEL(1, "HUF_buildCTable error \n");
         goto _cleanup;
     }
     huffLog = (U32)errorCode;
@@ -739,7 +740,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
     if (FSE_isError(errorCode)) {
         eSize = ERROR(GENERIC);
-        DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount");
+        DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
         goto _cleanup;
     }
     Offlog = (U32)errorCode;
@@ -748,7 +749,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
     if (FSE_isError(errorCode)) {
         eSize = ERROR(GENERIC);
-        DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount");
+        DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
         goto _cleanup;
     }
     mlLog = (U32)errorCode;
@@ -757,7 +758,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
     if (FSE_isError(errorCode)) {
         eSize = ERROR(GENERIC);
-        DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount");
+        DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
         goto _cleanup;
     }
     llLog = (U32)errorCode;
@@ -767,7 +768,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     {   size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
         if (HUF_isError(hhSize)) {
             eSize = ERROR(GENERIC);
-            DISPLAYLEVEL(1, "HUF_writeCTable error");
+            DISPLAYLEVEL(1, "HUF_writeCTable error \n");
             goto _cleanup;
         }
         dstPtr += hhSize;
@@ -778,7 +779,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     {   size_t const ohSize = FSE_writeNCount(dstPtr, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog);
         if (FSE_isError(ohSize)) {
             eSize = ERROR(GENERIC);
-            DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount");
+            DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount \n");
             goto _cleanup;
         }
         dstPtr += ohSize;
@@ -789,7 +790,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     {   size_t const mhSize = FSE_writeNCount(dstPtr, maxDstSize, matchLengthNCount, MaxML, mlLog);
         if (FSE_isError(mhSize)) {
             eSize = ERROR(GENERIC);
-            DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount");
+            DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount \n");
             goto _cleanup;
         }
         dstPtr += mhSize;
@@ -800,7 +801,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     {   size_t const lhSize = FSE_writeNCount(dstPtr, maxDstSize, litLengthNCount, MaxLL, llLog);
         if (FSE_isError(lhSize)) {
             eSize = ERROR(GENERIC);
-            DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount");
+            DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount \n");
             goto _cleanup;
         }
         dstPtr += lhSize;
@@ -810,7 +811,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
 
     if (maxDstSize<12) {
         eSize = ERROR(GENERIC);
-        DISPLAYLEVEL(1, "not enough space to write RepOffsets");
+        DISPLAYLEVEL(1, "not enough space to write RepOffsets \n");
         goto _cleanup;
     }
 # if 0
@@ -855,10 +856,14 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
     /* entropy tables */
     DISPLAYLEVEL(2, "\r%70s\r", "");   /* clean display line */
     DISPLAYLEVEL(2, "statistics ... \n");
-    hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
+    {   size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
                                   compressionLevel,
                                   samplesBuffer, samplesSizes, nbSamples,
                                   (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
+        if (ZDICT_isError(eSize)) return eSize;
+        hSize += eSize;
+    }
+
 
     if (hSize + dictContentSize < dictBufferCapacity)
         memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
@@ -901,7 +906,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
 
     /* display best matches */
     if (g_displayLevel>= 3) {
-        U32 const nb = 25;
+        U32 const nb = MIN(25, dictList[0].pos);
         U32 const dictContentSize = ZDICT_dictSize(dictList);
         U32 u;
         DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
@@ -920,7 +925,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
     /* create dictionary */
     {   U32 dictContentSize = ZDICT_dictSize(dictList);
         if (dictContentSize < targetDictSize/2) {
-            DISPLAYLEVEL(2, "!  warning : created dictionary significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
+            DISPLAYLEVEL(2, "!  warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
             if (minRep > MINRATIO) {
                 DISPLAYLEVEL(2, "!  consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1);
                 DISPLAYLEVEL(2, "!  note : larger dictionaries are not necessarily better, test its efficiency on samples \n");

lib/dictBuilder/zdict.h

@@ -38,6 +38,19 @@
 extern "C" {
 #endif
 
+
+/*======  Export for Windows  ======*/
+/*!
+*  ZSTD_DLL_EXPORT :
+*  Enable exporting of functions when building a Windows DLL
+*/
+#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZDICTLIB_API __declspec(dllexport)
+#else
+#  define ZDICTLIB_API
+#endif
+
+
 /*! ZDICT_trainFromBuffer() :
     Train a dictionary from an array of samples.
     Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
@@ -50,13 +63,13 @@ extern "C" {
            In general, it's recommended to provide a few thousands samples, but this can vary a lot.
            It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
 */
-size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
                        const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
 
 /*======   Helper functions   ======*/
-unsigned ZDICT_isError(size_t errorCode);
-const char* ZDICT_getErrorName(size_t errorCode);
+ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
+ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
 
 
 

lib/legacy/zstd_v01.c

@@ -225,7 +225,7 @@ typedef   signed long long  S64;
 #ifndef FSE_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define FSE_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define FSE_FORCE_MEMORY_ACCESS 1
 #  endif

lib/legacy/zstd_v02.c

@@ -44,6 +44,10 @@ extern "C" {
 /******************************************
 *  Compiler-specific
 ******************************************/
+#if defined(_MSC_VER)   /* Visual Studio */
+#   include <stdlib.h>  /* _byteswap_ulong */
+#   include <intrin.h>  /* _byteswap_* */
+#endif
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #  define ERR_STATIC static inline
 #elif defined(_MSC_VER)
@@ -190,7 +194,7 @@ extern "C" {
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -214,8 +218,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
 MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
@@ -228,8 +230,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
 MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
 
 #else
 
@@ -256,16 +256,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-MEM_STATIC void MEM_write32(void* memPtr, U32 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
-MEM_STATIC void MEM_write64(void* memPtr, U64 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
 #endif // MEM_FORCE_MEMORY_ACCESS
 
 
@@ -305,21 +295,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian())
-    {
-        MEM_write32(memPtr, val32);
-    }
-    else
-    {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val32;
-        p[1] = (BYTE)(val32>>8);
-        p[2] = (BYTE)(val32>>16);
-        p[3] = (BYTE)(val32>>24);
-    }
-}
 
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
@@ -333,25 +308,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian())
-    {
-        MEM_write64(memPtr, val64);
-    }
-    else
-    {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val64;
-        p[1] = (BYTE)(val64>>8);
-        p[2] = (BYTE)(val64>>16);
-        p[3] = (BYTE)(val64>>24);
-        p[4] = (BYTE)(val64>>32);
-        p[5] = (BYTE)(val64>>40);
-        p[6] = (BYTE)(val64>>48);
-        p[7] = (BYTE)(val64>>56);
-    }
-}
 
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
 {
@@ -361,14 +317,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
         return (size_t)MEM_readLE64(memPtr);
 }
 
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeLE32(memPtr, (U32)val);
-    else
-        MEM_writeLE64(memPtr, (U64)val);
-}
-
 #if defined (__cplusplus)
 }
 #endif
@@ -2983,98 +2931,6 @@ typedef struct {
 static unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
 
 
-/* *************************************
-*  Function body to include
-***************************************/
-static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
-
-MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
-{
-    if (MEM_isLittleEndian())
-    {
-        if (MEM_64bits())
-        {
-#       if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanForward64( &r, (U64)val );
-            return (int)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_ctzll((U64)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-#       endif
-        }
-        else /* 32 bits */
-        {
-#       if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r;
-            _BitScanForward( &r, (U32)val );
-            return (int)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_ctz((U32)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-        }
-    }
-    else   /* Big Endian CPU */
-    {
-        if (MEM_32bits())
-        {
-#       if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanReverse64( &r, val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_clzll(val) >> 3);
-#       else
-            unsigned r;
-            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
-            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
-            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        }
-        else /* 32 bits */
-        {
-#       if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanReverse( &r, (unsigned long)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_clz((U32)val) >> 3);
-#       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        }
-    }
-}
-
-
-MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
-{
-    const BYTE* const pStart = pIn;
-
-    while ((pIn<pInLimit-(sizeof(size_t)-1)))
-    {
-        size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
-        if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
-        pIn += ZSTD_NbCommonBytes(diff);
-        return (size_t)(pIn - pStart);
-    }
-
-    if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
-    if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
-    if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
-    return (size_t)(pIn - pStart);
-}
-
 
 /* *************************************************************
 *   Decompression section

lib/legacy/zstd_v03.c

@@ -44,6 +44,10 @@ extern "C" {
 /******************************************
 *  Compiler-specific
 ******************************************/
+#if defined(_MSC_VER)   /* Visual Studio */
+#   include <stdlib.h>  /* _byteswap_ulong */
+#   include <intrin.h>  /* _byteswap_* */
+#endif
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #  define ERR_STATIC static inline
 #elif defined(_MSC_VER)
@@ -190,7 +194,7 @@ extern "C" {
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -214,8 +218,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
 MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
@@ -228,8 +230,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
 MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
 
 #else
 
@@ -256,15 +256,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-MEM_STATIC void MEM_write32(void* memPtr, U32 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
-MEM_STATIC void MEM_write64(void* memPtr, U64 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
 
 #endif // MEM_FORCE_MEMORY_ACCESS
 
@@ -305,22 +296,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian())
-    {
-        MEM_write32(memPtr, val32);
-    }
-    else
-    {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val32;
-        p[1] = (BYTE)(val32>>8);
-        p[2] = (BYTE)(val32>>16);
-        p[3] = (BYTE)(val32>>24);
-    }
-}
-
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
     if (MEM_isLittleEndian())
@@ -333,25 +308,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian())
-    {
-        MEM_write64(memPtr, val64);
-    }
-    else
-    {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val64;
-        p[1] = (BYTE)(val64>>8);
-        p[2] = (BYTE)(val64>>16);
-        p[3] = (BYTE)(val64>>24);
-        p[4] = (BYTE)(val64>>32);
-        p[5] = (BYTE)(val64>>40);
-        p[6] = (BYTE)(val64>>48);
-        p[7] = (BYTE)(val64>>56);
-    }
-}
 
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
 {
@@ -361,13 +317,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
         return (size_t)MEM_readLE64(memPtr);
 }
 
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeLE32(memPtr, (U32)val);
-    else
-        MEM_writeLE64(memPtr, (U64)val);
-}
 
 #if defined (__cplusplus)
 }
@@ -2624,98 +2573,6 @@ typedef struct {
 static unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
 
 
-/* *************************************
-*  Function body to include
-***************************************/
-static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
-
-MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
-{
-    if (MEM_isLittleEndian())
-    {
-        if (MEM_64bits())
-        {
-#       if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanForward64( &r, (U64)val );
-            return (int)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_ctzll((U64)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-#       endif
-        }
-        else /* 32 bits */
-        {
-#       if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r;
-            _BitScanForward( &r, (U32)val );
-            return (int)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_ctz((U32)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-        }
-    }
-    else   /* Big Endian CPU */
-    {
-        if (MEM_32bits())
-        {
-#       if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanReverse64( &r, val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_clzll(val) >> 3);
-#       else
-            unsigned r;
-            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
-            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
-            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        }
-        else /* 32 bits */
-        {
-#       if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanReverse( &r, (unsigned long)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_clz((U32)val) >> 3);
-#       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        }
-    }
-}
-
-
-MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
-{
-    const BYTE* const pStart = pIn;
-
-    while ((pIn<pInLimit-(sizeof(size_t)-1)))
-    {
-        size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
-        if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
-        pIn += ZSTD_NbCommonBytes(diff);
-        return (size_t)(pIn - pStart);
-    }
-
-    if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
-    if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
-    if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
-    return (size_t)(pIn - pStart);
-}
-
 
 /* *************************************************************
 *   Decompression section

lib/legacy/zstd_v04.c

@@ -87,6 +87,10 @@ extern "C" {
 /******************************************
 *  Compiler-specific
 ******************************************/
+#if defined(_MSC_VER)   /* Visual Studio */
+#   include <stdlib.h>  /* _byteswap_ulong */
+#   include <intrin.h>  /* _byteswap_* */
+#endif
 #if defined(__GNUC__)
 #  define MEM_STATIC static __attribute__((unused))
 #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -140,7 +144,7 @@ extern "C" {
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -164,8 +168,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
 MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
@@ -178,8 +180,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
 MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
 
 #else
 
@@ -206,16 +206,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-MEM_STATIC void MEM_write32(void* memPtr, U32 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
-MEM_STATIC void MEM_write64(void* memPtr, U64 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
 #endif // MEM_FORCE_MEMORY_ACCESS
 
 
@@ -255,21 +245,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian())
-    {
-        MEM_write32(memPtr, val32);
-    }
-    else
-    {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val32;
-        p[1] = (BYTE)(val32>>8);
-        p[2] = (BYTE)(val32>>16);
-        p[3] = (BYTE)(val32>>24);
-    }
-}
 
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
@@ -283,25 +258,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian())
-    {
-        MEM_write64(memPtr, val64);
-    }
-    else
-    {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val64;
-        p[1] = (BYTE)(val64>>8);
-        p[2] = (BYTE)(val64>>16);
-        p[3] = (BYTE)(val64>>24);
-        p[4] = (BYTE)(val64>>32);
-        p[5] = (BYTE)(val64>>40);
-        p[6] = (BYTE)(val64>>48);
-        p[7] = (BYTE)(val64>>56);
-    }
-}
 
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
 {
@@ -311,13 +267,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
         return (size_t)MEM_readLE64(memPtr);
 }
 
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeLE32(memPtr, (U32)val);
-    else
-        MEM_writeLE64(memPtr, (U64)val);
-}
 
 #if defined (__cplusplus)
 }

lib/legacy/zstd_v05.c

@@ -140,7 +140,7 @@ extern "C" {
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -250,18 +250,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian()) {
-        MEM_write32(memPtr, val32);
-    } else {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val32;
-        p[1] = (BYTE)(val32>>8);
-        p[2] = (BYTE)(val32>>16);
-        p[3] = (BYTE)(val32>>24);
-    }
-}
 
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
@@ -274,22 +262,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
     }
 }
 
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian()) {
-        MEM_write64(memPtr, val64);
-    } else {
-        BYTE* p = (BYTE*)memPtr;
-        p[0] = (BYTE)val64;
-        p[1] = (BYTE)(val64>>8);
-        p[2] = (BYTE)(val64>>16);
-        p[3] = (BYTE)(val64>>24);
-        p[4] = (BYTE)(val64>>32);
-        p[5] = (BYTE)(val64>>40);
-        p[6] = (BYTE)(val64>>48);
-        p[7] = (BYTE)(val64>>56);
-    }
-}
 
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
 {
@@ -299,13 +271,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
         return (size_t)MEM_readLE64(memPtr);
 }
 
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeLE32(memPtr, (U32)val);
-    else
-        MEM_writeLE64(memPtr, (U64)val);
-}
 
 #if defined (__cplusplus)
 }
@@ -763,28 +728,6 @@ MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length)
     while (op < oend);
 }
 
-MEM_STATIC unsigned ZSTDv05_highbit(U32 val)
-{
-#   if defined(_MSC_VER)   /* Visual */
-    unsigned long r=0;
-    _BitScanReverse(&r, val);
-    return (unsigned)r;
-#   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
-    return 31 - __builtin_clz(val);
-#   else   /* Software version */
-    static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
-    U32 v = val;
-    int r;
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
-    return r;
-#   endif
-}
-
 
 /*-*******************************************
 *  Private interfaces
@@ -1350,11 +1293,6 @@ MEM_STATIC void FSEv05_initDState(FSEv05_DState_t* DStatePtr, BITv05_DStream_t*
     DStatePtr->table = dt + 1;
 }
 
-MEM_STATIC size_t FSEv05_getStateValue(FSEv05_DState_t* DStatePtr)
-{
-    return DStatePtr->state;
-}
-
 MEM_STATIC BYTE FSEv05_peakSymbol(FSEv05_DState_t* DStatePtr)
 {
     const FSEv05_decode_t DInfo = ((const FSEv05_decode_t*)(DStatePtr->table))[DStatePtr->state];

lib/legacy/zstd_v06.c

@@ -85,6 +85,10 @@ extern "C" {
 /*-****************************************
 *  Compiler specifics
 ******************************************/
+#if defined(_MSC_VER)   /* Visual Studio */
+#   include <stdlib.h>  /* _byteswap_ulong */
+#   include <intrin.h>  /* _byteswap_* */
+#endif
 #if defined(__GNUC__)
 #  define MEM_STATIC static __attribute__((unused))
 #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -138,7 +142,7 @@ extern "C" {
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -160,11 +164,8 @@ Only use if no other choice to achieve best performance on target platform */
 MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
 MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
 MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
@@ -175,11 +176,8 @@ typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed))
 MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
 MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
 MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
 
 #else
 
@@ -201,25 +199,11 @@ MEM_STATIC U64 MEM_read64(const void* memPtr)
     U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
-MEM_STATIC size_t MEM_readST(const void* memPtr)
-{
-    size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
-}
-
 MEM_STATIC void MEM_write16(void* memPtr, U16 value)
 {
     memcpy(memPtr, &value, sizeof(value));
 }
 
-MEM_STATIC void MEM_write32(void* memPtr, U32 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
-MEM_STATIC void MEM_write64(void* memPtr, U64 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
 
 #endif /* MEM_FORCE_MEMORY_ACCESS */
 
@@ -255,13 +239,6 @@ MEM_STATIC U64 MEM_swap64(U64 in)
 #endif
 }
 
-MEM_STATIC size_t MEM_swapST(size_t in)
-{
-    if (MEM_32bits())
-        return (size_t)MEM_swap32((U32)in);
-    else
-        return (size_t)MEM_swap64((U64)in);
-}
 
 /*=== Little endian r/w ===*/
 
@@ -294,13 +271,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
         return MEM_swap32(MEM_read32(memPtr));
 }
 
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian())
-        MEM_write32(memPtr, val32);
-    else
-        MEM_write32(memPtr, MEM_swap32(val32));
-}
 
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
@@ -310,13 +280,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
         return MEM_swap64(MEM_read64(memPtr));
 }
 
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian())
-        MEM_write64(memPtr, val64);
-    else
-        MEM_write64(memPtr, MEM_swap64(val64));
-}
 
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
 {
@@ -326,78 +289,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
         return (size_t)MEM_readLE64(memPtr);
 }
 
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeLE32(memPtr, (U32)val);
-    else
-        MEM_writeLE64(memPtr, (U64)val);
-}
 
-/*=== Big endian r/w ===*/
-
-MEM_STATIC U32 MEM_readBE32(const void* memPtr)
-{
-    if (MEM_isLittleEndian())
-        return MEM_swap32(MEM_read32(memPtr));
-    else
-        return MEM_read32(memPtr);
-}
-
-MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian())
-        MEM_write32(memPtr, MEM_swap32(val32));
-    else
-        MEM_write32(memPtr, val32);
-}
-
-MEM_STATIC U64 MEM_readBE64(const void* memPtr)
-{
-    if (MEM_isLittleEndian())
-        return MEM_swap64(MEM_read64(memPtr));
-    else
-        return MEM_read64(memPtr);
-}
-
-MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian())
-        MEM_write64(memPtr, MEM_swap64(val64));
-    else
-        MEM_write64(memPtr, val64);
-}
-
-MEM_STATIC size_t MEM_readBEST(const void* memPtr)
-{
-    if (MEM_32bits())
-        return (size_t)MEM_readBE32(memPtr);
-    else
-        return (size_t)MEM_readBE64(memPtr);
-}
-
-MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeBE32(memPtr, (U32)val);
-    else
-        MEM_writeBE64(memPtr, (U64)val);
-}
-
-
-/* function safe only for comparisons */
-MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
-{
-    switch (length)
-    {
-    default :
-    case 4 : return MEM_read32(memPtr);
-    case 3 : if (MEM_isLittleEndian())
-                return MEM_read32(memPtr)<<8;
-             else
-                return MEM_read32(memPtr)>>8;
-    }
-}
 
 #if defined (__cplusplus)
 }
@@ -887,27 +779,6 @@ MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length)
     while (op < oend);
 }
 
-MEM_STATIC unsigned ZSTDv06_highbit(U32 val)
-{
-#   if defined(_MSC_VER)   /* Visual */
-    unsigned long r=0;
-    _BitScanReverse(&r, val);
-    return (unsigned)r;
-#   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
-    return 31 - __builtin_clz(val);
-#   else   /* Software version */
-    static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
-    U32 v = val;
-    int r;
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
-    return r;
-#   endif
-}
 
 
 /*-*******************************************
@@ -926,15 +797,7 @@ typedef struct {
     U32 rep[ZSTDv06_REP_INIT];
 } ZSTDv06_optimal_t;
 
-#if ZSTDv06_OPT_DEBUG == 3
-    #include ".debug/zstd_stats.h"
-#else
-    typedef struct { U32  unused; } ZSTDv06_stats_t;
-    MEM_STATIC void ZSTDv06_statsPrint(ZSTDv06_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
-    MEM_STATIC void ZSTDv06_statsInit(ZSTDv06_stats_t* stats) { (void)stats; }
-    MEM_STATIC void ZSTDv06_statsResetFreqs(ZSTDv06_stats_t* stats) { (void)stats; }
-    MEM_STATIC void ZSTDv06_statsUpdatePrices(ZSTDv06_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
-#endif
+typedef struct { U32  unused; } ZSTDv06_stats_t;
 
 typedef struct {
     void* buffer;
@@ -1251,9 +1114,6 @@ MEM_STATIC unsigned BITv06_highbit32 (register U32 val)
 #   endif
 }
 
-/*=====    Local Constants   =====*/
-static const unsigned BITv06_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,  0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF };   /* up to 26 bits */
-
 
 
 /*-********************************************************
@@ -1299,29 +1159,6 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
     return srcSize;
 }
 
-MEM_STATIC size_t BITv06_getUpperBits(size_t bitContainer, U32 const start)
-{
-    return bitContainer >> start;
-}
-
-MEM_STATIC size_t BITv06_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
-{
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental */
-#  if defined(__x86_64__)
-    if (sizeof(bitContainer)==8)
-        return _bextr_u64(bitContainer, start, nbBits);
-    else
-#  endif
-        return _bextr_u32(bitContainer, start, nbBits);
-#else
-    return (bitContainer >> start) & BITv06_mask[nbBits];
-#endif
-}
-
-MEM_STATIC size_t BITv06_getLowerBits(size_t bitContainer, U32 const nbBits)
-{
-    return bitContainer & BITv06_mask[nbBits];
-}
 
 /*! BITv06_lookBits() :
  *  Provides next n bits from local register.
@@ -1332,12 +1169,8 @@ MEM_STATIC size_t BITv06_getLowerBits(size_t bitContainer, U32 const nbBits)
  */
  MEM_STATIC size_t BITv06_lookBits(const BITv06_DStream_t* bitD, U32 nbBits)
 {
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
-    return BITv06_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
-#else
     U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
     return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
-#endif
 }
 
 /*! BITv06_lookBitsFast() :
@@ -1501,8 +1334,6 @@ static void     FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t*
 
 static unsigned char FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);
 
-static unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr);
-
 /*!
 Let's now decompose FSEv06_decompress_usingDTable() into its unitary components.
 You will decode FSE-encoded symbols from the bitStream,
@@ -1626,11 +1457,6 @@ MEM_STATIC BYTE FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStre
     return symbol;
 }
 
-MEM_STATIC unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr)
-{
-    return DStatePtr->state == 0;
-}
-
 
 
 #ifndef FSEv06_COMMONDEFS_ONLY

lib/legacy/zstd_v07.c

@@ -256,10 +256,6 @@ extern "C" {
 #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 #endif
 
-/* code only tested on 32 and 64 bits systems */
-#define MEM_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
-MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
-
 
 /*-**************************************************************
 *  Basic Types
@@ -303,7 +299,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
@@ -325,11 +321,8 @@ Only use if no other choice to achieve best performance on target platform */
 MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
 MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
 MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
@@ -340,11 +333,8 @@ typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed))
 MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
 MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
 MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
 
 #else
 
@@ -366,26 +356,11 @@ MEM_STATIC U64 MEM_read64(const void* memPtr)
     U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
-MEM_STATIC size_t MEM_readST(const void* memPtr)
-{
-    size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
-}
-
 MEM_STATIC void MEM_write16(void* memPtr, U16 value)
 {
     memcpy(memPtr, &value, sizeof(value));
 }
 
-MEM_STATIC void MEM_write32(void* memPtr, U32 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
-MEM_STATIC void MEM_write64(void* memPtr, U64 value)
-{
-    memcpy(memPtr, &value, sizeof(value));
-}
-
 #endif /* MEM_FORCE_MEMORY_ACCESS */
 
 MEM_STATIC U32 MEM_swap32(U32 in)
@@ -420,13 +395,6 @@ MEM_STATIC U64 MEM_swap64(U64 in)
 #endif
 }
 
-MEM_STATIC size_t MEM_swapST(size_t in)
-{
-    if (MEM_32bits())
-        return (size_t)MEM_swap32((U32)in);
-    else
-        return (size_t)MEM_swap64((U64)in);
-}
 
 /*=== Little endian r/w ===*/
 
@@ -459,13 +427,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
         return MEM_swap32(MEM_read32(memPtr));
 }
 
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian())
-        MEM_write32(memPtr, val32);
-    else
-        MEM_write32(memPtr, MEM_swap32(val32));
-}
 
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
@@ -475,14 +436,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
         return MEM_swap64(MEM_read64(memPtr));
 }
 
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian())
-        MEM_write64(memPtr, val64);
-    else
-        MEM_write64(memPtr, MEM_swap64(val64));
-}
-
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
 {
     if (MEM_32bits())
@@ -491,78 +444,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
         return (size_t)MEM_readLE64(memPtr);
 }
 
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeLE32(memPtr, (U32)val);
-    else
-        MEM_writeLE64(memPtr, (U64)val);
-}
 
-/*=== Big endian r/w ===*/
-
-MEM_STATIC U32 MEM_readBE32(const void* memPtr)
-{
-    if (MEM_isLittleEndian())
-        return MEM_swap32(MEM_read32(memPtr));
-    else
-        return MEM_read32(memPtr);
-}
-
-MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
-{
-    if (MEM_isLittleEndian())
-        MEM_write32(memPtr, MEM_swap32(val32));
-    else
-        MEM_write32(memPtr, val32);
-}
-
-MEM_STATIC U64 MEM_readBE64(const void* memPtr)
-{
-    if (MEM_isLittleEndian())
-        return MEM_swap64(MEM_read64(memPtr));
-    else
-        return MEM_read64(memPtr);
-}
-
-MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
-{
-    if (MEM_isLittleEndian())
-        MEM_write64(memPtr, MEM_swap64(val64));
-    else
-        MEM_write64(memPtr, val64);
-}
-
-MEM_STATIC size_t MEM_readBEST(const void* memPtr)
-{
-    if (MEM_32bits())
-        return (size_t)MEM_readBE32(memPtr);
-    else
-        return (size_t)MEM_readBE64(memPtr);
-}
-
-MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
-{
-    if (MEM_32bits())
-        MEM_writeBE32(memPtr, (U32)val);
-    else
-        MEM_writeBE64(memPtr, (U64)val);
-}
-
-
-/* function safe only for comparisons */
-MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
-{
-    switch (length)
-    {
-    default :
-    case 4 : return MEM_read32(memPtr);
-    case 3 : if (MEM_isLittleEndian())
-                return MEM_read32(memPtr)<<8;
-             else
-                return MEM_read32(memPtr)>>8;
-    }
-}
 
 #if defined (__cplusplus)
 }
@@ -887,8 +769,6 @@ MEM_STATIC unsigned BITv07_highbit32 (register U32 val)
 #   endif
 }
 
-/*=====    Local Constants   =====*/
-static const unsigned BITv07_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,  0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF };   /* up to 26 bits */
 
 
 /*-********************************************************
@@ -934,29 +814,6 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
     return srcSize;
 }
 
-MEM_STATIC size_t BITv07_getUpperBits(size_t bitContainer, U32 const start)
-{
-    return bitContainer >> start;
-}
-
-MEM_STATIC size_t BITv07_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
-{
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental */
-#  if defined(__x86_64__)
-    if (sizeof(bitContainer)==8)
-        return _bextr_u64(bitContainer, start, nbBits);
-    else
-#  endif
-        return _bextr_u32(bitContainer, start, nbBits);
-#else
-    return (bitContainer >> start) & BITv07_mask[nbBits];
-#endif
-}
-
-MEM_STATIC size_t BITv07_getLowerBits(size_t bitContainer, U32 const nbBits)
-{
-    return bitContainer & BITv07_mask[nbBits];
-}
 
 /*! BITv07_lookBits() :
  *  Provides next n bits from local register.
@@ -967,12 +824,8 @@ MEM_STATIC size_t BITv07_getLowerBits(size_t bitContainer, U32 const nbBits)
  */
  MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits)
 {
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
-    return BITv07_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
-#else
     U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
     return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
-#endif
 }
 
 /*! BITv07_lookBitsFast() :
@@ -1236,7 +1089,6 @@ static void     FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t*
 
 static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD);
 
-static unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr);
 
 /**<
 Let's now decompose FSEv07_decompress_usingDTable() into its unitary components.
@@ -1356,11 +1208,6 @@ MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStre
     return symbol;
 }
 
-MEM_STATIC unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr)
-{
-    return DStatePtr->state == 0;
-}
-
 
 
 #ifndef FSEv07_COMMONDEFS_ONLY
@@ -3269,10 +3116,6 @@ typedef struct {
 } ZSTDv07_optimal_t;
 
 struct ZSTDv07_stats_s { U32 unused; };
-MEM_STATIC void ZSTDv07_statsPrint(ZSTDv07_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
-MEM_STATIC void ZSTDv07_statsInit(ZSTDv07_stats_t* stats) { (void)stats; }
-MEM_STATIC void ZSTDv07_statsResetFreqs(ZSTDv07_stats_t* stats) { (void)stats; }
-MEM_STATIC void ZSTDv07_statsUpdatePrices(ZSTDv07_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
 
 typedef struct {
     void* buffer;
@@ -3373,6 +3216,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
+#  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
 #else
 #  ifdef __GNUC__
 #    define FORCE_INLINE static inline __attribute__((always_inline))

lib/zstd.h

@@ -36,7 +36,7 @@
 extern "C" {
 #endif
 
-/*======  Dependency  ======*/
+/*======   Dependency   ======*/
 #include <stddef.h>   /* size_t */
 
 
@@ -52,10 +52,10 @@ extern "C" {
 #endif
 
 
-/*======  Version  ======*/
+/*=======   Version   =======*/
 #define ZSTD_VERSION_MAJOR    0
 #define ZSTD_VERSION_MINOR    8
-#define ZSTD_VERSION_RELEASE  0
+#define ZSTD_VERSION_RELEASE  1
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -84,23 +84,21 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
 *             potentially larger than what local system can handle as a single memory segment.
 *             In which case, it's necessary to use streaming mode to decompress data.
 *    note 2 : decompressed size is an optional field, that may not be present.
-*             When `return==0`, consider data to decompress could have any size.
-*             In which case, it's necessary to use streaming mode to decompress data,
-*             or rely on application's implied limits.
-*             (For example, it may know that its own data is necessarily cut into blocks <= 16 KB).
+*             When `return==0`, data to decompress can have any size.
+*             In which case, it's necessary to use streaming mode to decompress data.
+*             Optionally, application may rely on its own implied limits.
+*             (For example, application own data could be necessarily cut into blocks <= 16 KB).
 *    note 3 : decompressed size could be wrong or intentionally modified !
 *             Always ensure result fits within application's authorized limits !
-*             Each application can have its own set of conditions.
-*             If the intention is to decompress public data compressed by zstd command line utility,
-*             it is recommended to support at least 8 MB for extended compatibility.
+*             Each application can set its own limits.
 *    note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
-unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
 
 /*! ZSTD_decompress() :
     `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
     `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ).
     If originalSize is unknown, and if there is no implied application-specific limitations,
-    it's necessary to use streaming mode to decompress data.
+    it's preferable to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
               or an errorCode if it fails (which can be tested using ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
@@ -141,7 +139,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
 ***************************/
 /*! ZSTD_compress_usingDict() :
 *   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-*   Note : This function load the dictionary, resulting in a significant startup time. */
+*   Note : This function load the dictionary, resulting in significant startup delay. */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                            void* dst, size_t dstCapacity,
                                      const void* src, size_t srcSize,
@@ -151,7 +149,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
 /*! ZSTD_decompress_usingDict() :
 *   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
 *   Dictionary must be identical to the one used during compression.
-*   Note : This function load the dictionary, resulting in a significant startup time */
+*   Note : This function load the dictionary, resulting in significant startup delay */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                              void* dst, size_t dstCapacity,
                                        const void* src, size_t srcSize,
@@ -163,7 +161,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
 ****************************/
 /*! ZSTD_createCDict() :
 *   Create a digested dictionary, ready to start compression operation without startup delay.
-*   `dict` can be released after creation */
+*   `dict` can be released after ZSTD_CDict creation */
 typedef struct ZSTD_CDict_s ZSTD_CDict;
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -232,19 +230,19 @@ static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable f
 typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /*< from faster to stronger */
 
 typedef struct {
-    unsigned windowLog;      /*< largest match distance : larger == more compression, more memory needed during decompression */
-    unsigned chainLog;       /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
-    unsigned hashLog;        /*< dispatch table : larger == faster, more memory */
-    unsigned searchLog;      /*< nb of searches : larger == more compression, slower */
-    unsigned searchLength;   /*< match length searched : larger == faster decompression, sometimes less compression */
-    unsigned targetLength;   /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
+    unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
+    unsigned chainLog;       /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+    unsigned hashLog;        /**< dispatch table : larger == faster, more memory */
+    unsigned searchLog;      /**< nb of searches : larger == more compression, slower */
+    unsigned searchLength;   /**< match length searched : larger == faster decompression, sometimes less compression */
+    unsigned targetLength;   /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
     ZSTD_strategy strategy;
 } ZSTD_compressionParameters;
 
 typedef struct {
-    unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */
-    unsigned checksumFlag;    /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
-    unsigned noDictIDFlag;    /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
+    unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
+    unsigned checksumFlag;    /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
+    unsigned noDictIDFlag;    /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
 } ZSTD_frameParameters;
 
 typedef struct {
@@ -323,12 +321,122 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx);
 
 
 /* ******************************************************************
-*  Buffer-less streaming functions (synchronous mode)
+*  Streaming
+********************************************************************/
+
+typedef struct ZSTD_inBuffer_s {
+  const void* src;    /**< start of input buffer */
+  size_t size;        /**< size of input buffer */
+  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+  void*  dst;         /**< start of output buffer */
+  size_t size;        /**< size of output buffer */
+  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+
+/*======   compression   ======*/
+
+/*-***********************************************************************
+*  Streaming compression - howto
+*
+*  A ZSTD_CStream object is required to track streaming operation.
+*  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+*  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+*
+*  Start by initializing ZSTD_CStream.
+*  Use ZSTD_initCStream() to start a new compression operation.
+*  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+*
+*  Use ZSTD_compressStream() repetitively to consume input stream.
+*  The function will automatically update both `pos`.
+*  Note that it may not consume the entire input, in which case `pos < size`,
+*  and it's up to the caller to present again remaining data.
+*  @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
+*            or an error code, which can be tested using ZSTD_isError().
+*
+*  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+*  `output->pos` will be updated.
+*  Note some content might still be left within internal buffer if `output->size` is too small.
+*  @return : nb of bytes still present within internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZSTD_isError().
+*
+*  ZSTD_endStream() instructs to finish a frame.
+*  It will perform a flush and write frame epilogue.
+*  The epilogue is required for decoders to consider a frame completed.
+*  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+*  In which case, call again ZSTD_endStream() to complete the flush.
+*  @return : nb of bytes still present within internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************/
+
+typedef struct ZSTD_CStream_s ZSTD_CStream;
+ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+
+size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer */
+
+size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+/* advanced */
+ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
+                                 ZSTD_parameters params, unsigned long long pledgedSrcSize);
+
+
+/*======   decompression   ======*/
+
+/*-***************************************************************************
+*  Streaming decompression howto
+*
+*  A ZSTD_DStream object is required to track streaming operations.
+*  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+*  ZSTD_DStream objects can be re-init multiple times.
+*
+*  Use ZSTD_initDStream() to start a new decompression operation,
+*   or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
+*
+*  Use ZSTD_decompressStream() repetitively to consume your input.
+*  The function will update both `pos`.
+*  Note that it may not consume the entire input (pos < size),
+*  in which case it's up to the caller to present remaining input again.
+*  @return : 0 when a frame is completely decoded and fully flushed,
+*            1 when there is still some data left within internal buffer to flush,
+*            >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************************/
+
+typedef struct ZSTD_DStream_s ZSTD_DStream;
+ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+
+size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
+size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer */
+
+size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+/* advanced */
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+
+
+
+/* ******************************************************************
+*  Buffer-less and synchronous inner streaming functions
 ********************************************************************/
 /* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
-*  But it's also a complex one, with a lot of restrictions (documented below).
-*  For an easier streaming API, look into common/zbuff.h
-*  which removes all restrictions by allocating and managing its own internal buffer */
+*  But it's also a complex one, with many restrictions (documented below).
+*  Prefer using normal streaming API for an easier experience */
 
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);

programs/.gitignore

@@ -9,6 +9,8 @@ fuzzer
 fuzzer32
 zbufftest
 zbufftest32
+zstreamtest
+zstreamtest32
 datagen
 paramgrill
 paramgrill32

programs/Makefile

@@ -44,7 +44,7 @@ else
 ALIGN_LOOP =
 endif
 
-CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
+CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder
 CFLAGS ?= -O3
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
@@ -54,10 +54,12 @@ FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
 ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.o $(ZSTDDIR)/decompress/huf_decompress.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
 ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
+ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
+ZSTDDECOMP32_O = $(ZSTDDIR)/decompress/zstd_decompress32.o
 
 ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
@@ -86,21 +88,25 @@ ZSTDRTTEST= --test-large-data
 
 default: zstd
 
-all: zstd fullbench fuzzer zbufftest paramgrill datagen
+all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen
 
-all32: CFLAGS += -m32
-all32: EXT := 32$(EXT)
-all32: cleano32 all
+all32: cleano32 zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32
 
-$(ZSTDDIR)/decompress/zstd_decompress.o: CFLAGS += $(ALIGN_LOOP)
 
-zstd  : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
+$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
+	$(CC)    $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@
+
+$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
+	$(CC)  -m32 $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@
+
+zstd  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
         zstdcli.c fileio.c bench.c datagen.c dibio.c
 	$(CC)      $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
 
-zstd32: CFLAGS += -m32
-zstd32: EXT := 32$(EXT)
-zstd32: zstd
+zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
+        zstdcli.c fileio.c bench.c datagen.c dibio.c
+	$(CC)  -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
+
 
 zstd_nolegacy :
 	$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
@@ -116,49 +122,53 @@ zstd-pgo : clean zstd
 	rm zstd
 	$(MAKE) zstd MOREFLAGS=-fprofile-use
 
-zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
+zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
 	$(CC)      $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
 
 zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
-	$(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
+	zstdcli.c fileio.c
 	$(CC)      $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
 
 zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
-	$(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
+	zstdcli.c fileio.c
 	$(CC)      $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
 
 zstd-small: clean
 	CFLAGS="-Os -s" $(MAKE) zstd-frugal
 
-fullbench  : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
+fullbench  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-fullbench32 : CFLAGS += -m32
-fullbench32 : EXT := 32$(EXT)
-fullbench32 : fullbench
+fullbench32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
+	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer  : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
-fuzzer  : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
+fuzzer  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-fuzzer32 : CFLAGS += -m32
-fuzzer32 : EXT := 32$(EXT)
-fuzzer32 : fuzzer
+fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
+fuzzer32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
+	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
 
-zbufftest  : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
+zbufftest  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-zbufftest32 : CFLAGS += -m32
-zbufftest32 : EXT := 32$(EXT)
-zbufftest32 : zbufftest
+zbufftest32  : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
+	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
 
-paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c
+zstreamtest  : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c zstreamtest.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+zstreamtest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) datagen.c zstreamtest.c
+	$(CC) -m32  $(FLAGS) $^ -o $@$(EXT)
+
+paramgrill : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c paramgrill.c
 	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
 
 datagen : datagen.c datagencli.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
+roundTripCrash : $(ZSTDDECOMP_O) $(ZSTD_FILES) roundTripCrash.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 clean:
@@ -229,9 +239,9 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 zstd-playTests: datagen
 	ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
 
-test: test-zstd test-fullbench test-fuzzer test-zbuff
+test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream
 
-test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
+test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32
 
 test-all: test test32 valgrindTest
 
@@ -263,4 +273,11 @@ test-zbuff: zbufftest
 
 test-zbuff32: zbufftest32
 	./zbufftest32 $(ZBUFFTEST)
+
+test-zstream: zstreamtest
+	./zstreamtest $(ZBUFFTEST)
+
+test-zstream32: zstreamtest32
+	./zstreamtest32 $(ZBUFFTEST)
+
 endif

programs/bench.c

@@ -30,6 +30,7 @@
 #include <stdlib.h>      /* malloc, free */
 #include <string.h>      /* memset */
 #include <stdio.h>       /* fprintf, fopen, ftello64 */
+#include <time.h>         /* clock_t, clock, CLOCKS_PER_SEC */
 
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY
@@ -69,6 +70,13 @@ static U32 g_compressibilityDefault = 50;
 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static U32 g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
 
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
+            { g_time = clock(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stdout); } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
 
 /* *************************************
 *  Exceptions
@@ -101,7 +109,7 @@ void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalP
 void BMK_SetNbIterations(unsigned nbLoops)
 {
     g_nbIterations = nbLoops;
-    DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations);
+    DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
 }
 
 void BMK_SetBlockSize(size_t blockSize)
@@ -125,14 +133,6 @@ typedef struct
     size_t resSize;
 } blockParam_t;
 
-typedef struct
-{
-    double ratio;
-    size_t cSize;
-    double cSpeed;
-    double dSpeed;
-} benchResult_t;
-
 
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define MAX(a,b) ((a)>(b) ? (a) : (b))
@@ -140,9 +140,10 @@ typedef struct
 static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const char* displayName, int cLevel,
                         const size_t* fileSizes, U32 nbFiles,
-                        const void* dictBuffer, size_t dictBufferSize, benchResult_t *result)
+                        const void* dictBuffer, size_t dictBufferSize)
 {
     size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+    size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
     U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
     blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
     size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
@@ -190,13 +191,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     {   U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
-        U32 testNb;
+        U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
+        U64 totalCTime=0, totalDTime=0;
+        U32 cCompleted=0, dCompleted=0;
+#       define NB_MARKS 4
+        const char* const marks[NB_MARKS] = { " |", " /", " =",  "\\" };
+        U32 markNb = 0;
         size_t cSize = 0;
         double ratio = 0.;
 
         UTIL_getTime(&coolTime);
         DISPLAYLEVEL(2, "\r%79s\r", "");
-        for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
+        while (!cCompleted | !dCompleted) {
             UTIL_time_t clockStart;
             U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
 
@@ -208,16 +214,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             }
 
             /* Compression */
-            DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
-            memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
+            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
+            if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
 
             UTIL_sleepMilli(1);  /* give processor time to other processes */
             UTIL_waitForNextTick(ticksPerSecond);
             UTIL_getTime(&clockStart);
 
-            {   //size_t const refSrcSize = (nbBlocks == 1) ? srcSize : 0;
-                //ZSTD_parameters const zparams = ZSTD_getParams(cLevel, refSrcSize, dictBufferSize);
-                ZSTD_parameters const zparams = ZSTD_getParams(cLevel, blockSize, dictBufferSize);
+            if (!cCompleted) {   /* still some time to do compression tests */
+                ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
                 ZSTD_customMem const cmem = { NULL, NULL, NULL };
                 U32 nbLoops = 0;
                 ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
@@ -237,13 +242,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                 ZSTD_freeCDict(cdict);
                 {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
                     if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
+                    totalCTime += clockSpan;
+                    cCompleted = totalCTime>maxTime;
             }   }
 
             cSize = 0;
             { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
             ratio = (double)srcSize / (double)cSize;
-            DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
-                    testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
+            markNb = (markNb+1) % NB_MARKS;
+            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+                    marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
                     (double)srcSize / fastestC );
 
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
@@ -255,7 +263,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             UTIL_waitForNextTick(ticksPerSecond);
             UTIL_getTime(&clockStart);
 
-            {   U32 nbLoops = 0;
+            if (!dCompleted) {
+                U32 nbLoops = 0;
                 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
                 if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
                 do {
@@ -278,10 +287,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                 ZSTD_freeDDict(ddict);
                 {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
                     if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
+                    totalDTime += clockSpan;
+                    dCompleted = totalDTime>maxTime;
             }   }
 
-            DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
-                    testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
+            markNb = (markNb+1) % NB_MARKS;
+            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
+                    marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
                     (double)srcSize / fastestC,
                     (double)srcSize / fastestD );
 
@@ -312,10 +324,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
 #endif
         }   /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
 
-        result->ratio = ratio;
-        result->cSize = cSize;
-        result->cSpeed = (double)srcSize / fastestC;
-        result->dSpeed = (double)srcSize / fastestD;
+        if (g_displayLevel == 1) {
+            double cSpeed = (double)srcSize / fastestC;
+            double dSpeed = (double)srcSize / fastestD;
+            if (g_additionalParam)
+                DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
+            else
+                DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
+        }
         DISPLAYLEVEL(2, "%2i#\n", cLevel);
     }   /* Bench */
 
@@ -352,7 +368,6 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
                             const size_t* fileSizes, unsigned nbFiles,
                             const void* dictBuffer, size_t dictBufferSize)
 {
-    benchResult_t result;
     int l;
 
     const char* pch = strrchr(displayName, '\\'); /* Windows */
@@ -361,10 +376,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
 
     SET_HIGH_PRIORITY;
 
-    memset(&result, 0, sizeof(result));
-
     if (g_displayLevel == 1 && !g_additionalParam)
-        DISPLAY("bench %s %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
 
     if (cLevelLast < cLevel) cLevelLast = cLevel;
 
@@ -372,13 +385,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
         BMK_benchMem(srcBuffer, benchedSize,
                      displayName, l,
                      fileSizes, nbFiles,
-                     dictBuffer, dictBufferSize, &result);
-        if (g_displayLevel == 1) {
-            if (g_additionalParam)
-                DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam);
-            else
-                DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName);
-    }   }
+                     dictBuffer, dictBufferSize);
+    }
 }
 
 
@@ -401,7 +409,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
         }
         f = fopen(fileNamesTable[n], "rb");
         if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
-        DISPLAYLEVEL(2, "Loading %s...       \r", fileNamesTable[n]);
+        DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
         if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n;   /* buffer too small - stop after this file */
         { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
           if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);

programs/datagen.c

@@ -26,8 +26,10 @@
 /* *************************************
 *  Compiler Options
 ***************************************/
-#define _CRT_SECURE_NO_WARNINGS  /* removes Visual warning on strerror() */
-
+#if defined(_MSC_VER)
+#  define _CRT_SECURE_NO_WARNINGS    /* removes Visual warning on strerror() */
+#  define _CRT_SECURE_NO_DEPRECATE   /* removes VS2005 warning on strerror() */
+#endif
 
 /*-************************************
 *  Dependencies

programs/dibio.c

@@ -204,11 +204,11 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles");   /* should not happen */
     g_displayLevel = params.notificationLevel;
     if (nbFiles < 5) {
-        DISPLAYLEVEL(2, "!  Warning : nb of samples too low for proper processing \n");
-        DISPLAYLEVEL(2, "!  Please provide one file per sample \n");
-        DISPLAYLEVEL(2, "!  Avoid concatenating multiple samples into a single file \n");
-        DISPLAYLEVEL(2, "!  otherwise, dictBuilder will be unable to find the beginning of each sample \n");
-        DISPLAYLEVEL(2, "!  resulting in distorted statistics \n");
+        DISPLAYLEVEL(2, "!  Warning : nb of samples too low for proper processing ! \n");
+        DISPLAYLEVEL(2, "!  Please provide _one file per sample_. \n");
+        DISPLAYLEVEL(2, "!  Do not concatenate samples together into a single file, \n");
+        DISPLAYLEVEL(2, "!  as dictBuilder will be unable to find the beginning of each sample, \n");
+        DISPLAYLEVEL(2, "!  resulting in poor dictionary quality. \n");
     }
 
     /* init */

programs/fileio.c

@@ -22,8 +22,7 @@
   - zstd homepage : http://www.zstd.net
 */
 /*
-  Note : this is stand-alone program.
-  It is not part of ZSTD compression library, it is a user program of ZSTD library.
+  Note : this file is part of zstd command line, which is not library.
   The license of ZSTD library is BSD.
   The license of this file is GPLv2.
 */
@@ -41,9 +40,13 @@
 /* *************************************
 *  Compiler Options
 ***************************************/
-#define _POSIX_SOURCE 1          /* enable %llu on Windows */
-#define _CRT_SECURE_NO_WARNINGS  /* removes Visual warning on strerror() */
-
+#ifdef _MSC_VER   /* Visual */
+#  define _CRT_SECURE_NO_WARNINGS  /* removes Visual warning on strerror() */
+#  pragma warning(disable : 4204)  /* non-constant aggregate initializer */
+#endif
+#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
+#  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */ 
+#endif
 
 /*-*************************************
 *  Includes
@@ -59,8 +62,6 @@
 #include "fileio.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
 #include "zstd.h"
-#define ZBUFF_STATIC_LINKING_ONLY
-#include "zbuff.h"
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
 #  include "zstd_legacy.h"    /* ZSTD_isLegacy */
@@ -173,10 +174,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
         SET_BINARY_MODE(stdin);
     } else {
         f = fopen(srcFileName, "rb");
+        if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
     }
 
-    if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
-
     return f;
 }
 
@@ -212,9 +212,9 @@ static FILE* FIO_openDstFile(const char* dstFileName)
                     while ((ch!=EOF) && (ch!='\n')) ch = getchar();  /* flush rest of input line */
         }   }   }
         f = fopen( dstFileName, "wb" );
+        if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
     }
 
-    if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
     return f;
 }
 
@@ -240,7 +240,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
     if (fileSize > MAX_DICT_SIZE) {
         int seekResult;
         if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName);   /* avoid extreme cases */
-        DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, MAX_DICT_SIZE);
+        DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, (U32)MAX_DICT_SIZE);
         seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
         if (seekResult != 0) EXM_THROW(33, "zstd: %s: %s", fileName, strerror(errno));
         fileSize = MAX_DICT_SIZE;
@@ -265,7 +265,7 @@ typedef struct {
     size_t dstBufferSize;
     void*  dictBuffer;
     size_t dictBufferSize;
-    ZBUFF_CCtx* ctx;
+    ZSTD_CStream* cctx;
     FILE* dstFile;
     FILE* srcFile;
 } cRess_t;
@@ -275,11 +275,11 @@ static cRess_t FIO_createCResources(const char* dictFileName)
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
-    ress.ctx = ZBUFF_createCCtx();
-    if (ress.ctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZBUFF context");
-    ress.srcBufferSize = ZBUFF_recommendedCInSize();
+    ress.cctx = ZSTD_createCStream();
+    if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+    ress.srcBufferSize = ZSTD_CStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
-    ress.dstBufferSize = ZBUFF_recommendedCOutSize();
+    ress.dstBufferSize = ZSTD_CStreamOutSize();
     ress.dstBuffer = malloc(ress.dstBufferSize);
     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory");
 
@@ -295,8 +295,8 @@ static void FIO_freeCResources(cRess_t ress)
     free(ress.srcBuffer);
     free(ress.dstBuffer);
     free(ress.dictBuffer);
-    errorCode = ZBUFF_freeCCtx(ress.ctx);
-    if (ZBUFF_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
+    errorCode = ZSTD_freeCStream(ress.cctx);
+    if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode));
 }
 
 
@@ -324,8 +324,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
             params.cParams.windowLog = g_maxWLog;
             params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
         }
-        {   size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
-            if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
+        {   size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
+            if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode));
     }   }
 
     /* Main compression loop */
@@ -337,36 +337,35 @@ static int FIO_compressFilename_internal(cRess_t ress,
         DISPLAYUPDATE(2, "\rRead : %u MB  ", (U32)(readsize>>20));
 
         /* Compress using buffered streaming */
-        {   size_t usedInSize = inSize;
-            size_t cSize = ress.dstBufferSize;
-            { size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
-              if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); }
-            if (inSize != usedInSize)
+        {   ZSTD_inBuffer  inBuff = { ress.srcBuffer, inSize, 0 };
+            ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
+            { size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
+              if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); }
+            if (inBuff.pos != inBuff.size)
                 /* inBuff should be entirely consumed since buffer sizes are recommended ones */
                 EXM_THROW(24, "Compression error : input block not fully consumed");
 
             /* Write cBlock */
-            { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
-              if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
-            compressedfilesize += cSize;
+            { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
+              if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
+            compressedfilesize += outBuff.pos;
         }
         DISPLAYUPDATE(2, "\rRead : %u MB  ==> %.2f%%   ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
     }
 
     /* End of Frame */
-    {   size_t cSize = ress.dstBufferSize;
-        size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
+    {   ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
+        size_t const result = ZSTD_endStream(ress.cctx, &outBuff);
         if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
 
-        { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
-          if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
-        compressedfilesize += cSize;
+        { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
+          if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
+        compressedfilesize += outBuff.pos;
     }
 
     /* Status */
-    if (strlen(srcFileName) > 20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
     DISPLAYLEVEL(2, "\r%79s\r", "");
-    DISPLAYLEVEL(2,"%-20.20s :%6.2f%%   (%6llu => %6llu bytes, %s) \n", srcFileName,
+    DISPLAYLEVEL(2,"%-20s :%6.2f%%   (%6llu => %6llu bytes, %s) \n", srcFileName,
         (double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100,
         (unsigned long long)readsize, (unsigned long long) compressedfilesize,
          dstFileName);
@@ -496,7 +495,7 @@ typedef struct {
     size_t dstBufferSize;
     void*  dictBuffer;
     size_t dictBufferSize;
-    ZBUFF_DCtx* dctx;
+    ZSTD_DStream* dctx;
     FILE*  dstFile;
 } dRess_t;
 
@@ -506,11 +505,11 @@ static dRess_t FIO_createDResources(const char* dictFileName)
     memset(&ress, 0, sizeof(ress));
 
     /* Allocation */
-    ress.dctx = ZBUFF_createDCtx();
-    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
-    ress.srcBufferSize = ZBUFF_recommendedDInSize();
+    ress.dctx = ZSTD_createDStream();
+    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
+    ress.srcBufferSize = ZSTD_DStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
-    ress.dstBufferSize = ZBUFF_recommendedDOutSize();
+    ress.dstBufferSize = ZSTD_DStreamOutSize();
     ress.dstBuffer = malloc(ress.dstBufferSize);
     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
 
@@ -522,8 +521,8 @@ static dRess_t FIO_createDResources(const char* dictFileName)
 
 static void FIO_freeDResources(dRess_t ress)
 {
-    size_t const errorCode = ZBUFF_freeDCtx(ress.dctx);
-    if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
+    size_t const errorCode = ZSTD_freeDStream(ress.dctx);
+    if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode));
     free(ress.srcBuffer);
     free(ress.dstBuffer);
     free(ress.dictBuffer);
@@ -615,7 +614,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
     size_t readSize;
     U32 storedSkips = 0;
 
-    ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
+    ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
 
     /* Header loading (optional, saves one loop) */
     {   size_t const toLoad = 9 - alreadyLoaded;   /* assumption : 9 >= alreadyLoaded */
@@ -625,18 +624,18 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
 
     /* Main decompression Loop */
     while (1) {
-        size_t inSize=readSize, decodedSize=ress.dstBufferSize;
-        size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
-        if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
-        readSize -= inSize;
+        ZSTD_inBuffer  inBuff = { ress.srcBuffer, readSize, 0 };
+        ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
+        size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
+        if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead));
 
         /* Write block */
-        storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips);
-        frameSize += decodedSize;
+        storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips);
+        frameSize += outBuff.pos;
         DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
 
         if (toRead == 0) break;   /* end of frame */
-        if (readSize) EXM_THROW(37, "Decoding error : should consume entire input");
+        if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
 
         /* Fill input buffer */
         if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block");

programs/fuzzer.c

@@ -132,6 +132,16 @@ static int basicUnitTests(U32 seed, double compressibility)
     RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
 
     /* Basic tests */
+    DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++);
+    {   const char* errorString = ZSTD_getErrorName(0);
+        DISPLAYLEVEL(4, "OK : %s \n", errorString);
+    }
+
+    DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++);
+    {   const char* errorString = ZSTD_getErrorName(499);
+        DISPLAYLEVEL(4, "OK : %s \n", errorString);
+    }
+
     DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
     CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
                                CNBuffer, CNBuffSize, 1),

programs/playTests.sh

@@ -47,7 +47,7 @@ $ECHO "\n**** simple tests **** "
 $ZSTD -f tmp                      # trivial compression case, creates tmp.zst
 $ZSTD -df tmp.zst                 # trivial decompression case (overwrites tmp)
 $ECHO "test : too large compression level (must fail)"
-$ZSTD -99 tmp && die "too large compression level undetected"
+$ZSTD -99 -f tmp  # too large compression level, automatic sized down
 $ECHO "test : compress to stdout"
 $ZSTD tmp -c > tmpCompressed
 $ZSTD tmp --stdout > tmpCompressed       # long command format

programs/util.h

@@ -31,6 +31,9 @@ extern "C" {
 /* **************************************
 *  Compiler Options
 ****************************************/
+#if defined(__INTEL_COMPILER)
+#  pragma warning(disable : 177)    /* disable: message #177: function was declared but never referenced */
+#endif
 #if defined(_MSC_VER)
 #  define _CRT_SECURE_NO_WARNINGS    /* Disable some Visual warning messages for fopen, strncpy */
 #  define _CRT_SECURE_NO_DEPRECATE   /* VS2005 */
@@ -365,13 +368,10 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
                 pos += len + 1;
                 nbFiles++;
             }
-        }
-        else
-        {
+        } else {
             nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
             if (buf == NULL) return NULL;
-        }
-    }
+    }   }
 
     if (nbFiles == 0) { free(buf); return NULL; }
 

programs/zstd.1

@@ -128,7 +128,7 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
  benchmark file(s) using compression level #
 .TP
 .B \-i#
- iteration loops [1-9](default : 3), benchmark mode only
+ minimum evaluation time, in seconds (default : 3s), benchmark mode only
 .TP
 .B \-B#
  cut file into independent blocks of size # (default: no block)

programs/zstdcli.c

@@ -35,6 +35,10 @@
 #  define ZSTDCLI_CLEVEL_DEFAULT 3
 #endif
 
+#ifndef ZSTDCLI_CLEVEL_MAX
+#  define ZSTDCLI_CLEVEL_MAX 19
+#endif
+
 
 /*-************************************
 *  Includes
@@ -88,6 +92,8 @@
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
+#define DEFAULT_DISPLAY_LEVEL 2
+
 static const char*    g_defaultDictName = "dictionary";
 static const unsigned g_defaultMaxDictSize = 110 KB;
 static const int      g_defaultDictCLevel = 5;
@@ -100,7 +106,7 @@ static const unsigned g_defaultSelectivityLevel = 9;
 #define DISPLAY(...)           fprintf(displayOut, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...)   if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static FILE* displayOut;
-static unsigned displayLevel = 2;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
+static unsigned displayLevel = DEFAULT_DISPLAY_LEVEL;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
 
 
 /*-************************************
@@ -115,7 +121,7 @@ static int usage(const char* programName)
     DISPLAY( "          with no FILE, or when FILE is - , read standard input\n");
     DISPLAY( "Arguments :\n");
 #ifndef ZSTD_NOCOMPRESS
-    DISPLAY( " -#     : # compression level (1-%u, default:%u) \n", ZSTD_maxCLevel(), ZSTDCLI_CLEVEL_DEFAULT);
+    DISPLAY( " -#     : # compression level (1-%d, default:%d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
 #endif
 #ifndef ZSTD_NODECOMPRESS
     DISPLAY( " -d     : decompression \n");
@@ -136,14 +142,14 @@ static int usage_advanced(const char* programName)
     DISPLAY( "\n");
     DISPLAY( "Advanced arguments :\n");
     DISPLAY( " -V     : display Version number and exit\n");
-    DISPLAY( " -v     : verbose mode\n");
+    DISPLAY( " -v     : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL);
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
 #ifdef UTIL_HAS_CREATEFILELIST
     DISPLAY( " -r     : operate recursively on directories\n");
 #endif
 #ifndef ZSTD_NOCOMPRESS
-    DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
+    DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
     DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
     DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
 #endif
@@ -165,7 +171,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( "Benchmark arguments :\n");
     DISPLAY( " -b#    : benchmark file(s), using # compression level (default : 1) \n");
     DISPLAY( " -e#    : test all compression levels from -bX to # (default: 1)\n");
-    DISPLAY( " -i#    : iteration loops [1-9](default : 3)\n");
+    DISPLAY( " -i#    : minimum evaluation time in seconds (default : 3s)\n");
     DISPLAY( " -B#    : cut file into independent blocks of size # (default: no block)\n");
 #endif
     return 0;
@@ -201,7 +207,7 @@ static unsigned readU32FromChar(const char** stringPtr)
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
 
-int main(int argCount, const char** argv)
+int main(int argCount, char** argv)
 {
     int argNb,
         bench=0,
@@ -215,7 +221,8 @@ int main(int argCount, const char** argv)
         nextArgumentIsOutFileName=0,
         nextArgumentIsMaxDict=0,
         nextArgumentIsDictID=0,
-        nextArgumentIsFile=0;
+        nextArgumentIsFile=0,
+        ultra=0;
     int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
     int cLevelLast = 1;
     unsigned recursive = 0;
@@ -268,7 +275,7 @@ int main(int argCount, const char** argv)
             if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
             if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
             if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
-            if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
+            if (!strcmp(argument, "--ultra")) { ultra=1; FIO_setMaxWLog(0); continue; }
             if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
             if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
             if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
@@ -298,10 +305,7 @@ int main(int argCount, const char** argv)
     #ifndef ZSTD_NOCOMPRESS
                     /* compression Level */
                     if ((*argument>='0') && (*argument<='9')) {
-                        cLevel = readU32FromChar(&argument);
-                        dictCLevel = cLevel;
-                        if (dictCLevel > ZSTD_maxCLevel())
-                            CLEAN_RETURN(badusage(programName));
+                        dictCLevel = cLevel = readU32FromChar(&argument);
                         continue;
                     }
     #endif
@@ -317,7 +321,7 @@ int main(int argCount, const char** argv)
                     case 'd': decode=1; argument++; break;
 
                         /* Force stdout, even if stdout==console */
-                    case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); argument++; break;
+                    case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
 
                         /* Use file content as dictionary */
                     case 'D': nextEntryIsDictionary = 1; argument++; break;
@@ -441,8 +445,8 @@ int main(int argCount, const char** argv)
     if (recursive) {
         fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
         if (fileNamesTable) {
-            unsigned i;
-            for (i=0; i<fileNamesNb; i++) DISPLAYLEVEL(4, "%d %s\n", i, fileNamesTable[i]);
+            unsigned u;
+            for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, fileNamesTable[u]);
             free((void*)filenameTable);
             filenameTable = fileNamesTable;
             filenameIdx = fileNamesNb;
@@ -488,7 +492,14 @@ int main(int argCount, const char** argv)
         CLEAN_RETURN(filenameIdx);
     }
 
-    /* No warning message in pipe mode (stdin + stdout) or multiple mode */
+    /* check compression level limits */
+    {   int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
+        if (cLevel > maxCLevel) {
+            DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
+            cLevel = maxCLevel;
+    }   }
+
+    /* No warning message in pipe mode (stdin + stdout) or multi-files mode */
     if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
     if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1;
 

programs/zstreamtest.c

@@ -0,0 +1,659 @@
+/*
+    Fuzzer test tool for zstd streaming API
+    Copyright (C) Yann Collet 2016
+
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+    - ZSTD homepage : https://www.zstd.net/
+*/
+
+/*-************************************
+*  Compiler specific
+**************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  define _CRT_SECURE_NO_WARNINGS     /* fgets */
+#  pragma warning(disable : 4127)     /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4146)     /* disable: C4146: minus unsigned expression */
+#endif
+
+
+/*-************************************
+*  Includes
+**************************************/
+#include <stdlib.h>       /* free */
+#include <stdio.h>        /* fgets, sscanf */
+#include <sys/timeb.h>    /* timeb */
+#include <string.h>       /* strcmp */
+#include "mem.h"
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
+#include "zstd.h"         /* ZSTD_compressBound */
+#include "datagen.h"      /* RDG_genBuffer */
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"       /* XXH64_* */
+
+
+/*-************************************
+*  Constants
+**************************************/
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+static const U32 nbTestsDefault = 10000;
+#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
+#define FUZ_COMPRESSIBILITY_DEFAULT 50
+static const U32 prime1 = 2654435761U;
+static const U32 prime2 = 2246822519U;
+
+
+
+/*-************************************
+*  Display Macros
+**************************************/
+#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2;
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stdout); } }
+static const U32 g_refreshRate = 150;
+static U32 g_displayTime = 0;
+
+static U32 g_testTime = 0;
+
+
+/*-*******************************************************
+*  Fuzzer functions
+*********************************************************/
+#define MAX(a,b) ((a)>(b)?(a):(b))
+
+static U32 FUZ_GetMilliStart(void)
+{
+    struct timeb tb;
+    U32 nCount;
+    ftime( &tb );
+    nCount = (U32) (((tb.time & 0xFFFFF) * 1000) +  tb.millitm);
+    return nCount;
+}
+
+
+static U32 FUZ_GetMilliSpan(U32 nTimeStart)
+{
+    U32 const nCurrent = FUZ_GetMilliStart();
+    U32 nSpan = nCurrent - nTimeStart;
+    if (nTimeStart > nCurrent)
+        nSpan += 0x100000 * 1000;
+    return nSpan;
+}
+
+/*! FUZ_rand() :
+    @return : a 27 bits random value, from a 32-bits `seed`.
+    `seed` is also modified */
+#  define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+unsigned int FUZ_rand(unsigned int* seedPtr)
+{
+    U32 rand32 = *seedPtr;
+    rand32 *= prime1;
+    rand32 += prime2;
+    rand32  = FUZ_rotl32(rand32, 13);
+    *seedPtr = rand32;
+    return rand32 >> 5;
+}
+
+
+/*
+static unsigned FUZ_highbit32(U32 v32)
+{
+    unsigned nbBits = 0;
+    if (v32==0) return 0;
+    for ( ; v32 ; v32>>=1) nbBits++;
+    return nbBits;
+}
+*/
+
+static void* allocFunction(void* opaque, size_t size)
+{
+    void* address = malloc(size);
+    (void)opaque;
+    return address;
+}
+
+static void freeFunction(void* opaque, void* address)
+{
+    (void)opaque;
+    free(address);
+}
+
+static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
+{
+    int testResult = 0;
+    size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
+    void* CNBuffer = malloc(CNBufferSize);
+    size_t const skippableFrameSize = 11;
+    size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
+    void* compressedBuffer = malloc(compressedBufferSize);
+    size_t const decodedBufferSize = CNBufferSize;
+    void* decodedBuffer = malloc(decodedBufferSize);
+    size_t cSize;
+    U32 testNb=0;
+    ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
+    ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
+    ZSTD_inBuffer  inBuff;
+    ZSTD_outBuffer outBuff;
+
+    /* Create compressible test buffer */
+    if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
+        DISPLAY("Not enough memory, aborting\n");
+        goto _output_error;
+    }
+    RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
+
+    /* generate skippable frame */
+    MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
+    MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
+    cSize = skippableFrameSize + 8;
+
+    /* Basic compression test */
+    DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
+    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+      if (ZSTD_isError(r)) goto _output_error; }
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    { size_t const r = ZSTD_endStream(zc, &outBuff);
+      if (r != 0) goto _output_error; }  /*< error, or some data not flushed */
+    cSize += outBuff.pos;
+    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+
+    /* skippable frame test */
+    DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
+    ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+    inBuff.src = compressedBuffer;
+    inBuff.size = cSize;
+    inBuff.pos = 0;
+    outBuff.dst = decodedBuffer;
+    outBuff.size = CNBufferSize;
+    outBuff.pos = 0;
+    { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+      if (r != 0) goto _output_error; }
+    if (outBuff.pos != 0) goto _output_error;   /* skippable frame len is 0 */
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* Basic decompression test */
+    DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+    { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+      if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
+    if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* check regenerated data is byte exact */
+    DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+    {   size_t i;
+        for (i=0; i<CNBufferSize; i++) {
+            if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
+    }   }
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* Byte-by-byte decompression test */
+    DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
+    {   size_t r = 1;
+        ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+        inBuff.src = compressedBuffer;
+        outBuff.dst = decodedBuffer;
+        inBuff.pos = 0;
+        outBuff.pos = 0;
+        while (r) {   /* skippable frame */
+            inBuff.size = inBuff.pos + 1;
+            outBuff.size = outBuff.pos + 1;
+            r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+            if (ZSTD_isError(r)) goto _output_error;
+        }
+        ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+        r=1;
+        while (r) {   /* normal frame */
+            inBuff.size = inBuff.pos + 1;
+            outBuff.size = outBuff.pos + 1;
+            r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+            if (ZSTD_isError(r)) goto _output_error;
+        }
+    }
+    if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
+    if (inBuff.pos != cSize) goto _output_error;   /* should have read the entire frame */
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* check regenerated data is byte exact */
+    DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+    {   size_t i;
+        for (i=0; i<CNBufferSize; i++) {
+            if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
+    }   }
+    DISPLAYLEVEL(4, "OK \n");
+
+_end:
+    ZSTD_freeCStream(zc);
+    ZSTD_freeDStream(zd);
+    free(CNBuffer);
+    free(compressedBuffer);
+    free(decodedBuffer);
+    return testResult;
+
+_output_error:
+    testResult = 1;
+    DISPLAY("Error detected in Unit tests ! \n");
+    goto _end;
+}
+
+
+static size_t findDiff(const void* buf1, const void* buf2, size_t max)
+{
+    const BYTE* b1 = (const BYTE*)buf1;
+    const BYTE* b2 = (const BYTE*)buf2;
+    size_t u;
+    for (u=0; u<max; u++) {
+        if (b1[u] != b2[u]) break;
+    }
+    return u;
+}
+
+static size_t FUZ_rLogLength(U32* seed, U32 logLength)
+{
+    size_t const lengthMask = ((size_t)1 << logLength) - 1;
+    return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
+}
+
+static size_t FUZ_randomLength(U32* seed, U32 maxLog)
+{
+    U32 const logLength = FUZ_rand(seed) % maxLog;
+    return FUZ_rLogLength(seed, logLength);
+}
+
+#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
+
+#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
+                         DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }
+
+static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
+{
+    static const U32 maxSrcLog = 24;
+    static const U32 maxSampleLog = 19;
+    BYTE* cNoiseBuffer[5];
+    size_t srcBufferSize = (size_t)1<<maxSrcLog;
+    BYTE* copyBuffer;
+    size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
+    BYTE* cBuffer;
+    size_t cBufferSize   = ZSTD_compressBound(srcBufferSize);
+    BYTE* dstBuffer;
+    size_t dstBufferSize = srcBufferSize;
+    U32 result = 0;
+    U32 testNb = 0;
+    U32 coreSeed = seed;
+    ZSTD_CStream* zc;
+    ZSTD_DStream* zd;
+    U32 startTime = FUZ_GetMilliStart();
+
+    /* allocations */
+    zc = ZSTD_createCStream();
+    zd = ZSTD_createDStream();
+    cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
+    copyBuffer= (BYTE*)malloc (copyBufferSize);
+    dstBuffer = (BYTE*)malloc (dstBufferSize);
+    cBuffer   = (BYTE*)malloc (cBufferSize);
+    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
+           !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
+           "Not enough memory, fuzzer tests cancelled");
+
+    /* Create initial samples */
+    RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
+    RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
+    RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
+    RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
+    RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
+    memset(copyBuffer, 0x65, copyBufferSize);                             /* make copyBuffer considered initialized */
+
+    /* catch up testNb */
+    for (testNb=1; testNb < startTest; testNb++)
+        FUZ_rand(&coreSeed);
+
+    /* test loop */
+    for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
+        U32 lseed;
+        const BYTE* srcBuffer;
+        const BYTE* dict;
+        size_t maxTestSize, dictSize;
+        size_t cSize, totalTestSize, totalGenSize;
+        U32 n, nbChunks;
+        XXH64_state_t xxhState;
+        U64 crcOrig;
+
+        /* init */
+        DISPLAYUPDATE(2, "\r%6u", testNb);
+        if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u   ", nbTests);
+        FUZ_rand(&coreSeed);
+        lseed = coreSeed ^ prime1;
+
+        /* states full reset (unsynchronized) */
+        /* some issues only happen when reusing states in a specific sequence of parameters */
+        if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); }
+        if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); }
+
+        /* srcBuffer selection [0-4] */
+        {   U32 buffNb = FUZ_rand(&lseed) & 0x7F;
+            if (buffNb & 7) buffNb=2;   /* most common : compressible (P) */
+            else {
+                buffNb >>= 3;
+                if (buffNb & 7) {
+                    const U32 tnb[2] = { 1, 3 };   /* barely/highly compressible */
+                    buffNb = tnb[buffNb >> 3];
+                } else {
+                    const U32 tnb[2] = { 0, 4 };   /* not compressible / sparse */
+                    buffNb = tnb[buffNb >> 3];
+            }   }
+            srcBuffer = cNoiseBuffer[buffNb];
+        }
+
+        /* compression init */
+        {   U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
+            maxTestSize = FUZ_rLogLength(&lseed, testLog);
+            dictSize  = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
+            /* random dictionary selection */
+            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
+                dict = srcBuffer + dictStart;
+            }
+            {   ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
+                params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
+                params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+                {   size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0);
+                    CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError));
+        }   }   }
+
+        /* multi-segments compression test */
+        XXH64_reset(&xxhState, 0);
+        nbChunks    = (FUZ_rand(&lseed) & 127) + 2;
+        {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
+            for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
+                /* compress random chunk into random size dst buffer */
+                {   size_t const readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+                    ZSTD_inBuffer inBuff = { srcBuffer+srcStart, readChunkSize, 0 };
+                    outBuff.size = outBuff.pos + dstBuffSize;
+
+                    { size_t const compressionError = ZSTD_compressStream(zc, &outBuff, &inBuff);
+                      CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
+
+                    XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
+                    memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
+                    totalTestSize += inBuff.pos;
+                }
+
+                /* random flush operation, to mess around */
+                if ((FUZ_rand(&lseed) & 15) == 0) {
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
+                    outBuff.size = outBuff.pos + adjustedDstSize;
+                    {   size_t const flushError = ZSTD_flushStream(zc, &outBuff);
+                        CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
+            }   }   }
+
+            /* final frame epilogue */
+            {   size_t remainingToFlush = (size_t)(-1);
+                while (remainingToFlush) {
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
+                    U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush);
+                    outBuff.size = outBuff.pos + adjustedDstSize;
+                    remainingToFlush = ZSTD_endStream(zc, &outBuff);
+                    CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
+                    CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
+            }   }
+            crcOrig = XXH64_digest(&xxhState);
+            cSize = outBuff.pos;
+        }
+
+        /* multi - fragments decompression test */
+        ZSTD_initDStream_usingDict(zd, dict, dictSize);
+        {   size_t decompressionResult = 1;
+            ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
+            ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
+            for (totalGenSize = 0 ; decompressionResult ; ) {
+                size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
+                inBuff.size = inBuff.pos + readCSrcSize;
+                outBuff.size = inBuff.pos + dstBuffSize;
+                decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+                CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
+            }
+            CHECK (decompressionResult != 0, "frame not fully decoded");
+            CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size")
+            CHECK (inBuff.pos != cSize, "compressed data should be fully read")
+            {   U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
+                if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
+                CHECK (crcDest!=crcOrig, "decompressed data corrupted");
+        }   }
+
+        /*=====   noisy/erroneous src decompression test   =====*/
+
+        /* add some noise */
+        {   U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
+            U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
+                size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const noiseSize  = MIN((cSize/3) , randomNoiseSize);
+                size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
+                size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
+                memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
+        }   }
+
+        /* try decompression on noisy data */
+        ZSTD_initDStream(zd);
+        {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
+            ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
+            while (outBuff.pos < dstBufferSize) {
+                size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
+                outBuff.size = outBuff.pos + adjustedDstSize;
+                inBuff.size  = inBuff.pos + randomCSrcSize;
+                {   size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+                    if (ZSTD_isError(decompressError)) break;   /* error correctly detected */
+    }   }   }   }
+    DISPLAY("\r%u fuzzer tests completed   \n", testNb);
+
+_cleanup:
+    ZSTD_freeCStream(zc);
+    ZSTD_freeDStream(zd);
+    free(cNoiseBuffer[0]);
+    free(cNoiseBuffer[1]);
+    free(cNoiseBuffer[2]);
+    free(cNoiseBuffer[3]);
+    free(cNoiseBuffer[4]);
+    free(copyBuffer);
+    free(cBuffer);
+    free(dstBuffer);
+    return result;
+
+_output_error:
+    result = 1;
+    goto _cleanup;
+}
+
+
+/*-*******************************************************
+*  Command line
+*********************************************************/
+int FUZ_usage(const char* programName)
+{
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [args]\n", programName);
+    DISPLAY( "\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -i#    : Nb of tests (default:%u) \n", nbTestsDefault);
+    DISPLAY( " -s#    : Select seed (default:prompt user)\n");
+    DISPLAY( " -t#    : Select starting test number (default:0)\n");
+    DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -v     : verbose\n");
+    DISPLAY( " -p     : pause at the end\n");
+    DISPLAY( " -h     : display help and exit\n");
+    return 0;
+}
+
+
+int main(int argc, const char** argv)
+{
+    U32 seed=0;
+    int seedset=0;
+    int argNb;
+    int nbTests = nbTestsDefault;
+    int testNb = 0;
+    int proba = FUZ_COMPRESSIBILITY_DEFAULT;
+    int result=0;
+    U32 mainPause = 0;
+    const char* programName = argv[0];
+    ZSTD_customMem customMem = { allocFunction, freeFunction, NULL };
+    ZSTD_customMem customNULL = { NULL, NULL, NULL };
+
+    /* Check command line */
+    for(argNb=1; argNb<argc; argNb++) {
+        const char* argument = argv[argNb];
+        if(!argument) continue;   /* Protection if argument empty */
+
+        /* Parsing commands. Aggregated commands are allowed */
+        if (argument[0]=='-') {
+            argument++;
+
+            while (*argument!=0) {
+                switch(*argument)
+                {
+                case 'h':
+                    return FUZ_usage(programName);
+                case 'v':
+                    argument++;
+                    g_displayLevel=4;
+                    break;
+                case 'q':
+                    argument++;
+                    g_displayLevel--;
+                    break;
+                case 'p': /* pause at the end */
+                    argument++;
+                    mainPause = 1;
+                    break;
+
+                case 'i':
+                    argument++;
+                    nbTests=0; g_testTime=0;
+                    while ((*argument>='0') && (*argument<='9')) {
+                        nbTests *= 10;
+                        nbTests += *argument - '0';
+                        argument++;
+                    }
+                    break;
+
+                case 'T':
+                    argument++;
+                    nbTests=0; g_testTime=0;
+                    while ((*argument>='0') && (*argument<='9')) {
+                        g_testTime *= 10;
+                        g_testTime += *argument - '0';
+                        argument++;
+                    }
+                    if (*argument=='m') g_testTime *=60, argument++;
+                    if (*argument=='n') argument++;
+                    g_testTime *= 1000;
+                    break;
+
+                case 's':
+                    argument++;
+                    seed=0;
+                    seedset=1;
+                    while ((*argument>='0') && (*argument<='9')) {
+                        seed *= 10;
+                        seed += *argument - '0';
+                        argument++;
+                    }
+                    break;
+
+                case 't':
+                    argument++;
+                    testNb=0;
+                    while ((*argument>='0') && (*argument<='9')) {
+                        testNb *= 10;
+                        testNb += *argument - '0';
+                        argument++;
+                    }
+                    break;
+
+                case 'P':   /* compressibility % */
+                    argument++;
+                    proba=0;
+                    while ((*argument>='0') && (*argument<='9')) {
+                        proba *= 10;
+                        proba += *argument - '0';
+                        argument++;
+                    }
+                    if (proba<0) proba=0;
+                    if (proba>100) proba=100;
+                    break;
+
+                default:
+                    return FUZ_usage(programName);
+                }
+    }   }   }   /* for(argNb=1; argNb<argc; argNb++) */
+
+    /* Get Seed */
+    DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
+
+    if (!seedset) seed = FUZ_GetMilliStart() % 10000;
+    DISPLAY("Seed = %u\n", seed);
+    if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
+
+    if (nbTests<=0) nbTests=1;
+
+    if (testNb==0) {
+        result = basicUnitTests(0, ((double)proba) / 100, customNULL);  /* constant seed for predictability */
+        if (!result) {
+            DISPLAYLEVEL(4, "Unit tests using customMem :\n")
+            result = basicUnitTests(0, ((double)proba) / 100, customMem);  /* use custom memory allocation functions */
+    }   }
+
+    if (!result)
+        result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
+
+    if (mainPause) {
+        int unused;
+        DISPLAY("Press Enter \n");
+        unused = getchar();
+        (void)unused;
+    }
+    return result;
+}

projects/.gitignore

@@ -1,2 +1,13 @@
-build
 *Copy
+
+# Visual C++
+build/
+VS2005/
+VS2008/
+VS2010/
+VS2012/
+VS2013/
+VS2015/
+
+# CMake
+cmake/

projects/VS2005/fullbench/fullbench.vcproj

@@ -0,0 +1,448 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="8.00"
+	Name="fullbench"
+	ProjectGUID="{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}"
+	RootNamespace="fullbench"
+	Keyword="Win32Proj"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+		<Platform
+			Name="x64"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Debug|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath="..\..\..\programs\datagen.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\entropy_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\fse_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\fullbench.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\huf_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zbuff_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath="..\..\..\lib\common\bitstream.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_private.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_public.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\mem.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zbuff_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\zstd.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_internal.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_opt.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_static.h"
+				>
+			</File>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

projects/VS2005/fuzzer/fuzzer.vcproj

@@ -0,0 +1,460 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="8.00"
+	Name="fuzzer"
+	ProjectGUID="{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}"
+	RootNamespace="fuzzer"
+	Keyword="Win32Proj"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+		<Platform
+			Name="x64"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Debug|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath="..\..\..\programs\datagen.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\entropy_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\fse_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\fuzzer.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\huf_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath="..\..\..\lib\common\bitstream.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_private.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_public.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\mem.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zbuff_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\zstd.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_internal.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_opt.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_static.h"
+				>
+			</File>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

projects/VS2005/zstd.sln

@@ -0,0 +1,55 @@
+Microsoft Visual Studio Solution File, Format Version 9.00
+# Visual C++ Express 2005
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcproj", "{1A2AB08E-5CE7-4C5B-BE55-458157C14051}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcproj", "{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcproj", "{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcproj", "{99DE2A79-7298-4004-A0ED-030D7A3796CA}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.ActiveCfg = Debug|Win32
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.Build.0 = Debug|Win32
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.ActiveCfg = Debug|x64
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.Build.0 = Debug|x64
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.ActiveCfg = Release|Win32
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.Build.0 = Release|Win32
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.ActiveCfg = Release|x64
+		{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.Build.0 = Release|x64
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.ActiveCfg = Debug|Win32
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.Build.0 = Debug|Win32
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.ActiveCfg = Debug|x64
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.Build.0 = Debug|x64
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.ActiveCfg = Release|Win32
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.Build.0 = Release|Win32
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.ActiveCfg = Release|x64
+		{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.Build.0 = Release|x64
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.ActiveCfg = Debug|Win32
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.Build.0 = Debug|Win32
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.ActiveCfg = Debug|x64
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.Build.0 = Debug|x64
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.ActiveCfg = Release|Win32
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.Build.0 = Release|Win32
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.ActiveCfg = Release|x64
+		{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.Build.0 = Release|x64
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.ActiveCfg = Debug|Win32
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.Build.0 = Debug|Win32
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.ActiveCfg = Debug|x64
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.Build.0 = Debug|x64
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.ActiveCfg = Release|Win32
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.Build.0 = Release|Win32
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.ActiveCfg = Release|x64
+		{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

projects/VS2005/zstd/zstd.vcproj

@@ -0,0 +1,552 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="8.00"
+	Name="zstd"
+	ProjectGUID="{1A2AB08E-5CE7-4C5B-BE55-458157C14051}"
+	RootNamespace="zstd"
+	Keyword="Win32Proj"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+		<Platform
+			Name="x64"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="setargv.obj"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="setargv.obj"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Debug|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="setargv.obj"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="setargv.obj"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath="..\..\..\programs\bench.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\datagen.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\dibio.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\entropy_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\fileio.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\legacy\fileio_legacy.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\fse_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\huf_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zbuff_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v01.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v02.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v03.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v04.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v05.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v06.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v07.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\zstdcli.c"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath="..\..\..\lib\common\bitstream.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_private.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_public.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\mem.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zbuff.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zbuff_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\zstd.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_internal.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_opt.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v01.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v02.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v03.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v04.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v05.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v06.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v07.h"
+				>
+			</File>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

projects/VS2005/zstdlib/zstdlib.vcproj

@@ -0,0 +1,494 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="8.00"
+	Name="zstdlib"
+	ProjectGUID="{99DE2A79-7298-4004-A0ED-030D7A3796CA}"
+	RootNamespace="zstdlib"
+	Keyword="Win32Proj"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+		<Platform
+			Name="x64"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Debug|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				WarnAsError="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|x64"
+			OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			CharacterSet="2"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				OmitFramePointers="true"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="0"
+				WarningLevel="4"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\entropy_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\fse_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\huf_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zbuff_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath="..\..\..\lib\common\bitstream.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_private.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\error_public.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\fse_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\huf_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\mem.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zbuff.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zbuff_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\zstd.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_internal.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_opt.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_static.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v01.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v02.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v03.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v04.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\legacy\zstd_v05.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Resource Files"
+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
+			>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

projects/VS2010/CompileAsCpp.props

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <CompileAs>CompileAsCpp</CompileAs>
+    </ClCompile>
+  </ItemDefinitionGroup>
+</Project>

projects/VS2010/zstd/zstd.vcxproj

@@ -219,4 +219,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

tests/.gitignore

@@ -2,6 +2,7 @@
 zstdtest
 speedTest
 versionsTest
+namespaceTest
 
 # Local script
 startSpeedTest

tests/Makefile

@@ -1,6 +1,7 @@
 # ##########################################################################
 # Zstd tests - Makefile
-# Based on LZ4 version test, by Takayuki Matsuoka - 2015-2016
+# Copyright (C) Yann Collet 2015-2016
+# All rights reserved.
 #
 # GPL v2 License
 #
@@ -24,18 +25,22 @@
 # versionstest : Compatibility test between zstd versions stored on Github (v0.1+)
 # ##########################################################################
 
-PYTHON?= python3
-TESTDIR := zstdtest
+PYTHON ?= python3
+TESTARTEFACT := versionsTest namespaceTest
 
-.PHONY: default all clean versionsTest
+.PHONY: default all clean namespaceTest versionsTest
 
 default: all
 
-all: versionsTest
+all: namespaceTest versionsTest
+
+namespaceTest:
+	if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
+	$(RM) $@
 
 versionsTest:
 	$(PYTHON) test-zstd-versions.py
 
 clean:
-	@rm -fR $(TESTDIR)
+	@$(RM) -fR $(TESTARTEFACT)
 	@echo Cleaning completed

tests/namespaceTest.c

@@ -0,0 +1,38 @@
+/* ##########################################################################
+# namespaceTest
+# ensure xxhash namespace emulation is properly triggered
+# Copyright (C) Yann Collet 2016
+#
+# GPL v2 License
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ########################################################################*/
+
+
+#include <stddef.h>  /* size_t */
+#include <string.h>  /* strlen */
+
+/* symbol definition */
+extern unsigned XXH32(const void* src, size_t srcSize, unsigned seed);
+
+int main(int argc, const char** argv)
+{
+    const char* exename = argv[0];
+    unsigned result = XXH32(exename, strlen(exename), argc);
+    return !result;
+}

tests/test-zstd-speed.py

@@ -6,17 +6,26 @@ import string
 import subprocess
 import time
 import traceback
+import hashlib
 
-
+script_version = 'v0.8.0 (2016-08-03)'
 default_repo_url = 'https://github.com/Cyan4973/zstd.git'
 working_dir_name = 'speedTest'
 working_path = os.getcwd() + '/' + working_dir_name     # /path/to/zstd/tests/speedTest
 clone_path = working_path + '/' + 'zstd'                # /path/to/zstd/tests/speedTest/zstd
-email_header = '[ZSTD_speedTest]'
+email_header = 'ZSTD_speedTest'
 pid = str(os.getpid())
 verbose = False
 
 
+
+def hashfile(hasher, fname, blocksize=65536):
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(blocksize), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
 def log(text):
     print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
 
@@ -64,7 +73,7 @@ def send_email_with_attachments(branch, commit, last_commit, args, text, results
     with open(logFileName, "w") as myfile:
         myfile.writelines(text)
         myfile.close()
-        email_topic = '%s:%s Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
+        email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
                       % (email_header, pid, branch, commit, last_commit,
                          args.lowerLimit, args.ratioLimit)
         if have_mutt:
@@ -117,7 +126,7 @@ def get_last_results(resultsFileName):
     return commit, csize, cspeed, dspeed
 
 
-def benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName,
+def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName,
                           testFilePath, fileName, last_csize, last_cspeed, last_dspeed):
     sleepTime = 30
     while os.getloadavg()[0] > args.maxLoadAvg:
@@ -149,7 +158,7 @@ def benchmark_and_compare(branch, commit, last_commit, args, executableName, res
             if (float(last_csize[i])/csize[i] < args.ratioLimit):
                 text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName)
         if text:
-            text = args.message + ("\nmaxLoadAvg=%s  load average at start=%s end=%s  last_commit=%s\n" % (args.maxLoadAvg, start_load, end_load, last_commit)) + text
+            text = args.message + ("\nmaxLoadAvg=%s  load average at start=%s end=%s  last_commit=%s  md5=%s\n" % (args.maxLoadAvg, start_load, end_load, last_commit, md5sum)) + text
         return text
 
 
@@ -162,13 +171,13 @@ def update_config_file(branch, commit):
     return last_commit
 
 
-def double_check(branch, commit, args, executableName, resultsFileName, filePath, fileName):
+def double_check(branch, commit, args, executableName, md5sum, resultsFileName, filePath, fileName):
     last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
     if not args.dry_run:
-        text = benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
+        text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
         if text:
             log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
-            text = benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
+            text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
     return text
 
 
@@ -176,19 +185,31 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav
     local_branch = string.split(branch, '/')[1]
     version = local_branch.rpartition('-')[2] + '_' + commit
     if not args.dry_run:
-        execute('make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % (version, version))
+        execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version +
+                'mv programs/zstd programs/zstd_clang && ' +
+                'make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && ' % version +
+                'make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
+    md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd')
+    md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32')
+    md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang')
+    print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd32_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang))
     logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
     text_to_send = []
     results_files = ""
     for filePath in testFilePaths:
         fileName = filePath.rpartition('/')[2]
         resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
-        text = double_check(branch, commit, args, 'zstd', resultsFileName, filePath, fileName)
+        text = double_check(branch, commit, args, 'zstd', md5_zstd, resultsFileName, filePath, fileName)
         if text:
             text_to_send.append(text)
             results_files += resultsFileName + " "
         resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
-        text = double_check(branch, commit, args, 'zstd32', resultsFileName, filePath, fileName)
+        text = double_check(branch, commit, args, 'zstd32', md5_zstd32, resultsFileName, filePath, fileName)
+        if text:
+            text_to_send.append(text)
+            results_files += resultsFileName + " "
+        resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
+        text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, resultsFileName, filePath, fileName)
         if text:
             text_to_send.append(text)
             results_files += resultsFileName + " "
@@ -263,7 +284,7 @@ if __name__ == '__main__':
         log("ERROR: %s already exists, exiting" % pidfile)
         exit(1)
 
-    send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail)
+    send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
     file(pidfile, 'w').write(pid)
 
     while True:
@@ -288,11 +309,11 @@ if __name__ == '__main__':
             time.sleep(args.sleepTime)
         except Exception as e:
             stack = traceback.format_exc()
-            email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit)
+            email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
             send_email(args.emails, email_topic, stack, have_mutt, have_mail)
             print(stack)
             time.sleep(args.sleepTime)
         except KeyboardInterrupt:
             os.unlink(pidfile)
-            send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail)
+            send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
             exit(0)

zlibWrapper/zstd_zlibwrapper.c

@@ -29,14 +29,15 @@
     - zstd source repository : https://github.com/Cyan4973/zstd
 */
 
-#include <stdarg.h>            /* va_list, for z_gzprintf */
+#include <stdio.h>                 /* vsprintf */
+#include <stdarg.h>                /* va_list, for z_gzprintf */
 #include <zlib.h>
 #include "zstd_zlibwrapper.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_MAGICNUMBER */
 #include "zstd.h"
 #define ZBUFF_STATIC_LINKING_ONLY  /* ZBUFF_createCCtx_advanced */
 #include "zbuff.h"
-#include "zstd_internal.h"     /* defaultCustomMem */
+#include "zstd_internal.h"         /* defaultCustomMem */
 
 
 #define Z_INFLATE_SYNC              8
@@ -226,7 +227,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
 
     if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!");
 
-    if (flush == Z_FINISH || flush == Z_FULL_FLUSH) {
+    if (flush == Z_FINISH) {
         size_t bytesLeft;
         size_t dstCapacity = strm->avail_out;
         if (zwc->bytesLeft) {
@@ -245,6 +246,18 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END;
         zwc->bytesLeft = bytesLeft;
     }
+    
+    if (flush == Z_SYNC_FLUSH) {
+        size_t bytesLeft;
+        size_t dstCapacity = strm->avail_out;
+        bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity);
+        LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft);
+        if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR;
+        strm->next_out += dstCapacity;
+        strm->total_out += dstCapacity;
+        strm->avail_out -= dstCapacity;
+        zwc->bytesLeft = bytesLeft;
+    }
     return Z_OK;
 }
 

zstd.rb

@@ -1,8 +1,8 @@
 class Zstd < Formula
   desc "Zstandard - Fast real-time compression algorithm"
   homepage "http://www.zstd.net/"
-  url "https://github.com/Cyan4973/zstd/archive/v0.7.4.tar.gz"
-  sha256 "35ab3a5084d0194e9ff08e702edb6f507eab1bfb8c09c913639241cec852e2b7"
+  url "https://github.com/Cyan4973/zstd/archive/v0.7.5.tar.gz"
+  sha256 "6800defac9a93ddb1d9673d62c78526800df71cf9f353456f8e488ba4de51061"
 
   def install
     system "make", "install", "PREFIX=#{prefix}"

zstd_compression_format.md

@@ -60,8 +60,9 @@ explaining which parameter is unsupported.
 
 Overall conventions
 -----------
-In this document square brackets i.e. `[` and `]` are used to indicate optional fields or parameters.
-
+In this document:
+- square brackets i.e. `[` and `]` are used to indicate optional fields or parameters.
+- a naming convention for identifiers is `Mixed_Case_With_Underscores`
 
 Definitions
 -----------
@@ -106,7 +107,7 @@ The structure of a single Zstandard frame is following:
 
 __`Magic_Number`__
 
-4 Bytes, Little-endian format.
+4 Bytes, little-endian format.
 Value : 0xFD2FB527
 
 __`Frame_Header`__
@@ -122,7 +123,7 @@ __`Content_Checksum`__
 
 An optional 32-bit checksum, only present if `Content_Checksum_flag` is set.
 The content checksum is the result
-of [xxh64() hash function](https://www.xxHash.com)
+of [xxh64() hash function](http://www.xxhash.org)
 digesting the original (decoded) data as input, and a seed of zero.
 The low 4 bytes of the checksum are stored in little endian format.
 
@@ -214,11 +215,11 @@ __`Dictionary_ID_flag`__
 
 This is a 2-bits flag (`= FHD & 3`),
 telling if a dictionary ID is provided within the header.
-It also specifies the size of this field.
+It also specifies the size of this field as `Field_Size`.
 
-|  Value   |  0  |  1  |  2  |  3  |
-| -------- | --- | --- | --- | --- |
-|Field size|  0  |  1  |  2  |  4  |
+|`Flag_Value`|  0  |  1  |  2  |  3  |
+| ---------- | --- | --- | --- | --- |
+|`Field_Size`|  0  |  1  |  2  |  4  |
 
 ### `Window_Descriptor`
 
@@ -230,22 +231,22 @@ The `Window_Descriptor` byte is optional. It is absent when `Single_Segment_flag
 In this case, the maximum back-reference distance is the content size itself,
 which can be any value from 1 to 2^64-1 bytes (16 EB).
 
-| Bit numbers |    7-3   |    0-2   |
-| ----------- | -------- | -------- |
-| Field name  | Exponent | Mantissa |
+| Bit numbers |     7-3    |     0-2    |
+| ----------- | ---------- | ---------- |
+| Field name  | `Exponent` | `Mantissa` |
 
 Maximum distance is given by the following formulae :
 ```
 windowLog = 10 + Exponent;
 windowBase = 1 << windowLog;
 windowAdd = (windowBase / 8) * Mantissa;
-windowSize = windowBase + windowAdd;
+Window_Size = windowBase + windowAdd;
 ```
 The minimum window size is 1 KB.
 The maximum size is `15*(1<<38)` bytes, which is 1.875 TB.
 
 To properly decode compressed data,
-a decoder will need to allocate a buffer of at least `windowSize` bytes.
+a decoder will need to allocate a buffer of at least `Window_Size` bytes.
 
 In order to preserve decoder from unreasonable memory requirements,
 a decoder can refuse a compressed frame
@@ -287,7 +288,7 @@ the following ranges are reserved for future use and should not be used :
 This is the original (uncompressed) size. This information is optional.
 The `Field_Size` is provided according to value of `Frame_Content_Size_flag`.
 The `Field_Size` can be equal to 0 (not present), 1, 2, 4 or 8 bytes.
-Format is Little-endian.
+Format is little-endian.
 
 | `Field_Size` |    Range   |
 | ------------ | ---------- |
@@ -309,7 +310,7 @@ The structure of `Data_Block` is following:
 |:------------:|:------------:|:------------:|:---------------:|
 |   1 bit      |  2 bits      |  21 bits     |  n bytes        |
 
-The block header uses 3-bytes.
+The block header (`Last_Block`, `Block_Type`, and `Block_Size`) uses 3-bytes.
 
 __`Last_Block`__
 
@@ -379,7 +380,7 @@ Skippable frames defined in this specification are compatible with [LZ4] ones.
 
 __`Magic_Number`__
 
-4 Bytes, Little-endian format.
+4 Bytes, little-endian format.
 Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F.
 All 16 values are valid to identify a skippable frame.
 
@@ -387,7 +388,7 @@ __`Frame_Size`__
 
 This is the size, in bytes, of the following `User_Data`
 (without including the magic number nor the size field itself).
-This field is represented using 4 Bytes, Little-endian format, unsigned 32-bits.
+This field is represented using 4 Bytes, little-endian format, unsigned 32-bits.
 This means `User_Data` can’t be bigger than (2^32-1) bytes.
 
 __`User_Data`__
@@ -403,107 +404,108 @@ in order to properly allocate destination buffer.
 See [`Data_Block`](#the-structure-of-data_block) for more details.
 
 A compressed block consists of 2 sections :
-- [Literals section](#literals-section)
-- [Sequences section](#sequences-section)
+- [`Literals_Section`](#literals_section)
+- [`Sequences_Section`](#sequences_section)
 
 ### Prerequisites
 To decode a compressed block, the following elements are necessary :
-- Previous decoded blocks, up to a distance of `windowSize`,
+- Previous decoded blocks, up to a distance of `Window_Size`,
   or all previous blocks when `Single_Segment_flag` is set.
 - List of "recent offsets" from previous compressed block.
 - Decoding tables of previous compressed block for each symbol type
   (literals, litLength, matchLength, offset).
 
 
-### Literals section
+### `Literals_Section`
 
 During sequence phase, literals will be entangled with match copy operations.
 All literals are regrouped in the first part of the block.
 They can be decoded first, and then copied during sequence operations,
 or they can be decoded on the flow, as needed by sequence commands.
 
-| Literals section header | [Huffman Tree Description] | Stream1 | [Stream2] | [Stream3] | [Stream4] |
-| ----------------------- | -------------------------- | ------- | --------- | --------- | --------- |
+| `Literals_Section_Header` | [`Huffman_Tree_Description`] | Stream1 | [Stream2] | [Stream3] | [Stream4] |
+| ------------------------- | ---------------------------- | ------- | --------- | --------- | --------- |
 
 Literals can be stored uncompressed or compressed using Huffman prefix codes.
 When compressed, an optional tree description can be present,
 followed by 1 or 4 streams.
 
 
-#### Literals section header
+#### `Literals_Section_Header`
 
 Header is in charge of describing how literals are packed.
 It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes,
 using little-endian convention.
 
-| Literals Block Type | sizes format | regenerated size | [compressed size] |
-| ------------------- | ------------ | ---------------- | ----------------- |
-|   2 bits            |  1 - 2 bits  |    5 - 20 bits   |    0 - 18 bits    |
+| `Literals_Block_Type` | `Size_Format` | `Regenerated_Size` | [`Compressed_Size`] |
+| --------------------- | ------------- | ------------------ | ----------------- |
+|   2 bits              |  1 - 2 bits   |    5 - 20 bits     |    0 - 18 bits    |
 
 In this representation, bits on the left are smallest bits.
 
-__Literals Block Type__ :
+__`Literals_Block_Type`__ 
 
 This field uses 2 lowest bits of first byte, describing 4 different block types :
 
-|       Value         |  0  |  1  |      2     |      3      |
-| ------------------- | --- | --- | ---------- | ----------- |
-| Literals Block Type | Raw | RLE | Compressed | RepeatStats |    
+|       Value           |  0                   |  1                   |      2                      |       3                       |
+| --------------------- | -------------------- | -------------------- | --------------------------- | ----------------------------- |
+| `Literals_Block_Type` | `Raw_Literals_Block` | `RLE_Literals_Block` | `Compressed_Literals_Block` | `Repeat_Stats_Literals_Block` |
 
-- Raw literals block - Literals are stored uncompressed.
-- RLE literals block - Literals consist of a single byte value repeated N times.
-- Compressed literals block - This is a standard huffman-compressed block,
-        starting with a huffman tree description.
+- `Raw_Literals_Block` - Literals are stored uncompressed.
+- `RLE_Literals_Block` - Literals consist of a single byte value repeated N times.
+- `Compressed_Literals_Block` - This is a standard Huffman-compressed block,
+        starting with a Huffman tree description.
         See details below.
-- Repeat Stats literals block - This is a huffman-compressed block,
-        using huffman tree _from previous huffman-compressed literals block_.
+- `Repeat_Stats_Literals_Block` - This is a Huffman-compressed block,
+        using Huffman tree _from previous Huffman-compressed literals block_.
         Huffman tree description will be skipped.
 
-__Sizes format__ :
+__`Size_Format`__ 
 
-Sizes format are divided into 2 families :
+`Size_Format` is divided into 2 families :
 
-- For compressed block, it requires to decode both the compressed size
-  and the decompressed size. It will also decode the number of streams.
-- For Raw or RLE blocks, it's enough to decode the size to regenerate.
+- For `Compressed_Block`, it requires to decode both `Compressed_Size`
+  and `Regenerated_Size` (the decompressed size). It will also decode the number of streams.
+- For `Raw_Block` and `RLE_Block` it's enough to decode `Regenerated_Size`.
 
-For values spanning several bytes, convention is Little-endian.
+For values spanning several bytes, convention is little-endian.
 
-__Sizes format for Raw and RLE literals block__ :
+__`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ :
 
-- Value : x0 : Regenerated size uses 5 bits (0-31).
-               Total literal header size is 1 byte.
-               `size = h[0]>>3;`
-- Value : 01 : Regenerated size uses 12 bits (0-4095).
-               Total literal header size is 2 bytes.
-               `size = (h[0]>>4) + (h[1]<<4);`
-- Value : 11 : Regenerated size uses 20 bits (0-1048575).
-               Total literal header size is 3 bytes.
-               `size = (h[0]>>4) + (h[1]<<4) + (h[2]<<12);`
+- Value : x0 : `Regenerated_Size` uses 5 bits (0-31).
+               `Literals_Section_Header` has 1 byte.
+               `Regenerated_Size = Header[0]>>3`
+- Value : 01 : `Regenerated_Size` uses 12 bits (0-4095).
+               `Literals_Section_Header` has 2 bytes.
+               `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)`
+- Value : 11 : `Regenerated_Size` uses 20 bits (0-1048575).
+               `Literals_Section_Header` has 3 bytes.
+               `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)`
 
 Note : it's allowed to represent a short value (ex : `13`)
 using a long format, accepting the reduced compacity.
 
-__Sizes format for Compressed literals block and Repeat Stats literals block__ :
+__`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ :
 
 - Value : 00 : _Single stream_.
-               Compressed and regenerated sizes use 10 bits (0-1023).
-               Total literal header size is 3 bytes.
+               Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023).
+               `Literals_Section_Header` has 3 bytes.
 - Value : 01 : 4 streams.
-               Compressed and regenerated sizes use 10 bits (0-1023).
-               Total literal header size is 3 bytes.
+               Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023).
+               `Literals_Section_Header` has 3 bytes.
 - Value : 10 : 4 streams.
-               Compressed and regenerated sizes use 14 bits (0-16383).
-               Total literal header size is 4 bytes.
+               Both `Compressed_Size` and `Regenerated_Size` use 14 bits (0-16383).
+               `Literals_Section_Header` has 4 bytes.
 - Value : 11 : 4 streams.
-               Compressed and regenerated sizes use 18 bits (0-262143).
-               Total literal header size is 5 bytes.
+               Both `Compressed_Size` and `Regenerated_Size` use 18 bits (0-262143).
+               `Literals_Section_Header` has 5 bytes.
 
-Compressed and regenerated size fields follow little-endian convention.
+Both `Compressed_Size` and `Regenerated_Size` fields follow little-endian convention.
 
-#### Huffman Tree description
 
-This section is only present when literals block type is `Compressed` (`0`).
+#### `Huffman_Tree_Description`
+
+This section is only present when `Literals_Block_Type` type is `Compressed_Block` (`2`).
 
 Prefix coding represents symbols from an a priori known alphabet
 by bit sequences (codewords), one codeword for each symbol,
@@ -525,57 +527,57 @@ This specification limits maximum code length to 11 bits.
 ##### Representation
 
 All literal values from zero (included) to last present one (excluded)
-are represented by `weight` values, from 0 to `maxBits`.
-Transformation from `weight` to `nbBits` follows this formulae :
-`nbBits = weight ? maxBits + 1 - weight : 0;` .
-The last symbol's weight is deduced from previously decoded ones,
+are represented by `Weight` values, from 0 to `Max_Number_of_Bits`.
+Transformation from `Weight` to `Number_of_Bits` follows this formulae :
+`Number_of_Bits = Weight ? Max_Number_of_Bits + 1 - Weight : 0` .
+The last symbol's `Weight` is deduced from previously decoded ones,
 by completing to the nearest power of 2.
-This power of 2 gives `maxBits`, the depth of the current tree.
+This power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
 
 __Example__ :
-Let's presume the following huffman tree must be described :
+Let's presume the following Huffman tree must be described :
 
-| literal |  0  |  1  |  2  |  3  |  4  |  5  |
-| ------- | --- | --- | --- | --- | --- | --- |
-| nbBits  |  1  |  2  |  3  |  0  |  4  |  4  |
+|     literal      |  0  |  1  |  2  |  3  |  4  |  5  |
+| ---------------- | --- | --- | --- | --- | --- | --- |
+| `Number_of_Bits` |  1  |  2  |  3  |  0  |  4  |  4  |
 
 The tree depth is 4, since its smallest element uses 4 bits.
 Value `5` will not be listed, nor will values above `5`.
-Values from `0` to `4` will be listed using `weight` instead of `nbBits`.
-Weight formula is : `weight = nbBits ? maxBits + 1 - nbBits : 0;`
+Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
+Weight formula is : `Weight = Number_of_Bits ? Max_Number_of_Bits + 1 - Number_of_Bits : 0`.
 It gives the following serie of weights :
 
-| weights |  4  |  3  |  2  |  0  |  1  |
-| ------- | --- | --- | --- | --- | --- |
-| literal |  0  |  1  |  2  |  3  |  4  |
+| `Weight` |  4  |  3  |  2  |  0  |  1  |
+| -------- | --- | --- | --- | --- | --- |
+| literal  |  0  |  1  |  2  |  3  |  4  |
 
 The decoder will do the inverse operation :
 having collected weights of literals from `0` to `4`,
 it knows the last literal, `5`, is present with a non-zero weight.
 The weight of `5` can be deducted by joining to the nearest power of 2.
-Sum of 2^(weight-1) (excluding 0) is :
-`8 + 4 + 2 + 0 + 1 = 15`
+Sum of `2^(Weight-1)` (excluding 0) is :
+`8 + 4 + 2 + 0 + 1 = 15`.
 Nearest power of 2 is 16.
-Therefore, `maxBits = 4` and `weight[5] = 1`.
+Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = 1`.
 
 ##### Huffman Tree header
 
 This is a single byte value (0-255),
 which tells how to decode the list of weights.
 
-- if headerByte >= 128 : this is a direct representation,
-  where each weight is written directly as a 4 bits field (0-15).
-  The full representation occupies `((nbSymbols+1)/2)` bytes,
-  meaning it uses a last full byte even if nbSymbols is odd.
-  `nbSymbols = headerByte - 127;`.
-  Note that maximum nbSymbols is 255-127 = 128.
+- if `headerByte` >= 128 : this is a direct representation,
+  where each `Weight` is written directly as a 4 bits field (0-15).
+  The full representation occupies `((Number_of_Symbols+1)/2)` bytes,
+  meaning it uses a last full byte even if `Number_of_Symbols` is odd.
+  `Number_of_Symbols = headerByte - 127`.
+  Note that maximum `Number_of_Symbols` is 255-127 = 128.
   A larger serie must necessarily use FSE compression.
 
-- if headerByte < 128 :
+- if `headerByte` < 128 :
   the serie of weights is compressed by FSE.
   The length of the FSE-compressed serie is `headerByte` (0-127).
 
-##### FSE (Finite State Entropy) compression of huffman weights
+##### FSE (Finite State Entropy) compression of Huffman weights
 
 The serie of weights is compressed using FSE compression.
 It's a single bitstream with 2 interleaved states,
@@ -590,7 +592,7 @@ and last symbol value is not represented.
 An FSE bitstream starts by a header, describing probabilities distribution.
 It will create a Decoding Table.
 Table must be pre-allocated, which requires to support a maximum accuracy.
-For a list of huffman weights, maximum accuracy is 7 bits.
+For a list of Huffman weights, maximum accuracy is 7 bits.
 
 FSE header is [described in relevant chapter](#fse-distribution-table--condensed-format),
 and so is [FSE bitstream](#bitstream).
@@ -602,31 +604,31 @@ by tracking bitStream overflow condition.
 When both states have overflowed the bitstream, end is reached.
 
 
-##### Conversion from weights to huffman prefix codes
+##### Conversion from weights to Huffman prefix codes
 
-All present symbols shall now have a `weight` value.
-It is possible to transform weights into nbBits, using this formula :
-`nbBits = nbBits ? maxBits + 1 - weight : 0;` .
+All present symbols shall now have a `Weight` value.
+It is possible to transform weights into Number_of_Bits, using this formula :
+`Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0` .
 
-Symbols are sorted by weight. Within same weight, symbols keep natural order.
-Symbols with a weight of zero are removed.
+Symbols are sorted by `Weight`. Within same `Weight`, symbols keep natural order.
+Symbols with a `Weight` of zero are removed.
 Then, starting from lowest weight, prefix codes are distributed in order.
 
 __Example__ :
 Let's presume the following list of weights has been decoded :
 
-| Literal |  0  |  1  |  2  |  3  |  4  |  5  |
-| ------- | --- | --- | --- | --- | --- | --- |
-|  weight |  4  |  3  |  2  |  0  |  1  |  1  |
+| Literal  |  0  |  1  |  2  |  3  |  4  |  5  |
+| -------- | --- | --- | --- | --- | --- | --- |
+| `Weight` |  4  |  3  |  2  |  0  |  1  |  1  |
 
 Sorted by weight and then natural order,
 it gives the following distribution :
 
-| Literal      |  3  |  4  |  5  |  2  |  1  |   0  |
-| ------------ | --- | --- | --- | --- | --- | ---- |
-| weight       |  0  |  1  |  1  |  2  |  3  |   4  |
-| nb bits      |  0  |  4  |  4  |  3  |  2  |   1  |
-| prefix codes | N/A | 0000| 0001| 001 | 01  |   1  |
+| Literal          |  3  |  4  |  5  |  2  |  1  |   0  |
+| ---------------- | --- | --- | --- | --- | --- | ---- |
+| `Weight`         |  0  |  1  |  1  |  2  |  3  |   4  |
+| `Number_of_Bits` |  0  |  4  |  4  |  3  |  2  |   1  |
+| prefix codes     | N/A | 0000| 0001| 001 | 01  |   1  |
 
 
 #### Literals bitstreams
@@ -634,10 +636,10 @@ it gives the following distribution :
 ##### Bitstreams sizes
 
 As seen in a previous paragraph,
-there are 2 flavors of huffman-compressed literals :
+there are 2 flavors of Huffman-compressed literals :
 single stream, and 4-streams.
 
-4-streams is useful for CPU with multiple execution units and OoO operations.
+4-streams is useful for CPU with multiple execution units and out-of-order operations.
 Since each stream can be decoded independently,
 it's possible to decode them up to 4x faster than a single stream,
 presuming the CPU has enough parallelism available.
@@ -652,11 +654,11 @@ Regenerated size of each stream can be calculated by `(totalSize+3)/4`,
 except for last one, which can be up to 3 bytes smaller, to reach `totalSize`.
 
 Compressed size is provided explicitly : in the 4-streams variant,
-bitstreams are preceded by 3 unsigned Little-Endian 16-bits values.
+bitstreams are preceded by 3 unsigned little-endian 16-bits values.
 Each value represents the compressed size of one stream, in order.
 The last stream size is deducted from total compressed size
 and from previously decoded stream sizes :
-`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize;`
+`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize`.
 
 ##### Bitstreams read and decode
 
@@ -675,7 +677,7 @@ Starting from the end,
 it's possible to read the bitstream in a little-endian fashion,
 keeping track of already used bits.
 
-Reading the last `maxBits` bits,
+Reading the last `Max_Number_of_Bits` bits,
 it's then possible to compare extracted value to decoding table,
 determining the symbol to decode and number of bits to discard.
 
@@ -685,7 +687,7 @@ hence reaching exactly its beginning position with _all_ bits consumed,
 the decoding process is considered faulty.
 
 
-### Sequences section
+### `Sequences_Section`
 
 A compressed block is a succession of _sequences_ .
 A sequence is a literal copy command, followed by a match copy command.
@@ -695,7 +697,7 @@ A match copy command specifies an offset and a length.
 The offset gives the position to copy from,
 which can be within a previous block.
 
-There are 3 symbol types, `literalLength`, `matchLength` and `offset`,
+There are 3 symbol types, literals lengths, offsets and match lengths, 
 which are encoded together, interleaved in a single _bitstream_.
 
 Each symbol is a _code_ in its own context,
@@ -707,126 +709,125 @@ The Sequences section starts by a header,
 followed by optional Probability tables for each symbol type,
 followed by the bitstream.
 
-| Header | [LitLengthTable] | [OffsetTable] | [MatchLengthTable] | bitStream |
-| ------ | ---------------- | ------------- | ------------------ | --------- |
+| `Sequences_Section_Header` | [`Literals_Length_Table`] | [`Offset_Table`] | [`Match_Length_Table`] | bitStream |
+| -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- |
 
 To decode the Sequence section, it's required to know its size.
 This size is deducted from `blockSize - literalSectionSize`.
 
 
-#### Sequences section header
+#### `Sequences_Section_Header`
 
 Consists in 2 items :
-- Nb of Sequences
-- Flags providing Symbol compression types
+- `Number_of_Sequences`
+- Symbol compression modes
 
-__Nb of Sequences__
+__`Number_of_Sequences`__
 
-This is a variable size field, `nbSeqs`, using between 1 and 3 bytes.
+This is a variable size field using between 1 and 3 bytes.
 Let's call its first byte `byte0`.
 - `if (byte0 == 0)` : there are no sequences.
             The sequence section stops there.
             Regenerated content is defined entirely by literals section.
-- `if (byte0 < 128)` : `nbSeqs = byte0;` . Uses 1 byte.
-- `if (byte0 < 255)` : `nbSeqs = ((byte0-128) << 8) + byte1;` . Uses 2 bytes.
-- `if (byte0 == 255)`: `nbSeqs = byte1 + (byte2<<8) + 0x7F00;` . Uses 3 bytes.
+- `if (byte0 < 128)` : `Number_of_Sequences = byte0` . Uses 1 byte.
+- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0-128) << 8) + byte1` . Uses 2 bytes.
+- `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00` . Uses 3 bytes.
 
-__Symbol encoding modes__
+__Symbol compression modes__
 
 This is a single byte, defining the compression mode of each symbol type.
 
-|  BitNb  |   7-6  |   5-4  |   3-2  |    1-0   |
-| ------- | ------ | ------ | ------ | -------- |
-|FieldName| LLType | OFType | MLType | Reserved |
+|Bit number|   7-6                   |   5-4          |   3-2                |     1-0    |
+| -------- | ----------------------- | -------------- | -------------------- | ---------- |
+|Field name| `Literals_Lengths_Mode` | `Offsets_Mode` | `Match_Lengths_Mode` | `Reserved` |
 
 The last field, `Reserved`, must be all-zeroes.
 
-`LLType`, `OFType` and `MLType` define the compression mode of
-Literal Lengths, Offsets and Match Lengths respectively.
+`Literals_Lengths_Mode`, `Offsets_Mode` and `Match_Lengths_Mode` define the compression mode of
+literals lengths, offsets and match lengths respectively.
 
 They follow the same enumeration :
 
-|       Value      |    0   |  1  |      2     |    3   |
-| ---------------- | ------ | --- | ---------- | ------ |
-| Compression Mode | predef | RLE | Compressed | Repeat |
+|        Value       |         0         |      1     |           2           |       3       |
+| ------------------ | ----------------- | ---------- | --------------------- | ------------- |
+| `Compression_Mode` | `Predefined_Mode` | `RLE_Mode` | `FSE_Compressed_Mode` | `Repeat_Mode` |
 
-- "predef" : uses a pre-defined distribution table.
-- "RLE" : it's a single code, repeated `nbSeqs` times.
-- "Repeat" : re-use distribution table from previous compressed block.
-- "Compressed" : standard FSE compression.
+- `Predefined_Mode` : uses a predefined distribution table.
+- `RLE_Mode` : it's a single code, repeated `Number_of_Sequences` times.
+- `Repeat_Mode` : re-use distribution table from previous compressed block.
+- `FSE_Compressed_Mode` : standard FSE compression.
           A distribution table will be present.
           It will be described in [next part](#distribution-tables).
 
 #### Symbols decoding
 
-##### Literal Lengths codes
+##### Literals Length codes
 
-Literal lengths codes are values ranging from `0` to `35` included.
+Literals length codes are values ranging from `0` to `35` included.
 They define lengths from 0 to 131071 bytes.
 
-|  Code  | 0-15 |
-| ------ | ---- |
-| length | Code |
-| nbBits |   0  |
+| `Literals_Length_Code` |         0-15           |
+| ---------------------- | ---------------------- |
+| length                 | `Literals_Length_Code` |
+| `Number_of_Bits`       |          0             |
 
+| `Literals_Length_Code` |  16  |  17  |  18  |  19  |  20  |  21  |  22  |  23  |
+| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline               |  16  |  18  |  20  |  22  |  24  |  28  |  32  |  40  |
+| `Number_of_Bits`       |   1  |   1  |   1  |   1  |   2  |   2  |   3  |   3  |
 
-|   Code   |  16  |  17  |  18  |  19  |  20  |  21  |  22  |  23  |
-| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
-| Baseline |  16  |  18  |  20  |  22  |  24  |  28  |  32  |  40  |
-| nb Bits  |   1  |   1  |   1  |   1  |   2  |   2  |   3  |   3  |
+| `Literals_Length_Code` |  24  |  25  |  26  |  27  |  28  |  29  |  30  |  31  |
+| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline               |  48  |  64  |  128 |  256 |  512 | 1024 | 2048 | 4096 |
+| `Number_of_Bits`       |   4  |   6  |   7  |   8  |   9  |  10  |  11  |  12  |
 
-|   Code   |  24  |  25  |  26  |  27  |  28  |  29  |  30  |  31  |
-| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
-| Baseline |  48  |  64  |  128 |  256 |  512 | 1024 | 2048 | 4096 |
-| nb Bits  |   4  |   6  |   7  |   8  |   9  |  10  |  11  |  12  |
-
-|   Code   |  32  |  33  |  34  |  35  |
-| -------- | ---- | ---- | ---- | ---- |
-| Baseline | 8192 |16384 |32768 |65536 |
-| nb Bits  |  13  |  14  |  15  |  16  |
+| `Literals_Length_Code` |  32  |  33  |  34  |  35  |
+| ---------------------- | ---- | ---- | ---- | ---- |
+| Baseline               | 8192 |16384 |32768 |65536 |
+| `Number_of_Bits`       |  13  |  14  |  15  |  16  |
 
 __Default distribution__
 
-When "compression mode" is "predef"",
-a pre-defined distribution is used for FSE compression.
+When `Compression_Mode` is `Predefined_Mode`,
+a predefined distribution is used for FSE compression.
 
 Below is its definition. It uses an accuracy of 6 bits (64 states).
 ```
-short literalLengths_defaultDistribution[36] =
+short literalsLength_defaultDistribution[36] =
         { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
           2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
          -1,-1,-1,-1 };
 ```
 
-##### Match Lengths codes
+##### Match Length codes
 
-Match lengths codes are values ranging from `0` to `52` included.
+Match length codes are values ranging from `0` to `52` included.
 They define lengths from 3 to 131074 bytes.
 
-|  Code  |   0-31   |
-| ------ | -------- |
-| value  | Code + 3 |
-| nbBits |     0    |
+| `Match_Length_Code` |         0-31            |
+| ------------------- | ----------------------- |
+| value               | `Match_Length_Code` + 3 |
+| `Number_of_Bits`    |          0              |
 
-|   Code   |  32  |  33  |  34  |  35  |  36  |  37  |  38  |  39  |
-| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
-| Baseline |  35  |  37  |  39  |  41  |  43  |  47  |  51  |  59  |
-| nb Bits  |   1  |   1  |   1  |   1  |   2  |   2  |   3  |   3  |
+| `Match_Length_Code` |  32  |  33  |  34  |  35  |  36  |  37  |  38  |  39  |
+| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline            |  35  |  37  |  39  |  41  |  43  |  47  |  51  |  59  |
+| `Number_of_Bits`    |   1  |   1  |   1  |   1  |   2  |   2  |   3  |   3  |
 
-|   Code   |  40  |  41  |  42  |  43  |  44  |  45  |  46  |  47  |
-| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
-| Baseline |  67  |  83  |  99  |  131 |  258 |  514 | 1026 | 2050 |
-| nb Bits  |   4  |   4  |   5  |   7  |   8  |   9  |  10  |  11  |
+| `Match_Length_Code` |  40  |  41  |  42  |  43  |  44  |  45  |  46  |  47  |
+| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline            |  67  |  83  |  99  |  131 |  258 |  514 | 1026 | 2050 |
+| `Number_of_Bits`    |   4  |   4  |   5  |   7  |   8  |   9  |  10  |  11  |
 
-|   Code   |  48  |  49  |  50  |  51  |  52  |
-| -------- | ---- | ---- | ---- | ---- | ---- |
-| Baseline | 4098 | 8194 |16486 |32770 |65538 |
-| nb Bits  |  12  |  13  |  14  |  15  |  16  |
+| `Match_Length_Code` |  48  |  49  |  50  |  51  |  52  |
+| ------------------- | ---- | ---- | ---- | ---- | ---- |
+| Baseline            | 4098 | 8194 |16486 |32770 |65538 |
+| `Number_of_Bits`    |  12  |  13  |  14  |  15  |  16  |
 
 __Default distribution__
 
-When "compression mode" is defined as "predef",
-a pre-defined distribution is used for FSE compression.
+When `Compression_Mode` is defined as `Predefined_Mode`,
+a predefined distribution is used for FSE compression.
 
 Here is its definition. It uses an accuracy of 6 bits (64 states).
 ```
@@ -839,31 +840,32 @@ short matchLengths_defaultDistribution[53] =
 
 ##### Offset codes
 
-Offset codes are values ranging from `0` to `N`,
-with `N` being limited by maximum backreference distance.
+Offset codes are values ranging from `0` to `N`.
 
 A decoder is free to limit its maximum `N` supported.
 Recommendation is to support at least up to `22`.
 For information, at the time of this writing.
 the reference decoder supports a maximum `N` value of `28` in 64-bits mode.
 
-An offset code is also the nb of additional bits to read,
-and can be translated into an `OFValue` using the following formulae :
+An offset code is also the number of additional bits to read,
+and can be translated into an `Offset_Value` using the following formulae :
 
 ```
-OFValue = (1 << offsetCode) + readNBits(offsetCode);
-if (OFValue > 3) offset = OFValue - 3;
+Offset_Value = (1 << offsetCode) + readNBits(offsetCode);
+if (Offset_Value > 3) offset = Offset_Value - 3;
 ```
+It means that maximum `Offset_Value` is `2^(N+1))-1` and it supports back-reference distance up to 2^(N+1))-4
+but is limited by [maximum back-reference distance](#window_descriptor).
 
-OFValue from 1 to 3 are special : they define "repeat codes",
+Offset_Value from 1 to 3 are special : they define "repeat codes",
 which means one of the previous offsets will be repeated.
 They are sorted in recency order, with 1 meaning the most recent one.
 See [Repeat offsets](#repeat-offsets) paragraph.
 
 __Default distribution__
 
-When "compression mode" is defined as "predef",
-a pre-defined distribution is used for FSE compression.
+When `Compression_Mode` is defined as `Predefined_Mode`,
+a predefined distribution is used for FSE compression.
 
 Here is its definition. It uses an accuracy of 5 bits (32 states),
 and supports a maximum `N` of 28, allowing offset values up to 536,870,908 .
@@ -881,38 +883,38 @@ short offsetCodes_defaultDistribution[53] =
 
 Following the header, up to 3 distribution tables can be described.
 When present, they are in this order :
-- Literal lengthes
+- Literals lengths
 - Offsets
-- Match Lengthes
+- Match Lengths
 
 The content to decode depends on their respective encoding mode :
-- Predef : no content. Use pre-defined distribution table.
-- RLE : 1 byte. This is the only code to use across the whole compressed block.
-- FSE : A distribution table is present.
-- Repeat mode : no content. Re-use distribution from previous compressed block.
+- `Predefined_Mode` : no content. Use predefined distribution table.
+- `RLE_Mode` : 1 byte. This is the only code to use across the whole compressed block.
+- `FSE_Compressed_Mode` : A distribution table is present.
+- `Repeat_Mode` : no content. Re-use distribution from previous compressed block.
 
 ##### FSE distribution table : condensed format
 
 An FSE distribution table describes the probabilities of all symbols
 from `0` to the last present one (included)
-on a normalized scale of `1 << AccuracyLog` .
+on a normalized scale of `1 << Accuracy_Log` .
 
 It's a bitstream which is read forward, in little-endian fashion.
 It's not necessary to know its exact size,
 since it will be discovered and reported by the decoding process.
 
 The bitstream starts by reporting on which scale it operates.
-`AccuracyLog = low4bits + 5;`
-Note that maximum `AccuracyLog` for literal and match lengthes is `9`,
+`Accuracy_Log = low4bits + 5`.
+Note that maximum `Accuracy_Log` for literal and match length is `9`,
 and for offsets it is `8`. Higher values are considered errors.
 
 Then follow each symbol value, from `0` to last present one.
-The nb of bits used by each field is variable.
+The number of bits used by each field is variable.
 It depends on :
 
 - Remaining probabilities + 1 :
   __example__ :
-  Presuming an AccuracyLog of 8,
+  Presuming an `Accuracy_Log` of 8,
   and presuming 100 probabilities points have already been distributed,
   the decoder may read any value from `0` to `255 - 100 + 1 == 156` (included).
   Therefore, it must read `log2sup(156) == 8` bits.
@@ -926,17 +928,17 @@ It depends on :
   values from 99 to 156 use 8 bits.
   This is achieved through this scheme :
 
-  | Value read | Value decoded | nb Bits used |
-  | ---------- | ------------- | ------------ |
-  |   0 -  98  |   0 -  98     |  7           |
-  |  99 - 127  |  99 - 127     |  8           |
-  | 128 - 226  |   0 -  98     |  7           |
-  | 227 - 255  | 128 - 156     |  8           |
+  | Value read | Value decoded | Number of bits used |
+  | ---------- | ------------- | ------------------- |
+  |   0 -  98  |   0 -  98     |  7                  |
+  |  99 - 127  |  99 - 127     |  8                  |
+  | 128 - 226  |   0 -  98     |  7                  |
+  | 227 - 255  | 128 - 156     |  8                  |
 
 Symbols probabilities are read one by one, in order.
 
 Probability is obtained from Value decoded by following formulae :
-`Proba = value - 1;`
+`Proba = value - 1`
 
 It means value `0` becomes negative probability `-1`.
 `-1` is a special probability, which means `less than 1`.
@@ -951,9 +953,9 @@ This repeat flag tells how many probabilities of zeroes follow the current one.
 It provides a number ranging from 0 to 3.
 If it is a 3, another 2-bits repeat flag follows, and so on.
 
-When last symbol reaches cumulated total of `1 << AccuracyLog`,
+When last symbol reaches cumulated total of `1 << Accuracy_Log`,
 decoding is complete.
-If the last symbol makes cumulated total go above `1 << AccuracyLog`,
+If the last symbol makes cumulated total go above `1 << Accuracy_Log`,
 distribution is considered corrupted.
 
 Then the decoder can tell how many bytes were used in this process,
@@ -968,14 +970,14 @@ to create a unique decoding table.
 
 It follows the following build rule :
 
-The table has a size of `tableSize = 1 << AccuracyLog;`.
+The table has a size of `tableSize = 1 << Accuracy_Log`.
 Each cell describes the symbol decoded,
 and instructions to get the next state.
 
 Symbols are scanned in their natural order for `less than 1` probabilities.
 Symbols with this probability are being attributed a single cell,
 starting from the end of the table.
-These symbols define a full state reset, reading `AccuracyLog` bits.
+These symbols define a full state reset, reading `Accuracy_Log` bits.
 
 All remaining symbols are sorted in their natural order.
 Starting from symbol `0` and table position `0`,
@@ -1004,7 +1006,7 @@ It receives 5 state values. States are sorted in natural order.
 
 Next power of 2 is 8.
 Space of probabilities is divided into 8 equal parts.
-Presuming the AccuracyLog is 7, it defines 128 states.
+Presuming the `Accuracy_Log` is 7, it defines 128 states.
 Divided by 8, each share is 16 large.
 
 In order to reach 8, 8-5=3 lowest states will count "double",
@@ -1013,13 +1015,13 @@ requiring one more bit in the process.
 
 Numbering starts from higher states using less bits.
 
-| state order |   0   |   1   |    2   |   3  |   4   |
-| ----------- | ----- | ----- | ------ | ---- | ----- |
-| width       |  32   |  32   |   32   |  16  |  16   |
-| nb Bits     |   5   |   5   |    5   |   4  |   4   |
-| range nb    |   2   |   4   |    6   |   0  |   1   |
-| baseline    |  32   |  64   |   96   |   0  |  16   |
-| range       | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
+| state order      |   0   |   1   |    2   |   3  |   4   |
+| ---------------- | ----- | ----- | ------ | ---- | ----- |
+| width            |  32   |  32   |   32   |  16  |  16   |
+| `Number_of_Bits` |   5   |   5   |    5   |   4  |   4   |
+| range number     |   2   |   4   |    6   |   0  |   1   |
+| baseline         |  32   |  64   |   96   |   0  |  16   |
+| range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
 
 Next state is determined from current state
 by reading the required number of bits, and adding the specified baseline.
@@ -1043,9 +1045,9 @@ The bitstream starts with initial state values,
 each using the required number of bits in their respective _accuracy_,
 decoded previously from their normalized distribution.
 
-It starts by `Literal Length State`,
-followed by `Offset State`,
-and finally `Match Length State`.
+It starts by `Literals_Length_State`,
+followed by `Offset_State`,
+and finally `Match_Length_State`.
 
 Reminder : always keep in mind that all values are read _backward_.
 
@@ -1055,9 +1057,9 @@ A state gives a code.
 A code provides a baseline and number of bits to add.
 See [Symbol Decoding] section for details on each symbol.
 
-Decoding starts by reading the nb of bits required to decode offset.
+Decoding starts by reading the number of bits required to decode offset.
 It then does the same for match length,
-and then for literal length.
+and then for literals length.
 
 Offset / matchLength / litLength define a sequence.
 It starts by inserting the number of literals defined by `litLength`,
@@ -1065,11 +1067,11 @@ then continue by copying `matchLength` bytes from `currentPos - offset`.
 
 The next operation is to update states.
 Using rules pre-calculated in the decoding tables,
-`Literal Length State` is updated,
-followed by `Match Length State`,
-and then `Offset State`.
+`Literals_Length_State` is updated,
+followed by `Match_Length_State`,
+and then `Offset_State`.
 
-This operation will be repeated `NbSeqs` times.
+This operation will be repeated `Number_of_Sequences` times.
 At the end, the bitstream shall be entirely consumed,
 otherwise bitstream is considered corrupted.
 
@@ -1080,7 +1082,7 @@ otherwise bitstream is considered corrupted.
 As seen in [Offset Codes], the first 3 values define a repeated offset.
 They are sorted in recency order, with 1 meaning "most recent one".
 
-There is an exception though, when current sequence's literal length is `0`.
+There is an exception though, when current sequence's literals length is `0`.
 In which case, repcodes are "pushed by one",
 so 1 becomes 2, 2 becomes 3,
 and 3 becomes "offset_1 - 1_byte".
@@ -1111,31 +1113,32 @@ But dictionaries created by `zstd --train` follow a format, described here.
 __Pre-requisites__ : a dictionary has a known length,
                      defined either by a buffer limit, or a file size.
 
-| Header | DictID | Stats | Content |
-| ------ | ------ | ----- | ------- |
+| `Magic_Number` | `Dictionary_ID` | `Entropy_Tables` | `Content` |
+| -------------- | --------------- | ---------------- | --------- |
 
-__Header__ : 4 bytes ID, value 0xEC30A437, Little-Endian format
+__`Magic_Number`__ : 4 bytes ID, value 0xEC30A437, little-endian format
 
-__Dict_ID__ : 4 bytes, stored in Little-Endian format.
-              DictID can be any value, except 0 (which means no DictID).
+__`Dictionary_ID`__ : 4 bytes, stored in little-endian format.
+              `Dictionary_ID` can be any value, except 0 (which means no `Dictionary_ID`).
               It's used by decoders to check if they use the correct dictionary.
-              _Reserved ranges :_
+
+_Reserved ranges :_
               If the frame is going to be distributed in a private environment,
-              any dictionary ID can be used.
+              any `Dictionary_ID` can be used.
               However, for public distribution of compressed frames,
-              some ranges are reserved for future use :
+              the following ranges are reserved for future use and should not be used :
 
-              - low range : 1 - 32767 : reserved
-              - high range : >= (2^31) : reserved
+              - low range : 1 - 32767 
+              - high range : >= (2^31) 
 
-__Stats__ : Entropy tables, following the same format as a [compressed blocks].
+__`Entropy_Tables`__ : following the same format as a [compressed blocks].
             They are stored in following order :
             Huffman tables for literals, FSE table for offset,
             FSE table for matchLenth, and FSE table for litLength.
             It's finally followed by 3 offset values, populating recent offsets,
             stored in order, 4-bytes little-endian each, for a total of 12 bytes.
 
-__Content__ : Where the actual dictionary content is.
+__`Content`__ : Where the actual dictionary content is.
               Content size depends on Dictionary size.
 
 [compressed blocks]: #the-format-of-compressed_block
@@ -1144,6 +1147,6 @@ __Content__ : Where the actual dictionary content is.
 Version changes
 ---------------
 - 0.2.0 : numerous format adjustments for zstd v0.8
-- 0.1.2 : limit huffman tree depth to 11 bits
+- 0.1.2 : limit Huffman tree depth to 11 bits
 - 0.1.1 : reserved dictID ranges
 - 0.1.0 : initial release