Merge pull request #197 from inikep/dev070

improved Travis tests, skippable frames
2016-06-01 13:41:19 +02:00
parent c30d8d50da 8b452453ce
commit 38a0215b53
28 changed files with 295 additions and 326 deletions
@@ -31,5 +31,7 @@ projects/VS2015
 .directory
 _codelite/
 _zstdbench/
-zlib_wrapper/
 .clang_complete
+
+# CMake
+projects/cmake/
@@ -1,34 +1,97 @@
 language: c
-
-before_install:
-  - sudo apt-get update  -qq
-  - sudo apt-get install -qq clang
-  - sudo apt-get install -qq g++-multilib
-  - sudo apt-get install -qq gcc-multilib
-  - sudo apt-get install -qq valgrind
-
-env:
-  - ZSTD_TRAVIS_CI_ENV=travis-install
-  - ZSTD_TRAVIS_CI_ENV=cmaketest
-  - ZSTD_TRAVIS_CI_ENV=clangtest
-  - ZSTD_TRAVIS_CI_ENV=gpptest
-  - ZSTD_TRAVIS_CI_ENV=gnu90test
-  - ZSTD_TRAVIS_CI_ENV=c99test
-  - ZSTD_TRAVIS_CI_ENV=gnu99test
-  - ZSTD_TRAVIS_CI_ENV=armtest-w-install
-  - ZSTD_TRAVIS_CI_ENV=test
-  - ZSTD_TRAVIS_CI_ENV="-C programs test32"
-  - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy"
-  #- ZSTD_TRAVIS_CI_ENV="-C versionsTest"  
-  - ZSTD_TRAVIS_CI_ENV=usan
-  - ZSTD_TRAVIS_CI_ENV=asan
-  - ZSTD_TRAVIS_CI_ENV=asan32
-  - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest"
-
 compiler: gcc
-
-script:
-  - make $ZSTD_TRAVIS_CI_ENV
-
 matrix:
  fast_finish: true
+  include: 
+    # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
+    - os: linux  
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
+    # Standard Ubuntu 12.04 LTS Server Edition 64 bit
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest"
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
+    #- os: linux
+    #  sudo: required
+    #  env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=zlibwrapper
+    # Ubuntu 14.04 LTS Server Edition 64 bit
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
+    # OS X Mavericks
+    - os: osx  
+      env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
+    - os: osx
+      env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
+    - os: osx
+      env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
+  exclude:
+    - compiler: gcc
+
+before_install:
+  - set -e
+  - |
+    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+      CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
+      echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
+      if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
+        sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
+      fi
+    fi
+
+script: 
+  - make $MAKE_PARAM
+
@@ -32,6 +32,7 @@

 PRGDIR  = programs
 ZSTDDIR = lib
+ZWRAPDIR = zlibWrapper

 # Define nul output
 ifneq (,$(filter Windows%,$(OS)))
@@ -40,7 +41,7 @@ else
 VOID = /dev/null
 endif

-.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
+.PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan

 default: zstdprogram

@@ -51,12 +52,17 @@ all:
 zstdprogram:
 	$(MAKE) -C $(PRGDIR)

+zlibwrapper:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(ZWRAPDIR) all
+
 test:
 	$(MAKE) -C $(PRGDIR) $@

 clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
 	@echo Cleaning completed


@@ -118,10 +124,7 @@ armtest: clean

 # for Travis CI
 arminstall: clean
-	sudo apt-get install -q qemu
-	sudo apt-get install -q binfmt-support
-	sudo apt-get install -q qemu-user-static
-	sudo apt-get install -q gcc-arm-linux-gnueabi
+	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi

 # for Travis CI
 armtest-w-install: clean arminstall armtest
@@ -132,15 +135,23 @@ ppctest: clean

 # for Travis CI
 ppcinstall: clean
-	sudo apt-get install -q qemu
-	sudo apt-get install -q binfmt-support
-	sudo apt-get install -q qemu-user-static
-	sudo apt-get update  -q
-	sudo apt-get install -q gcc-powerpc-linux-gnu   # unfortunately, doesn't work on Travis CI (package not available)
+	sudo apt-get update  -y -q
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04

 # for Travis CI
 ppctest-w-install: clean ppcinstall ppctest

+ppc64test: clean
+	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
+	$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" 
+
+ppc64install: clean
+	sudo apt-get update  -y -q
+	sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
+	update-binfmts --displ
+
+ppc64test-w-install: clean ppc64install ppc64test
+
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"

@@ -1,24 +1,24 @@
 version: 1.0.{build}
 environment:
  matrix:
+  - COMPILER: "visual"
+    CONFIGURATION: "Debug"
+    PLATFORM: "x64"
+  - COMPILER: "visual"
+    CONFIGURATION: "Debug"
+    PLATFORM: "Win32"
+  - COMPILER: "visual"
+    CONFIGURATION: "Release"
+    PLATFORM: "x64"
+  - COMPILER: "visual"
+    CONFIGURATION: "Release"
+    PLATFORM: "Win32"
  - COMPILER: "gcc"
    MAKE_PARAMS: "test"
    PLATFORM: "mingw64"
  - COMPILER: "gcc"
    MAKE_PARAMS: "test"
    PLATFORM: "mingw32"
-  - COMPILER: "visual"
-    CONFIGURATION: "Release"
-    PLATFORM: "Win32"
-  - COMPILER: "visual"
-    CONFIGURATION: "Release"
-    PLATFORM: "x64"
-  - COMPILER: "visual"
-    CONFIGURATION: "Debug"
-    PLATFORM: "Win32"
-  - COMPILER: "visual"
-    CONFIGURATION: "Debug"
-    PLATFORM: "x64"

 install:
  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
@@ -80,7 +80,7 @@ build_script:
      ECHO *** &&
      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\
    )

 test_script:
@@ -91,12 +91,11 @@ test_script:
    )
  - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
      CD programs &&
-      SET ZSTD=..\projects\zstd.exe &&
-      SET DATAGEN=..\projects\datagen.exe &&
-      CALL tests\playTests.bat --test-large-data &&
+      SET ZSTD=./zstd.exe &&
+      sh -e playTests.sh --test-large-data &&
+      fullbench.exe -i1 &&
+      fullbench.exe -i1 -P0 &&
      CD .. &&
-      projects\fullbench.exe -i1 &&
-      projects\fullbench.exe -i1 -P0 &&
      projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
      projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
      projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
@@ -253,6 +253,6 @@ typedef struct {

 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
-
+int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);

 #endif   /* ZSTD_CCOMMON_H_MODULE */
@@ -51,7 +51,8 @@ extern "C" {
 /*-*************************************
 *  Constants
 ***************************************/
-#define ZSTD_MAGICNUMBER 0xFD2FB527   /* v0.7 */
+#define ZSTD_MAGICNUMBER            0xFD2FB526   /* v0.6 */
+#define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U


 /*-*************************************
@@ -203,6 +204,7 @@ typedef struct {
 #define ZSTD_FRAMEHEADERSIZE_MAX 18    /* for static allocation */
 static const size_t ZSTD_frameHeaderSize_min = 6;
 static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
+static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
 ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input */

 ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
@@ -218,12 +218,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                    break;
                }
                if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
+                    const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
                    size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
-                        zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
+                        zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
                        ip, neededInSize);
                    if (ZSTD_isError(decodedSize)) return decodedSize;
                    ip += neededInSize;
-                    if (!decodedSize) break;   /* this was just a header */
+                    if (!decodedSize && !isSkipFrame) break;   /* this was just a header */
                    zbd->outEnd = zbd->outStart +  decodedSize;
                    zbd->stage = ZBUFFds_flush;
                    break;
@@ -243,12 +244,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                if (loadedSize < toLoad) { notDone = 0; break; }   /* not enough input, wait for more */

                /* decode loaded input */
-                {   size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
+                {  const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
+                   size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
                        zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
                        zbd->inBuff, neededInSize);
                    if (ZSTD_isError(decodedSize)) return decodedSize;
                    zbd->inPos = 0;   /* input is consumed */
-                    if (!decodedSize) { zbd->stage = ZBUFFds_read; break; }   /* this was just a header */
+                    if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; }   /* this was just a header */
                    zbd->outEnd = zbd->outStart +  decodedSize;
                    zbd->stage = ZBUFFds_flush;
                    // break; /* ZBUFFds_flush follows */
@@ -276,7 +278,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
    *srcSizePtr = ip-istart;
    *dstCapacityPtr = op-ostart;
    {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
-        if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize;   /* get following block header too */
+//        if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize;   /* get following block header too */
        nextSrcSizeHint -= zbd->inPos;   /* already loaded*/
        return nextSrcSizeHint;
    }
@@ -103,7 +103,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
 *   Context management
 ***************************************************************/
 typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
-               ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
+               ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
+               ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;

 struct ZSTD_DCtx_s
 {
@@ -329,7 +330,15 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
    const BYTE* ip = (const BYTE*)src;

    if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
-    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
+        if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+            if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
+            fparamsPtr->frameContentSize = 0;
+            fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;
+            return 0;
+        }
+        return ERROR(prefix_unknown);
+    }

    /* ensure there is enough `srcSize` to fully read/decode frame header */
    { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
@@ -1028,6 +1037,11 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
    return dctx->expected;
 }

+int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
+{
+    return dctx->stage == ZSTDds_skipFrame;
+}
+
 /** ZSTD_decompressContinue() :
 *   @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
 *             or an error code, which can be tested using ZSTD_isError() */
@@ -1042,6 +1056,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
    {
    case ZSTDds_getFrameHeaderSize :
        if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);   /* impossible */
+        if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+            memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
+            dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
+            dctx->stage = ZSTDds_decodeSkippableHeader;
+            return 0;
+        }
        dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
        if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
        memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
@@ -1108,6 +1128,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
            if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
            return rSize;
        }
+    case ZSTDds_decodeSkippableHeader:
+        {   memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
+            dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
+            dctx->stage = ZSTDds_skipFrame;
+            return 0;
+        }
+    case ZSTDds_skipFrame:
+        {   dctx->expected = 0;
+            dctx->stage = ZSTDds_getFrameHeaderSize;
+            return 0;
+        }
    default:
        return ERROR(GENERIC);   /* impossible */
    }
@@ -38,7 +38,7 @@ MANDIR  = $(PREFIX)/share/man/man1

 ZSTDDIR = ../lib

-CPPFLAGS= -I$(ZSTDDIR)/common -DXXH_NAMESPACE=ZSTD_
+CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
 CFLAGS ?= -O3  # -falign-loops=32   # not always beneficial
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
@@ -208,7 +208,7 @@ endif
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 zstd-playTests: datagen
-	ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST)
+	ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)

 test: test-zstd test-fullbench test-fuzzer test-zbuff

@@ -32,7 +32,7 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
-#include "../lib/dictBuilder/zdict_static.h"   /* ZDICT_params_t */
+#include "zdict_static.h"   /* ZDICT_params_t */


 /*-*************************************
@@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
                continue;
            }
 #endif
-            if (magic !=  ZSTD_MAGICNUMBER) {
+            if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) {
                if (g_overwrite)   /* -df : pass-through mode */
                    return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
                else {
@@ -15,22 +15,30 @@ roundTripTest() {

    rm -f tmp1 tmp2
    $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
-    ./datagen $1 $p | md5sum > tmp1
-    ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d  | md5sum > tmp2
+    ./datagen $1 $p | $MD5SUM > tmp1
+    ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d  | $MD5SUM > tmp2
    diff -q tmp1 tmp2
 }

 isWindows=false
 ECHO="echo"
-if [[ "$OS" == "Windows"* ]]; then
+case "$OS" in
+  Windows*) 
    isWindows=true
    ECHO="echo -e"
+    ;;
+esac
+
+MD5SUM="md5sum"
+if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+    MD5SUM="md5 -r"
 fi

-$ECHO "\nStarting playTests.sh isWindows=$isWindows"
+$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME"

 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"

+file $ZSTD
 $ECHO "\n**** simple tests **** "

 ./datagen > tmp
@@ -120,8 +128,8 @@ rm tmpSparse*
 $ECHO "\n**** dictionary tests **** "

 ./datagen > tmpDict
-./datagen -g1M | md5sum > tmp1
-./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
+./datagen -g1M | $MD5SUM > tmp1
+./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
 diff -q tmp1 tmp2
 $ECHO "Create first dictionary"
 $ZSTD --train *.c -o tmpDict
@@ -1,157 +0,0 @@
-@echo off
-ECHO Start playTests.bat
-ECHO ZSTD^: %ZSTD%
-ECHO DATAGEN^: %DATAGEN%
-if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1
-if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1
-
-SET ROUNDTRIPTEST=tests\roundTripTest.bat
-
-echo. && echo **** simple tests ****
-%DATAGEN% > tmp
-%ZSTD% -f tmp                             && REM trivial compression case, creates tmp.zst
-%ZSTD% -df tmp.zst                        && REM trivial decompression case (overwrites tmp)
-echo test : too large compression level (must fail)
-%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1)
-echo test : compress to stdout
-%ZSTD% tmp -c > tmpCompressed                 
-%ZSTD% tmp --stdout > tmpCompressed       && REM long command format
-echo test : null-length file roundtrip
-echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1)
-echo test : decompress file with wrong suffix (must fail)
-%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1)
-%ZSTD% -d tmpCompressed -c > tmpResult    && REM decompression using stdout   
-%ZSTD% --decompress tmpCompressed -c > tmpResult
-%ZSTD% --decompress tmpCompressed --stdout > tmpResult
-REM %ZSTD% -d    < tmp.zst > NUL              && REM combine decompression, stdin & stdout
-REM %ZSTD% -d  - < tmp.zst > NUL
-%ZSTD% -dc   < tmp.zst > NUL
-%ZSTD% -dc - < tmp.zst > NUL
-%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1)
-%ZSTD% -q -f tmp
-%ZSTD% -q --force tmp
-%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1)
-
-echo. && echo **** Pass-Through mode ****
-echo "Hello world !" | %ZSTD% -df
-echo "Hello world !" | %ZSTD% -dcf
-
-
-echo. && echo **** frame concatenation ****
-
-echo hello > hello.tmp
-echo world! > world.tmp
-cat hello.tmp world.tmp > helloworld.tmp
-%ZSTD% -c hello.tmp > hello.zstd
-%ZSTD% -c world.tmp > world.zstd
-cat hello.zstd world.zstd > helloworld.zstd
-%ZSTD% -dc helloworld.zstd > result.tmp
-cat result.tmp
-fc /b helloworld.tmp result.tmp
-rm *.tmp *.zstd
-
-echo frame concatenation tests completed
-
-
-REM echo. && echo **** flush write error test ****
-
-REM echo echo foo ^| %ZSTD% ^> v:\full
-REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1)
-REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full"
-REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1)
-
-
-echo. && echo **** dictionary tests ****
-
-%DATAGEN% > tmpDict
-%DATAGEN% -g1M | md5sum > tmp1
-%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2
-fc tmp1 tmp2
-%ZSTD% --train *.c *.h -o tmpDict
-%ZSTD% xxhash.c -D tmpDict -of tmp
-%ZSTD% -d tmp -D tmpDict -of result
-fc xxhash.c result
-
-
-echo. && echo **** multiple files tests ****
-
-%DATAGEN% -s1        > tmp1 2> NUL
-%DATAGEN% -s2 -g100K > tmp2 2> NUL
-%DATAGEN% -s3 -g1M   > tmp3 2> NUL
-%ZSTD% -f tmp*
-echo compress tmp* :
-ls -ls tmp*
-rm tmp1 tmp2 tmp3
-echo decompress tmp* :
-%ZSTD% -df *.zst
-ls -ls tmp*
-
-echo compress tmp* into stdout ^> tmpall :
-%ZSTD% -c tmp1 tmp2 tmp3 > tmpall
-ls -ls tmp*
-echo decompress tmpall* into stdout ^> tmpdec :
-cp tmpall tmpall2
-%ZSTD% -dc tmpall* > tmpdec
-ls -ls tmp*
-echo compress multiple files including a missing one (notHere) :
-%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1)
-
-
-echo. && echo **** integrity tests ****
-echo test one file (tmp1.zst)
-%ZSTD% -t tmp1.zst
-%ZSTD% --test tmp1.zst
-echo test multiple files (*.zst)
-%ZSTD% -t *.zst
-echo test good and bad files (*)
-%ZSTD% -t * && (echo bad files not detected! && exit /b 1)
-
-
-echo. && echo **** zstd round-trip tests ****
-
-CALL %ROUNDTRIPTEST%
-CALL %ROUNDTRIPTEST% -g15K       && REM TableID==3
-CALL %ROUNDTRIPTEST% -g127K      && REM TableID==2
-CALL %ROUNDTRIPTEST% -g255K      && REM TableID==1
-CALL %ROUNDTRIPTEST% -g513K      && REM TableID==0
-CALL %ROUNDTRIPTEST% -g512K 6    && REM greedy, hash chain
-CALL %ROUNDTRIPTEST% -g512K 16   && REM btlazy2 
-CALL %ROUNDTRIPTEST% -g512K 19   && REM btopt
-
-rm tmp*
-echo Param = %1
-if NOT "%1"=="--test-large-data" (
-    echo skipping large data tests
-    exit /b 0
-)
-
-CALL %ROUNDTRIPTEST% -g270000000 1
-CALL %ROUNDTRIPTEST% -g270000000 2
-CALL %ROUNDTRIPTEST% -g270000000 3
-
-CALL %ROUNDTRIPTEST% -g140000000 -P60 4
-CALL %ROUNDTRIPTEST% -g140000000 -P60 5
-CALL %ROUNDTRIPTEST% -g140000000 -P60 6
-
-CALL %ROUNDTRIPTEST% -g70000000 -P70 7
-CALL %ROUNDTRIPTEST% -g70000000 -P70 8
-CALL %ROUNDTRIPTEST% -g70000000 -P70 9
-
-CALL %ROUNDTRIPTEST% -g35000000 -P75 10
-CALL %ROUNDTRIPTEST% -g35000000 -P75 11
-CALL %ROUNDTRIPTEST% -g35000000 -P75 12
-
-CALL %ROUNDTRIPTEST% -g18000000 -P80 13
-CALL %ROUNDTRIPTEST% -g18000000 -P80 14
-CALL %ROUNDTRIPTEST% -g18000000 -P80 15
-CALL %ROUNDTRIPTEST% -g18000000 -P80 16
-CALL %ROUNDTRIPTEST% -g18000000 -P80 17
-
-CALL %ROUNDTRIPTEST% -g50000000 -P94 18
-CALL %ROUNDTRIPTEST% -g50000000 -P94 19
-
-CALL %ROUNDTRIPTEST% -g99000000 -P99 20
-CALL %ROUNDTRIPTEST% -g6000000000 -P99 1
-
-rm tmp*
-exit /b 0
@@ -1,8 +0,0 @@
-@echo off
-if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) 
-rm -f tmp1 tmp2
-echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d
-%DATAGEN% %1 %P% | md5sum > tmp1
-%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d  | md5sum > tmp2
-fc tmp1 tmp2
-EXIT /B %ERRORLEVEL%
@@ -147,11 +147,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    int testResult = 0;
    size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
    void* CNBuffer = malloc(CNBufferSize);
-    size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
-    void* const compressedBuffer = malloc(compressedBufferSize);
+    size_t const skippableFrameSize = 11;
+    size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
+    void* compressedBuffer = malloc(compressedBufferSize);
    size_t const decodedBufferSize = CNBufferSize;
-    void* const decodedBuffer = malloc(decodedBufferSize);
-    size_t cSize, readSize, genSize;
+    void* decodedBuffer = malloc(decodedBufferSize);
+    size_t cSize, readSize, readSkipSize, genSize;
    U32 testNb=0;
    ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
    ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
@@ -163,15 +164,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    }
    RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);

+    /* generate skippable frame */
+    MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
+    MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
+    cSize = skippableFrameSize + 8;
    /* Basic compression test */
    DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
    ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
    readSize = CNBufferSize;
    genSize = compressedBufferSize;
-    { size_t const r = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
+    { size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
      if (ZBUFF_isError(r)) goto _output_error; }
    if (readSize != CNBufferSize) goto _output_error;   /* entire input should be consumed */
-    cSize = genSize;
+    cSize += genSize;
    genSize = compressedBufferSize - cSize;
    { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
      if (r != 0) goto _output_error; }  /*< error, or some data not flushed */
@@ -181,12 +186,18 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    /* Basic decompression test */
    DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
-    readSize = cSize;
+    readSkipSize = cSize;
    genSize = CNBufferSize;
-    { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
+    { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
+      if (r != 0) goto _output_error; }
+    if (genSize != 0) goto _output_error;   /* skippable frame len is 0 */
+    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
+    readSize = cSize - readSkipSize;
+    genSize = CNBufferSize;
+    { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
      if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
    if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
-    if (readSize != cSize) goto _output_error;   /* should have read the entire frame */
+    if (readSize+readSkipSize != cSize) goto _output_error;   /* should have read the entire frame */
    DISPLAYLEVEL(4, "OK \n");

    /* check regenerated data is byte exact */
@@ -200,17 +211,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo

    /* Byte-by-byte decompression test */
    DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
-    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
-    {   size_t r = 1, pIn=0, pOut=0;
-        while (r) {
-            size_t inS = 1;
-            size_t outS = 1;
-            r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
-            pIn += inS;
-            pOut += outS;
-        }
-        readSize = pIn;
-        genSize = pOut;
+    {   size_t r, pIn=0, pOut=0;
+        do 
+        {   ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
+            r = 1;
+            while (r) {
+                size_t inS = 1;
+                size_t outS = 1;
+                r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
+                pIn += inS;
+                pOut += outS;
+            }
+            readSize = pIn;
+            genSize = pOut;
+        } while (genSize==0);
    }
    if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
    if (readSize != cSize) goto _output_error;   /* should have read the entire frame */
@@ -52,7 +52,7 @@
 #  include <io.h>       /* _isatty */
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
 #else
-#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
+#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE)
 #  include <unistd.h>   /* isatty */
 #  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
 #else
@@ -344,6 +344,10 @@
 				RelativePath="..\..\..\lib\common\fse_decompress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\xxhash.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\programs\fullbench.c"
 				>
@@ -415,6 +419,10 @@
 				>
 			</File>
 			<File
+				RelativePath="..\..\..\lib\common\xxhash.h"
+				>
+			</File>
+            <File
 				RelativePath="..\..\..\lib\common\mem.h"
 				>
 			</File>
@@ -44,7 +44,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -121,7 +121,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -196,7 +196,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -274,7 +274,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -44,7 +44,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -120,7 +120,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -194,7 +194,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -271,7 +271,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <ItemGroup Label="ProjectConfigurations">
    <ProjectConfiguration Include="Debug|Win32">
@@ -159,6 +159,7 @@
    <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
    <ClCompile Include="..\..\..\lib\common\xxhash.c" />
    <ClCompile Include="..\..\..\lib\common\zstd_common.c" />
+    <ClCompile Include="..\..\..\lib\common\xxhash.c" />
    <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
    <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
    <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <ItemGroup Label="ProjectConfigurations">
    <ProjectConfiguration Include="Debug|Win32">
@@ -119,27 +119,27 @@
  <PropertyGroup Label="UserMacros" />
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
    <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
    <LibraryPath>$(LibraryPath)</LibraryPath>
    <IntDir>$(Platform)\$(Configuration)\</IntDir>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
    <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
    <LibraryPath>$(LibraryPath);</LibraryPath>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
    <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
    <LibraryPath>$(LibraryPath)</LibraryPath>
    <IntDir>$(Platform)\$(Configuration)\</IntDir>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
    <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
    <LibraryPath>$(LibraryPath);</LibraryPath>
  </PropertyGroup>
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <ItemGroup Label="ProjectConfigurations">
    <ProjectConfiguration Include="Debug|Win32">
@@ -99,28 +99,28 @@
    <LinkIncremental>true</LinkIncremental>
    <TargetName>zstdlib_x86</TargetName>
    <IntDir>$(Platform)\$(Configuration)\</IntDir>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
    <LinkIncremental>true</LinkIncremental>
    <TargetName>zstdlib_x64</TargetName>
    <IntDir>$(Platform)\$(Configuration)\</IntDir>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
    <LinkIncremental>false</LinkIncremental>
    <TargetName>zstdlib_x86</TargetName>
    <IntDir>$(Platform)\$(Configuration)\</IntDir>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
    <LinkIncremental>false</LinkIncremental>
    <TargetName>zstdlib_x64</TargetName>
    <IntDir>$(Platform)\$(Configuration)\</IntDir>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
    <RunCodeAnalysis>false</RunCodeAnalysis>
  </PropertyGroup>
  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
@@ -44,13 +44,6 @@ ELSE (ZSTD_LEGACY_SUPPORT)
    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
 ENDIF (ZSTD_LEGACY_SUPPORT)

-# TARGET_INCLUDE_DIRECTORIES can use in version 2.8.11 and greate
-IF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
-    SET(WORKAROUND_OUTDATED_CODE_STYLE TRUE)
-ELSE ()
-    SET(WORKAROUND_OUTDATED_CODE_STYLE FALSE)
-ENDIF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
-
 ADD_SUBDIRECTORY(lib)
 ADD_SUBDIRECTORY(programs)

@@ -96,16 +96,23 @@ IF (ZSTD_LEGACY_SUPPORT)
            ${LIBRARY_LEGACY_DIR}/zstd_v01.c
            ${LIBRARY_LEGACY_DIR}/zstd_v02.c
            ${LIBRARY_LEGACY_DIR}/zstd_v03.c
-            ${LIBRARY_LEGACY_DIR}/zstd_v04.c)
+            ${LIBRARY_LEGACY_DIR}/zstd_v04.c
+            ${LIBRARY_LEGACY_DIR}/zstd_v05.c)

    SET(Headers ${Headers}
            ${LIBRARY_LEGACY_DIR}/zstd_legacy.h
            ${LIBRARY_LEGACY_DIR}/zstd_v01.h
            ${LIBRARY_LEGACY_DIR}/zstd_v02.h
            ${LIBRARY_LEGACY_DIR}/zstd_v03.h
-            ${LIBRARY_LEGACY_DIR}/zstd_v04.h)
+            ${LIBRARY_LEGACY_DIR}/zstd_v04.h
+            ${LIBRARY_LEGACY_DIR}/zstd_v05.h)
 ENDIF (ZSTD_LEGACY_SUPPORT)

+IF (MSVC)
+    SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib)
+    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc)
+ENDIF (MSVC)
+
 # Split project to static and shared libraries build
 ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources})
 ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
@@ -116,16 +123,6 @@ IF (MSVC)
    SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
 ENDIF (MSVC)

-# Define include directories
-IF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
-    TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common)
-    TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common)
-    IF (ZSTD_LEGACY_SUPPORT)
-        TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR})
-        TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR})
-    ENDIF (ZSTD_LEGACY_SUPPORT)
-ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
-
 # Define library base name
 IF (MSVC)
    SET(LIBRARY_BASE_NAME zstdlib)
@@ -44,12 +44,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBui

 IF (ZSTD_LEGACY_SUPPORT)
    SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
-    INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR})
-
-    IF (WORKAROUND_OUTDATED_CODE_STYLE)
-        INCLUDE_DIRECTORIES(${LIBRARY_DIR}/legacy)
-    ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
-
+    INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
    SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c)
 ENDIF (ZSTD_LEGACY_SUPPORT)

@@ -15,14 +15,14 @@ IMPLIB    = -lz ../lib/libzstd.a
 endif

 ZLIBWRAPPER_PATH = .
-EXAMPLE_PATH = examples/
+EXAMPLE_PATH = examples
 CC = gcc
 CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89
 LDFLAGS = $(LOC)
 RM = rm -f


-all: example example_d
+all: clean test testdll testzstd

 test: example
 	./example
@@ -30,6 +30,9 @@ test: example
 testdll: example_d
 	./example_d

+testzstd: example_zstd
+	./example_zstd
+
 .c.o:
 	$(CC) $(CFLAGS) -c -o $@ $<

@@ -39,12 +42,18 @@ example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
 example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
 	$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB)

+example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o
+	$(CC) $(LDFLAGS)  -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB)
+
 $(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c
 	$(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c

 $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
 	$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c

+$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
+	$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
+
 clean:
 	-$(RM) $(ZLIBWRAPPER_PATH)/*.o
 	-$(RM) $(EXAMPLE_PATH)/*.o
@@ -35,17 +35,18 @@
 #include "zstd_zlibwrapper.h"
 #include "zstd.h"
 #include "zstd_static.h"      /* ZSTD_MAGICNUMBER */
-#include "zstd_internal.h"    /* MIN */
 #include "zbuff.h"


 #define Z_INFLATE_SYNC              8
 #define ZWRAP_HEADERSIZE            4
-#define ZSTD_FRAMEHEADERSIZE_MIN    5
+#define ZWRAP_DEFAULT_CLEVEL        5   /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */

 #define LOG_WRAPPER(...)  // printf(__VA_ARGS__)


+#define MIN(a,b) ((a)<(b)?(a):(b))
+
 #define FINISH_WITH_ERR(msg) { \
    fprintf(stderr, "ERROR: %s\n", msg); \
    return Z_MEM_ERROR; \
@@ -116,6 +117,9 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
    zwc = ZWRAP_createCCtx();
    if (zwc == NULL) return Z_MEM_ERROR;

+    if (level == Z_DEFAULT_COMPRESSION)
+        level = ZWRAP_DEFAULT_CLEVEL;
+
    { size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level);
      if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }

@@ -127,7 +131,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
 }


-ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int  level, int  method,
+ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method,
                                      int windowBits, int memLevel,
                                      int strategy, const char *version,
                                      int stream_size))
@@ -253,9 +257,8 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm,

 typedef struct {
    ZBUFF_DCtx* zbd;
-    char headerBuf[ZSTD_FRAMEHEADERSIZE_MIN];
+    char headerBuf[ZWRAP_HEADERSIZE];
    int errorCount;
-    int requiredHeaderSize;
    
    /* zlib params */
    int stream_size;
@@ -298,7 +301,6 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
    zwd->zalloc = strm->zalloc;
    zwd->zfree = strm->zfree;
    zwd->opaque = strm->opaque;
-    zwd->requiredHeaderSize = ZWRAP_HEADERSIZE;

    strm->state = (struct internal_state*) zwd;
    strm->total_in = 0;
@@ -360,14 +362,14 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
        size_t errorCode, dstCapacity, srcSize;
        ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
        LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
-        while (strm->total_in < ZSTD_FRAMEHEADERSIZE_MIN)
+        if (strm->total_in < ZWRAP_HEADERSIZE)
        {
-            srcSize = MIN(strm->avail_in, zwd->requiredHeaderSize - strm->total_in);
+            srcSize = MIN(strm->avail_in, ZWRAP_HEADERSIZE - strm->total_in);
            memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
            strm->total_in += srcSize;
            strm->next_in += srcSize;
            strm->avail_in -= srcSize;
-            if (strm->total_in < zwd->requiredHeaderSize) return Z_OK;
+            if (strm->total_in < ZWRAP_HEADERSIZE) return Z_OK;

            if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
                z_stream strm2;
@@ -401,16 +403,11 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                return inflate(strm, flush);
            }

-            if (zwd->requiredHeaderSize < ZSTD_FRAMEHEADERSIZE_MIN) {
-                zwd->requiredHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
-                continue;
-            }
-
            zwd->zbd = ZBUFF_createDCtx();
            { size_t const errorCode = ZBUFF_decompressInit(zwd->zbd);
              if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }

-            srcSize = ZSTD_FRAMEHEADERSIZE_MIN;
+            srcSize = ZWRAP_HEADERSIZE;
            dstCapacity = 0;
            errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize);
            LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
@@ -419,7 +416,6 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                return Z_MEM_ERROR;
            }
            if (strm->avail_in == 0) return Z_OK;
-            break;
        }

        srcSize = strm->avail_in;
@@ -40,6 +40,9 @@ extern "C" {
 #define Z_PREFIX
 #include <zlib.h>

+#if !defined(z_const)
+#  define z_const const 
+#endif

 void useZSTD(int turn_on);
 int isUsingZSTD();