Merge branch 'dev'

Prevent fuzz testers from combining large dicts with high clevel

.buckconfig

@@ -0,0 +1,9 @@
+[cxx]
+  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
+  cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
+  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
+  cxxflags = -std=c++11 -Wno-deprecated-declarations
+  gtest_dep = //contrib/pzstd:gtest
+
+[httpserver]
+  port = 0

.buckversion

@@ -0,0 +1 @@
+c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b

.gitattributes

@@ -19,3 +19,6 @@
 # Windows
 *.bat text eol=crlf
 *.cmd text eol=crlf
+
+# .travis.yml merging
+.travis.yml merge=ours

.gitignore

@@ -15,6 +15,7 @@
 
 # Executables
 zstd
+zstdmt
 *.exe
 *.out
 *.app
@@ -36,3 +37,5 @@ googletest/
 
 # Directories
 bin/
+.buckd/
+buck-out/

.travis.yml

@@ -1,148 +1,51 @@
+# Long tests: run on commits to master branch/cron builds
+
 language: c
+sudo: required
+dist: trusty
 matrix:
-  fast_finish: true
   include:
-    # OS X Mavericks
-    - env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
-      os: osx
+    # Ubuntu 14.04
+    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test'
+    - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test32'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
+    - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'
+
+    - env: Cmd='make arminstall && make armtest'
+    - env: Cmd='make arminstall && make aarch64test'
+    - env: Cmd='make ppcinstall && make ppctest'
+    - env: Cmd='make ppcinstall && make ppc64test'
 
 
-    # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
-    - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install"
-      os: linux
-      sudo: false
-
-    - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
-      os: linux
-      sudo: false
-      language: cpp
-      install:
-        - export CXX="g++-4.8" CC="gcc-4.8"
-        - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - gcc-4.8
-            - g++-4.8
-
-    - env: Ubu=12.04cont Cmd="make usan"
-      os: linux
-      sudo: false
-
-    - env: Ubu=12.04cont Cmd="make asan"
-      os: linux
-      sudo: false
-
-
-    # Standard Ubuntu 12.04 LTS Server Edition 64 bit
-    - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C tests test-gzstd && make -C programs clean && make -C tests versionsTest"
-      os: linux
-      sudo: required
-
-    - env: Ubu=12.04 Cmd="make asan32"
-      os: linux
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - libc6-dev-i386
-            - gcc-multilib
-
-    - env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..'
-      os: linux
-      sudo: required
+    - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
+    - env: Cmd='make -C tests versionsTest'
+    - env: Cmd='make gpp6install && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
       install:
         - export CXX="g++-6" CC="gcc-6"
-        - export LDFLAGS="-fuse-ld=gold"
-        - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - gcc-6
-            - g++-6
+    - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zstd-nolegacy'
+    - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zbuff'
 
+    # OS X Mavericks
+    - env: Cmd="make gnu90build && make clean && make test && make clean && make travis-install"
+      os: osx
 
-    # Ubuntu 14.04 LTS Server Edition 64 bit
-    - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test"
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - qemu-system-arm
-            - qemu-user-static
-            - gcc-arm-linux-gnueabi
-            - libc6-dev-armel-cross 
-            - gcc-aarch64-linux-gnu
-            - libc6-dev-arm64-cross
+git:
+  depth: 1
 
-    - env: Ubu=14.04 Cmd='make ppctest && make clean && make ppc64test'
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - qemu-system-ppc
-            - qemu-user-static
-            - gcc-powerpc-linux-gnu
-
-    - env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
-      os: linux
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - valgrind
-
-    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
-      os: linux
-      dist: trusty
-      sudo: required
-      install:
-        - export CXX="g++-4.8" CC="gcc-4.8"
-      addons:
-        apt:
-          packages:
-            - libc6-dev-i386
-            - g++-multilib
-            - gcc-4.8
-            - gcc-4.8-multilib
-            - g++-4.8
-            - g++-4.8-multilib
-
-    - env: Ubu=14.04 Cmd="make -C tests test32"
-      os: linux
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - libc6-dev-i386
-            - gcc-multilib
-
-    - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test"
-      os: linux
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - gcc-multilib
-            - gcc-5
-            - gcc-5-multilib
-            - gcc-6
-            - gcc-6-multilib
+branches:
+  only:
+  - dev
+  - master
 
 script:
   - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
-  # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi
-  - sh -c "$Cmd"
+  - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
+  - export FUZZERTEST=-T5mn;
+    export ZSTREAM_TESTTIME=-T5mn;
+    export DECODECORPUS_TESTTIME=-T1mn;
+    if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
+        git fetch origin dev;
+        git checkout -f FETCH_HEAD;
+    fi;
+    sh -c "$Cmd" || travis_terminate 1;

Makefile

@@ -23,16 +23,36 @@ EXT =
 endif
 
 .PHONY: default
-default: lib zstd
+default: lib-release zstd-release
 
 .PHONY: all
-all:
-	$(MAKE) -C $(ZSTDDIR) $@
-	$(MAKE) -C $(PRGDIR) $@ zstd32
-	$(MAKE) -C $(TESTDIR) $@ all32
+all: | allmost examples manual
+
+.PHONY: allmost
+allmost:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
+	$(MAKE) -C $(ZWRAPDIR) all
+
+#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
+.PHONY: allarch
+allarch:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
+
+.PHONY: all32
+all32:
+	$(MAKE) -C $(PRGDIR) zstd32
+	$(MAKE) -C $(TESTDIR) all32
 
 .PHONY: lib
 lib:
+	@$(MAKE) -C $(ZSTDDIR) $@
+
+.PHONY: lib-release
+lib-release:
 	@$(MAKE) -C $(ZSTDDIR)
 
 .PHONY: zstd
@@ -40,24 +60,47 @@ zstd:
 	@$(MAKE) -C $(PRGDIR) $@
 	cp $(PRGDIR)/zstd$(EXT) .
 
+.PHONY: zstd-release
+zstd-release:
+	@$(MAKE) -C $(PRGDIR)
+	cp $(PRGDIR)/zstd$(EXT) .
+
+.PHONY: zstdmt
+zstdmt:
+	@$(MAKE) -C $(PRGDIR) $@
+	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+
 .PHONY: zlibwrapper
 zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) test
 
+.PHONY: shortest
+shortest:
+	$(MAKE) -C $(TESTDIR) $@
+
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
 
+.PHONY: examples
+examples:
+	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
+
+.PHONY: manual
+manual:
+	$(MAKE) -C contrib/gen_html $@
+
 .PHONY: clean
 clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
-	@$(RM) zstd$(EXT) tmp*
+	@$(MAKE) -C examples/ $@ > $(VOID)
+	@$(MAKE) -C contrib/gen_html $@ > $(VOID)
+	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	@echo Cleaning completed
 
-
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, OSX, Hurd and some BSD targets
 #------------------------------------------------------------------------------
@@ -76,8 +119,52 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
+gppbuild: clean
+	g++ -v
+	CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+
+gcc5build: clean
+	gcc-5 -v
+	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
+
+gcc6build: clean
+	gcc-6 -v
+	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
+
+clangbuild: clean
+	clang -v
+	CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+
+m32build: clean
+	gcc -v
+	$(MAKE) all32
+
+armbuild: clean
+	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+aarch64build: clean
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+ppcbuild: clean
+	CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allarch
+
+ppc64build: clean
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
+
+armfuzz: clean
+	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+aarch64fuzz: clean
+	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppcfuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppc64fuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest
+
 gpptest: clean
-	$(MAKE) -C programs all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+	CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
 gcc5test: clean
 	gcc-5 -v
@@ -89,7 +176,7 @@ gcc6test: clean
 
 clangtest: clean
 	clang -v
-	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+	$(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
 
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
@@ -107,20 +194,69 @@ ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
+arm-ppc-compilation:
+	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
+
+# run UBsan with -fsanitize-recover=signed-integer-overflow
+# due to a bug in UBsan when doing pointer subtraction
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
+
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined"
 
 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
 
+asan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $*
+
 msan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer"   # datagen.c fails this test for no obvious reason
 
+msan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" $(MAKE) -C $(TESTDIR) $*
+
 asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined"
+
+uasan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
+
+tsan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $*
+apt-install:
+	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
+
+apt-add-repo:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+
+ppcinstall:
+	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
+
+arminstall:
+	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
+
+valgrindinstall:
+	APT_PACKAGES="valgrind" $(MAKE) apt-install
+
+libc6install:
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install
+
+gcc6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
+
+gpp6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
+
+clang38install:
+	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
 
 endif
 
@@ -135,36 +271,45 @@ endif
 #make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-cmaketest:
+cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
 	mkdir $(BUILDIR)/cmake/build
 	cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
 
-c90test: clean
-	CFLAGS="-std=c90" $(MAKE) all  # will fail, due to // and long long
+c90build: clean
+	gcc -v
+	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
-gnu90test: clean
-	CFLAGS="-std=gnu90" $(MAKE) all
+gnu90build: clean
+	gcc -v
+	CFLAGS="-std=gnu90" $(MAKE) allmost
 
-c99test: clean
-	CFLAGS="-std=c99" $(MAKE) all
+c99build: clean
+	gcc -v
+	CFLAGS="-std=c99" $(MAKE) allmost
 
-gnu99test: clean
-	CFLAGS="-std=gnu99" $(MAKE) all
+gnu99build: clean
+	gcc -v
+	CFLAGS="-std=gnu99" $(MAKE) allmost
 
-c11test: clean
-	CFLAGS="-std=c11" $(MAKE) all
+c11build: clean
+	gcc -v
+	CFLAGS="-std=c11" $(MAKE) allmost
 
-bmix64test: clean
+bmix64build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
 
-bmix32test: clean
+bmix32build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
 
-bmi32test: clean
+bmi32build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
 
 staticAnalyze: clean
+	gcc -v
 	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
 endif

NEWS

@@ -1,3 +1,35 @@
+v1.1.4
+cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
+cli : new : advanced benchmark command --priority=rt
+cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
+cli : fix : --rm remains silent when input is stdin
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
+build: new: meson build system in contrib/meson, by Dima Krasner
+build: improved cmake script, by @Majlen
+build: added -Wformat-security flag, as recommended by Padraig Brady
+doc : new : educational decoder, by Sean Purcell
+
+v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
+cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
+cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
+cli : fix zstdless on Mac OS-X, by Andrew Janke
+cli : fix #232 "compress non-files"
+dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
+API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
+API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
+API : new : ZDICT_finalizeDictionary()
+API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+build : support for Solaris target, by Przemyslaw Skibinski
+doc : clarified specification, by Sean Purcell
+
 v1.1.2
 API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
 API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()

README.md

@@ -2,7 +2,7 @@
  targeting real-time compression scenarios at zlib-level and better compression ratios.
 
 It is provided as an open-source BSD-licensed **C** library,
-and a command line utility producing and decoding `.zst` compressed files.
+and a command line utility producing and decoding `.zst` and `.gz` files.
 For other programming languages,
 you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
 
@@ -11,23 +11,26 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
 |master      | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=master)](https://travis-ci.org/facebook/zstd) |
 |dev         | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |
 
-As a reference, several fast compression algorithms were tested and compared on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.4.0, with the [Silesia compression corpus].
+As a reference, several fast compression algorithms were tested and compared
+on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`),
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench v1.6], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].
 
-[lzbench]: https://github.com/inikep/lzbench
+[lzbench v1.6]: https://github.com/inikep/lzbench
 [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 
-
-| Name                    | Ratio | C.speed | D.speed |
-|-------------------------|-------|--------:|--------:|
-|                         |       |   MB/s  |  MB/s   |
-| **zstd 0.8.2 -1**     |**2.877**| **330** | **940** |
-| [zlib] 1.2.8 deflate -1 | 2.730 |    95   |   360   |
-| brotli 0.4 -0           | 2.708 |   320   |   375   |
-| QuickLZ 1.5             | 2.237 |   510   |   605   |
-| LZO 2.09                | 2.106 |   610   |   870   |
-| [LZ4] r131              | 2.101 |   620   |  3100   |
-| Snappy 1.1.3            | 2.091 |   480   |  1600   |
-| LZF 3.6                 | 2.077 |   375   |   790   |
+| Compressor name         | Ratio | Compression| Decompress.|
+| ---------------         | ------| -----------| ---------- |
+| **zstd 1.1.3 -1**       | 2.877 |   430 MB/s |  1110 MB/s |
+| zlib 1.2.8 -1           | 2.743 |   110 MB/s |   400 MB/s |
+| brotli 0.5.2 -0         | 2.708 |   400 MB/s |   430 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |
+| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |
+| lz4 1.7.5               | 2.101 |   720 MB/s |  3600 MB/s |
+| snappy 1.1.3            | 2.091 |   500 MB/s |  1650 MB/s |
+| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |
 
 [zlib]:http://www.zlib.net/
 [LZ4]: http://www.lz4.org/
@@ -47,18 +50,27 @@ For a larger picture including very slow modes, [click on this link](doc/images/
 
 ### The case for Small Data compression
 
-Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. The smaller the amount of data to compress, the more difficult it is to achieve any significant compression.
+Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
 
-This problem is common to many compression algorithms. The reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new file, there is no "past" to build upon.
+The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon.
 
-To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored in a file called "dictionary", which can be loaded before compression and decompression. Using this dictionary, the compression ratio achievable on small data improves dramatically:
+To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data.
+Training Zstandard is achieved by provide it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression.
+Using this dictionary, the compression ratio achievable on small data improves dramatically.
 
-![Compressing Small Data](doc/images/smallData.png "Compressing Small Data")
+The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users).
+It consists of roughly 10K records weighting about 1KB each.
 
-These compression gains are achieved while simultaneously providing faster compression and decompression speeds.
+Compression Ratio | Compression Speed | Decompression Speed
+------------------|-------------------|--------------------
+![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed")
 
-Dictionary works if there is some correlation in a family of small data (there is no _universal dictionary_).
-Hence, deploying one dictionary per type of data will provide the greatest benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will rely more and more on previously decoded content to compress the rest of the file.
+
+These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds.
+
+Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_).
+Hence, deploying one dictionary per type of data will provide the greatest benefits.
+Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
 
 #### Dictionary compression How To :
 
@@ -68,11 +80,12 @@ Hence, deploying one dictionary per type of data will provide the greatest benef
 
 2) Compress with dictionary
 
-`zstd FILE -D dictionaryName`
+`zstd -D dictionaryName FILE`
 
 3) Decompress with dictionary
 
-`zstd --decompress FILE.zst -D dictionaryName`
+`zstd -D dictionaryName --decompress FILE.zst`
+
 
 ### Build
 
@@ -94,7 +107,11 @@ A `cmake` project generator is provided within `build/cmake`.
 It can generate Makefiles or other build scripts
 to create `zstd` binary, and `libzstd` dynamic and static libraries.
 
-#### Visual (Windows)
+#### Meson
+
+A Meson project is provided within `contrib/meson`.
+
+#### Visual Studio (Windows)
 
 Going into `build` directory, you will find additional possibilities :
 - Projects for Visual Studio 2005, 2008 and 2010

TESTING.md

@@ -0,0 +1,44 @@
+Testing
+=======
+
+Zstandard CI testing is split up into three sections:
+short, medium, and long tests.
+
+Short Tests
+-----------
+Short tests run on CircleCI for new commits on every branch and pull request.
+They consist of the following tests:
+- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
+- Compilation on various versions of gcc, clang, and g++
+- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
+- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64
+
+Medium Tests
+------------
+Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
+They consist of the following tests:
+- The following tests run with UBsan and Asan on x86_64 and x86, as well as with
+  Msan on x86_64
+  - `tests/playTests.sh --test-long-data`
+  - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
+- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
+- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
+- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64
+
+Long Tests
+----------
+Long tests run on all commits to `master` branch,
+and once a day on the current version of `dev` branch,
+on TravisCI.
+They consist of the following tests:
+- Entire test suite (including fuzzers and some other specialized tests) on:
+  - x86_64 and x86 with UBsan and Asan
+  - x86_64 with Msan
+  - ARM, AArch64, PowerPC, and PowerPC64
+- Streaming mode fuzzer with Tsan (for the `zstdmt` testing)
+- ZlibWrapper tests, including under valgrind
+- Versions test (ensuring `zstd` can decode files from all previous versions)
+- `pzstd` with asan and tsan, as well as in 32-bits mode
+- Testing `zstd` with legacy mode off
+- Testing `zbuff` (old streaming API)
+- Entire test suite and make install on OS X

appveyor.yml

@@ -2,11 +2,14 @@ version: 1.0.{build}
 environment:
   matrix:
   - COMPILER: "gcc"
-    MAKE_PARAMS: "test"
     PLATFORM: "mingw64"
+    MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
   - COMPILER: "gcc"
-    MAKE_PARAMS: "test"
     PLATFORM: "mingw32"
+    MAKE_PARAMS: '"make -C tests test-zstd test-fullbench test-fuzzer test-invalidDictionaries"'
+  - COMPILER: "gcc"
+    PLATFORM: "clang"
+    MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""'
   - COMPILER: "visual"
     CONFIGURATION: "Debug"
     PLATFORM: "x64"
@@ -25,7 +28,6 @@ install:
   - MKDIR bin
   - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
   - if [%COMPILER%]==[gcc] (
-      SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
       SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
       SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
       COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe &&
@@ -38,33 +40,17 @@ build_script:
   - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
   - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
-  - if [%PLATFORM%]==[mingw64] (
-      make clean &&
-      ECHO *** &&
-      ECHO *** Building clang &&
-      ECHO *** &&
-      ECHO make %CLANG_PARAMS% &&
-      make %CLANG_PARAMS% &&
-      COPY tests\fuzzer.exe tests\fuzzer_clang.exe &&
-      make clean &&
-      ECHO *** &&
-      ECHO *** Building pzstd for %PLATFORM% &&
-      ECHO *** &&
-      make -C contrib\pzstd googletest-mingw64 &&
-      make -C contrib\pzstd pzstd.exe &&
-      make -C contrib\pzstd tests &&
-      make -C contrib\pzstd check &&
-      make -C contrib\pzstd clean
-    )
+  - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
   - if [%COMPILER%]==[gcc] (
       ECHO *** &&
       ECHO *** Building %PLATFORM% &&
       ECHO *** &&
       make -v &&
       cc -v &&
-      ECHO make %MAKE_PARAMS% &&
-      make %MAKE_PARAMS%
+      ECHO %MAKE_PARAMS% &&
+      sh -c %MAKE_PARAMS%
     )
+  - if [%PLATFORM%]==[clang] COPY tests\fuzzer.exe tests\fuzzer_clang.exe
   - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
       COPY programs\zstd.exe bin\zstd.exe &&
       appveyor PushArtifact bin\zstd.exe
@@ -128,8 +114,24 @@ build_script:
 test_script:
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - SET FUZZERTEST=-T1mn
-  - if [%COMPILER%]==[gcc] (
-      if [%PLATFORM%]==[mingw64] tests\fuzzer_clang.exe %FUZZERTEST%
+  - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[clang] (
+      tests\fuzzer_clang.exe %FUZZERTEST% &&
+      ECHO *** &&
+      ECHO *** Building cmake for %PLATFORM% &&
+      ECHO *** &&
+      mkdir build\cmake\build &&
+      cd build\cmake\build &&
+      cmake -G "Visual Studio 14 2015 Win64" .. &&
+      cd ..\..\.. &&
+      make clean &&
+      ECHO *** &&
+      ECHO *** Building pzstd for %PLATFORM% &&
+      ECHO *** &&
+      make -C contrib\pzstd googletest-mingw64 &&
+      make -C contrib\pzstd pzstd.exe &&
+      make -C contrib\pzstd tests &&
+      make -C contrib\pzstd check &&
+      make -C contrib\pzstd clean
     )
   - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
       CD tests &&
@@ -144,6 +146,11 @@ test_script:
       fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
     )
 
+branches:
+  only:
+  - dev
+  - master
+
 artifacts:
   - path: bin\zstd.exe
   - path: bin\zstd32.exe

build/VS2005/fuzzer/fuzzer.vcproj

@@ -331,6 +331,10 @@
 				RelativePath="..\..\..\programs\datagen.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\cover.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
@@ -339,10 +343,22 @@
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\fse_compress.c"
 				>
@@ -389,6 +405,14 @@
 			Filter="h;hpp;hxx;hm;inl;inc;xsd"
 			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
 			>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\bitstream.h"
 				>
@@ -445,6 +469,10 @@
 				RelativePath="..\..\..\lib\common\zstd_internal.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>

build/VS2005/zstd/zstd.vcproj

@@ -43,8 +43,8 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -120,8 +120,8 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,8 +195,8 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -273,8 +273,8 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -343,6 +343,10 @@
 				RelativePath="..\..\..\programs\dibio.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\cover.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
@@ -391,6 +395,10 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
@@ -529,6 +537,10 @@
 				RelativePath="..\..\..\lib\legacy\zstd_v07.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 		</Filter>
 	</Files>
 	<Globals>

build/VS2005/zstdlib/zstdlib.vcproj

@@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -120,7 +120,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -194,7 +194,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -271,7 +271,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -327,10 +327,22 @@
 			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\cover.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
@@ -359,6 +371,10 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
 				>
@@ -375,6 +391,10 @@
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
@@ -425,6 +445,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.h"
 				>
@@ -485,6 +513,10 @@
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\zstd_static.h"
 				>

build/VS2008/fuzzer/fuzzer.vcproj

@@ -332,10 +332,22 @@
 				RelativePath="..\..\..\programs\datagen.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\cover.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
@@ -376,6 +388,10 @@
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
@@ -398,6 +414,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.h"
 				>
@@ -446,6 +470,10 @@
 				RelativePath="..\..\..\lib\common\zstd_internal.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>

build/VS2008/zstd/zstd.vcproj

@@ -44,8 +44,8 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,8 +121,8 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -196,8 +196,8 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -274,8 +274,8 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -344,6 +344,10 @@
 				RelativePath="..\..\..\programs\dibio.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\cover.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
@@ -376,6 +380,14 @@
 				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
@@ -428,6 +440,10 @@
 				RelativePath="..\..\..\programs\zstdcli.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 		</Filter>
 		<Filter
 			Name="Header Files"
@@ -446,10 +462,6 @@
 				RelativePath="..\..\..\lib\common\error_private.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\common\fse.h"
 				>
@@ -470,6 +482,14 @@
 				RelativePath="..\..\..\lib\common\mem.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\xxhash.h"
 				>
@@ -486,6 +506,10 @@
 				RelativePath="..\..\..\lib\zstd.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\zstd_internal.h"
 				>
@@ -530,6 +554,10 @@
 				RelativePath="..\..\..\lib\legacy\zstd_v07.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 		</Filter>
 	</Files>
 	<Globals>

build/VS2008/zstdlib/zstdlib.vcproj

@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -328,10 +328,22 @@
 			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\cover.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
@@ -368,6 +380,10 @@
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
@@ -418,6 +434,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.h"
 				>
@@ -474,6 +498,10 @@
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\zstd_static.h"
 				>

build/VS2010/fullbench-dll/fullbench-dll.vcxproj

@@ -99,6 +99,7 @@
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -138,6 +139,7 @@
       <OptimizeReferences>true</OptimizeReferences>
       <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

build/VS2010/fuzzer/fuzzer.vcxproj

@@ -155,6 +155,8 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\lib\common\pool.c" />
+    <ClCompile Include="..\..\..\lib\common\threading.c" />
     <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
     <ClCompile Include="..\..\..\lib\common\error_private.c" />
     <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
@@ -163,14 +165,18 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\tests\fuzzer.c" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\..\lib\common\pool.h" />
+    <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
@@ -178,6 +184,7 @@
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />

build/VS2010/libzstd-dll/libzstd-dll.rc

@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet"
-            VALUE "FileDescription", "Fast and efficient compression algorithm"
+            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "libzstd.dll"
-            VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
             VALUE "OriginalFilename", "libzstd.dll"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -19,6 +19,8 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\lib\common\pool.c" />
+    <ClCompile Include="..\..\..\lib\common\threading.c" />
     <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
     <ClCompile Include="..\..\..\lib\common\error_private.c" />
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
@@ -27,10 +29,13 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
+    <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
@@ -41,7 +46,9 @@
     <ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
   </ItemGroup>
-  <ItemGroup>    
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\common\pool.h" />
+    <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
@@ -61,6 +68,7 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="libzstd-dll.rc" />
@@ -141,7 +149,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -161,7 +169,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -181,7 +189,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -203,7 +211,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

build/VS2010/libzstd/libzstd.vcxproj

@@ -19,6 +19,8 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\lib\common\pool.c" />
+    <ClCompile Include="..\..\..\lib\common\threading.c" />
     <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
     <ClCompile Include="..\..\..\lib\common\error_private.c" />
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
@@ -27,10 +29,13 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
+    <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
@@ -41,7 +46,9 @@
     <ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
   </ItemGroup>
-  <ItemGroup>    
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\common\pool.h" />
+    <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
@@ -61,6 +68,7 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
@@ -138,7 +146,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -158,7 +166,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -178,7 +186,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -200,7 +208,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

build/VS2010/zstd/zstd.vcxproj

@@ -21,14 +21,18 @@
   <ItemGroup>
     <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
     <ClCompile Include="..\..\..\lib\common\error_private.c" />
+    <ClCompile Include="..\..\..\lib\common\pool.c" />
+    <ClCompile Include="..\..\..\lib\common\threading.c" />
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
     <ClCompile Include="..\..\..\lib\common\zstd_common.c" />
     <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
@@ -45,7 +49,10 @@
     <ClCompile Include="..\..\..\programs\zstdcli.c" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\..\lib\common\pool.h" />
+    <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
@@ -66,6 +73,7 @@
     <ClInclude Include="..\..\..\programs\datagen.h" />
     <ClInclude Include="..\..\..\programs\dibio.h" />
     <ClInclude Include="..\..\..\programs\fileio.h" />
+    <ClInclude Include="..\..\..\programs\platform.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
   </ItemGroup>
   <ItemGroup>
@@ -119,25 +127,25 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath)</LibraryPath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath);</LibraryPath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath)</LibraryPath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath);</LibraryPath>
   </PropertyGroup>
@@ -147,7 +155,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
     </ClCompile>
@@ -163,7 +171,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
     </ClCompile>
@@ -181,7 +189,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <TreatWarningAsError>false</TreatWarningAsError>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -202,10 +210,11 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <AdditionalOptions>/DZSTD_MULTITHREAD %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

build/cmake/CMakeLists.txt

@@ -11,10 +11,12 @@ PROJECT(zstd)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
 
 OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
+OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
+OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
 
 IF (ZSTD_LEGACY_SUPPORT)
     MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
-    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=1)
+    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
 ELSE (ZSTD_LEGACY_SUPPORT)
     MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
     ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
@@ -23,6 +25,9 @@ ENDIF (ZSTD_LEGACY_SUPPORT)
 ADD_SUBDIRECTORY(lib)
 ADD_SUBDIRECTORY(programs)
 ADD_SUBDIRECTORY(tests)
+IF (ZSTD_BUILD_CONTRIB)
+    ADD_SUBDIRECTORY(contrib)
+ENDIF (ZSTD_BUILD_CONTRIB)
 
 #-----------------------------------------------------------------------------
 # Add extra compilation flags

build/cmake/CMakeModules/AddExtraCompilationFlags.cmake

@@ -132,10 +132,8 @@ MACRO(ADD_EXTRA_COMPILATION_FLAGS)
         endif (ACTIVATE_WARNING_CAST_ALIGN)
 
         if (ACTIVATE_WARNING_STRICT_PROTOTYPES)
-            list(APPEND CMAKE_CXX_FLAGS ${WARNING_STRICT_PROTOTYPES})
             list(APPEND CMAKE_C_FLAGS ${WARNING_STRICT_PROTOTYPES})
         else ()
-            string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
             string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
         endif (ACTIVATE_WARNING_STRICT_PROTOTYPES)
 

build/cmake/contrib/CMakeLists.txt

@@ -0,0 +1,16 @@
+# ################################################################
+# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ################################################################
+
+PROJECT(contrib)
+
+ADD_SUBDIRECTORY(pzstd)
+

build/cmake/contrib/pzstd/CMakeLists.txt

@@ -0,0 +1,30 @@
+# ################################################################
+# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ################################################################
+
+PROJECT(pzstd)
+
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+
+# Define project root directory
+SET(ROOT_DIR ../../../..)
+
+# Define programs directory, where sources and header files are located
+SET(LIBRARY_DIR ${ROOT_DIR}/lib)
+SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
+SET(PZSTD_DIR ${ROOT_DIR}/contrib/pzstd)
+INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
+
+ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
+TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread)
+SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG")
+SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow")
+

build/cmake/lib/CMakeLists.txt

@@ -1,30 +1,10 @@
 # ################################################################
-# zstd - Makefile
-# Copyright (C) Yann Collet 2014-2016
-# All rights reserved.
-#
-# BSD license
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright notice, this
-#   list of conditions and the following disclaimer in the documentation and/or
-#   other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# * Copyright (c) 2014-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
 #
 # You can contact the author at :
 #  - zstd homepage : http://www.zstd.net/
@@ -58,22 +38,29 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
 
 SET(Sources
         ${LIBRARY_DIR}/common/entropy_common.c
+        ${LIBRARY_DIR}/common/fse_decompress.c
+        ${LIBRARY_DIR}/common/threading.c
+        ${LIBRARY_DIR}/common/pool.c
         ${LIBRARY_DIR}/common/zstd_common.c
         ${LIBRARY_DIR}/common/error_private.c
         ${LIBRARY_DIR}/common/xxhash.c
-        ${LIBRARY_DIR}/common/fse_decompress.c
         ${LIBRARY_DIR}/compress/fse_compress.c
         ${LIBRARY_DIR}/compress/huf_compress.c
         ${LIBRARY_DIR}/compress/zstd_compress.c
+        ${LIBRARY_DIR}/compress/zstdmt_compress.c
         ${LIBRARY_DIR}/decompress/huf_decompress.c
         ${LIBRARY_DIR}/decompress/zstd_decompress.c
+        ${LIBRARY_DIR}/dictBuilder/cover.c
         ${LIBRARY_DIR}/dictBuilder/divsufsort.c
         ${LIBRARY_DIR}/dictBuilder/zdict.c
+        ${LIBRARY_DIR}/deprecated/zbuff_common.c
         ${LIBRARY_DIR}/deprecated/zbuff_compress.c
         ${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
 
 SET(Headers
         ${LIBRARY_DIR}/zstd.h
+        ${LIBRARY_DIR}/common/pool.h
+        ${LIBRARY_DIR}/common/threading.h
         ${LIBRARY_DIR}/common/bitstream.h
         ${LIBRARY_DIR}/common/error_private.h
         ${LIBRARY_DIR}/common/zstd_errors.h
@@ -81,6 +68,7 @@ SET(Headers
         ${LIBRARY_DIR}/common/huf.h
         ${LIBRARY_DIR}/common/mem.h
         ${LIBRARY_DIR}/common/zstd_internal.h
+        ${LIBRARY_DIR}/compress/zstdmt_compress.h
         ${LIBRARY_DIR}/dictBuilder/zdict.h
         ${LIBRARY_DIR}/deprecated/zbuff.h)
 
@@ -109,12 +97,12 @@ IF (ZSTD_LEGACY_SUPPORT)
 ENDIF (ZSTD_LEGACY_SUPPORT)
 
 IF (MSVC)
-    SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/zstdlib)
-    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc)
+    SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/libzstd-dll)
+    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
 ENDIF (MSVC)
 
 # Split project to static and shared libraries build
-ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources})
+ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers})
 ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
 
 # Add specific compile definitions for MSVC project
@@ -178,23 +166,17 @@ IF (UNIX)
     SET(SHARED_LIBRARY_SYMLINK1_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK1})
     SET(SHARED_LIBRARY_SYMLINK2_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK2})
 
-    if (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
-        FILE(REMOVE ${SHARED_LIBRARY_SYMLINK1_PATH})
-    endif (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
-
-    if (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
-        FILE(REMOVE ${SHARED_LIBRARY_SYMLINK2_PATH})
-    endif (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
+    ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
+            DEPENDS  ${SHARED_LIBRARY_LINK_PATH}
+            COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK1}")
 
     ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
-            COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
-            DEPENDS ${SHARED_LIBRARY_LINK_PATH}
-            COMMENT "Generating symbolic link")
+            COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
+            DEPENDS  ${SHARED_LIBRARY_LINK_PATH}
+            COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK2}")
 
-    ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
-            COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
-            DEPENDS ${SHARED_LIBRARY_LINK_PATH}
-            COMMENT "Generating symbolic link")
+    SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${SHARED_LIBRARY_SYMLINK1};${SHARED_LIBRARY_SYMLINK2}")
 
     INSTALL(FILES ${SHARED_LIBRARY_SYMLINK1_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})
     INSTALL(FILES ${SHARED_LIBRARY_SYMLINK2_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})

build/cmake/programs/CMakeLists.txt

@@ -1,30 +1,10 @@
 # ################################################################
-# zstd - Makefile
-# Copyright (C) Yann Collet 2014-2016
-# All rights reserved.
-#
-# BSD license
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright notice, this
-#   list of conditions and the following disclaimer in the documentation and/or
-#   other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
 #
 # You can contact the author at :
 #  - zstd homepage : http://www.zstd.net/
@@ -40,18 +20,31 @@ SET(ROOT_DIR ../../..)
 # Define programs directory, where sources and header files are located
 SET(LIBRARY_DIR ${ROOT_DIR}/lib)
 SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
-INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder)
+INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
 IF (ZSTD_LEGACY_SUPPORT)
     SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
     INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
 ENDIF (ZSTD_LEGACY_SUPPORT)
 
-ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c)
-TARGET_LINK_LIBRARIES(zstd libzstd_static)
+IF (MSVC)
+    SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/zstd)
+    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
+ENDIF (MSVC)
 
+ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+TARGET_LINK_LIBRARIES(zstd libzstd_static)
 IF (UNIX)
     ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
     TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
     SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
 ENDIF (UNIX)
+
+IF (ZSTD_MULTITHREAD_SUPPORT)
+    ADD_EXECUTABLE(zstdmt ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+    SET_TARGET_PROPERTIES(zstdmt PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+    TARGET_LINK_LIBRARIES(zstdmt libzstd_static)
+    IF (UNIX)
+        TARGET_LINK_LIBRARIES(zstdmt pthread)
+    ENDIF (UNIX)
+ENDIF (ZSTD_MULTITHREAD_SUPPORT)

build/cmake/tests/CMakeLists.txt

@@ -41,7 +41,7 @@ SET(ROOT_DIR ../../..)
 SET(LIBRARY_DIR ${ROOT_DIR}/lib)
 SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
 SET(TESTS_DIR ${ROOT_DIR}/tests)
-INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder)
+INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
 ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fullbench.c)
 TARGET_LINK_LIBRARIES(fullbench libzstd_static)

circle.yml

@@ -0,0 +1,75 @@
+dependencies:
+  override:
+    - sudo dpkg --add-architecture i386
+    - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
+    - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+    - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
+    - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
+
+test:
+  override:
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; make all   && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c99build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c11build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gppbuild     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build    && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gcc6build    && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make m32build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make aarch64build && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild     && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make ppc64build   && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true              && make clean; fi #could add another test here
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make shortest     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
+      :
+        parallel: true
+
+  post:
+    - echo Circle CI tests finished
+
+  # Longer tests
+    #- make -C tests test-zstd-nolegacy && make clean
+    #- pyenv global 3.4.4; make -C tests versionsTest && make clean
+    #- make zlibwrapper         && make clean
+    #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+    #- make uasan               && make clean
+    #- make asan32              && make clean
+    #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" 
+  # Valgrind tests
+    #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
+    #- make -C tests valgrindTest && make clean
+  # ARM, AArch64, PowerPC, PowerPC64 tests
+    #- make ppctest             && make clean
+    #- make ppc64test           && make clean
+    #- make armtest             && make clean
+    #- make aarch64test         && make clean

contrib/gen_html/Makefile

@@ -7,12 +7,18 @@
 # of patent rights can be found in the PATENTS file in the same directory.
 # ##########################################################################
 
-
 CFLAGS ?= -O3
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
 CFLAGS += $(MOREFLAGS)
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+FLAGS   = $(CPPFLAGS) $(CFLAGS) $(CXXFLAGS) $(LDFLAGS)
 
+ZSTDAPI = ../../lib/zstd.h
+ZSTDMANUAL = ../../doc/zstd_manual.html
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LIBVER := $(shell echo $(LIBVER_SCRIPT))
 
 
 # Define *.exe as extension for Windows systems
@@ -23,14 +29,23 @@ EXT =
 endif
 
 
-.PHONY: default gen_html
-
+.PHONY: default
 default: gen_html
 
+.PHONY: all
+all: manual
+
 gen_html: gen_html.cpp
-	$(CXX)      $(FLAGS) $^ -o $@$(EXT)
+	$(CXX) $(FLAGS) $^ -o $@$(EXT)
 
+$(ZSTDMANUAL): gen_html $(ZSTDAPI)
+	echo "Update zstd manual in /doc"
+	./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
 
+.PHONY: manual
+manual: gen_html $(ZSTDMANUAL)
+
+.PHONY: clean
 clean:
 	@$(RM) gen_html$(EXT)
 	@echo Cleaning completed

contrib/gen_html/gen_html.cpp

@@ -19,7 +19,7 @@ void trim(string& s, string characters)
 {
     size_t p = s.find_first_not_of(characters);
     s.erase(0, p);
- 
+
     p = s.find_last_not_of(characters);
     if (string::npos != p)
        s.erase(p+1);
@@ -48,7 +48,7 @@ vector<string> get_lines(vector<string>& input, int& linenum, string terminator)
         line = input[linenum];
 
         if (terminator.empty() && line.empty()) { linenum--; break; }
-        
+
         epos = line.find(terminator);
         if (!terminator.empty() && epos!=string::npos) {
             out.push_back(line);
@@ -168,7 +168,11 @@ int main(int argc, char *argv[]) {
             sout << "<pre><b>";
             for (l=0; l<lines.size(); l++) {
               //  fprintf(stderr, "line[%d]=%s\n", l, lines[l].c_str());
-                print_line(sout, lines[l]);
+                string fline = lines[l];
+                if (fline.substr(0, 12) == "ZSTDLIB_API " ||
+                    fline.substr(0, 12) == string(12, ' '))
+                  fline = fline.substr(12);
+                print_line(sout, fline);
             }
             sout << "</b><p>";
             for (l=0; l<comments.size(); l++) {
@@ -217,4 +221,4 @@ int main(int argc, char *argv[]) {
     ostream << "</html>" << endl << "</body>" << endl;
 
     return 0;
-}
+}

contrib/meson/README

@@ -0,0 +1,3 @@
+This Meson project is provided with no guarantee and maintained by Dima Krasner <dima@dimakrasner.com>.
+
+It outputs one libzstd, either shared or static, depending on default_library.

contrib/meson/meson.build

@@ -0,0 +1,79 @@
+project('zstd', 'c', license: 'BSD')
+
+libm = meson.get_compiler('c').find_library('m', required: true)
+
+lib_dir = join_paths(meson.source_root(), '..', '..', 'lib')
+common_dir = join_paths(lib_dir, 'common')
+compress_dir = join_paths(lib_dir, 'compress')
+decompress_dir = join_paths(lib_dir, 'decompress')
+dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
+deprecated_dir = join_paths(lib_dir, 'deprecated')
+
+libzstd_srcs = [join_paths(common_dir, 'entropy_common.c'), join_paths(common_dir, 'fse_decompress.c'), join_paths(common_dir, 'threading.c'), join_paths(common_dir, 'pool.c'), join_paths(common_dir, 'zstd_common.c'), join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstdmt_compress.c'), join_paths(decompress_dir, 'huf_decompress.c'), join_paths(decompress_dir, 'zstd_decompress.c'), join_paths(dictbuilder_dir, 'cover.c'), join_paths(dictbuilder_dir, 'divsufsort.c'), join_paths(dictbuilder_dir, 'zdict.c'), join_paths(deprecated_dir, 'zbuff_common.c'), join_paths(deprecated_dir, 'zbuff_compress.c'), join_paths(deprecated_dir, 'zbuff_decompress.c')]
+
+libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]
+
+if get_option('legacy_support')
+    message('Enabling legacy support')
+    libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=4']
+
+    legacy_dir = join_paths(lib_dir, 'legacy')
+    libzstd_includes += [include_directories(legacy_dir)]
+    libzstd_srcs += [join_paths(legacy_dir, 'zstd_v01.c'), join_paths(legacy_dir, 'zstd_v02.c'), join_paths(legacy_dir, 'zstd_v03.c'), join_paths(legacy_dir, 'zstd_v04.c'), join_paths(legacy_dir, 'zstd_v05.c'), join_paths(legacy_dir, 'zstd_v06.c'), join_paths(legacy_dir, 'zstd_v07.c')]
+else
+    libzstd_cflags = []
+endif
+
+if get_option('multithread')
+    message('Enabling multi-threading support')
+    add_global_arguments('-DZSTD_MULTITHREAD', language: 'c')
+    libzstd_deps = [dependency('threads')]
+else
+    libzstd_deps = []
+endif
+
+libzstd = library('zstd',
+                  libzstd_srcs,
+                  include_directories: libzstd_includes,
+                  c_args: libzstd_cflags,
+                  dependencies: libzstd_deps,
+                  install: true)
+
+programs_dir = join_paths(meson.source_root(), '..', '..', 'programs')
+
+zstd = executable('zstd',
+                  join_paths(programs_dir, 'bench.c'), join_paths(programs_dir, 'datagen.c'), join_paths(programs_dir, 'dibio.c'), join_paths(programs_dir, 'fileio.c'), join_paths(programs_dir, 'zstdcli.c'),
+                  include_directories: libzstd_includes,
+                  c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
+                  link_with: libzstd,
+                  install: true)
+
+tests_dir = join_paths(meson.source_root(), '..', '..', 'tests')
+datagen_c = join_paths(programs_dir, 'datagen.c')
+test_includes = libzstd_includes + [include_directories(programs_dir)]
+
+fullbench = executable('fullbench',
+                       datagen_c, join_paths(tests_dir, 'fullbench.c'),
+                       include_directories: test_includes,
+                       link_with: libzstd)
+test('fullbench', fullbench)
+
+fuzzer = executable('fuzzer',
+                    datagen_c, join_paths(tests_dir, 'fuzzer.c'),
+                    include_directories: test_includes,
+                    link_with: libzstd)
+test('fuzzer', fuzzer)
+
+if target_machine.system() != 'windows'
+    paramgrill = executable('paramgrill',
+                            datagen_c, join_paths(tests_dir, 'paramgrill.c'),
+                            include_directories: test_includes,
+                            link_with: libzstd,
+                            dependencies: libm)
+    test('paramgrill', paramgrill)
+
+    datagen = executable('datagen',
+                         datagen_c, join_paths(tests_dir, 'datagencli.c'),
+                         include_directories: test_includes,
+                         link_with: libzstd)
+endif

contrib/meson/meson_options.txt

@@ -0,0 +1,2 @@
+option('multithread', type: 'boolean', value: false)
+option('legacy_support', type: 'boolean', value: false)

contrib/pzstd/BUCK

@@ -0,0 +1,72 @@
+cxx_library(
+    name='libpzstd',
+    visibility=['PUBLIC'],
+    header_namespace='',
+    exported_headers=[
+        'ErrorHolder.h',
+        'Logging.h',
+        'Pzstd.h',
+    ],
+    headers=[
+        'SkippableFrame.h',
+    ],
+    srcs=[
+        'Pzstd.cpp',
+        'SkippableFrame.cpp',
+    ],
+    deps=[
+        ':options',
+        '//contrib/pzstd/utils:utils',
+        '//lib:mem',
+        '//lib:zstd',
+    ],
+)
+
+cxx_library(
+    name='options',
+    visibility=['PUBLIC'],
+    header_namespace='',
+    exported_headers=['Options.h'],
+    srcs=['Options.cpp'],
+    deps=[
+        '//contrib/pzstd/utils:scope_guard',
+        '//lib:zstd',
+        '//programs:util',
+    ],
+)
+
+cxx_binary(
+    name='pzstd',
+    visibility=['PUBLIC'],
+    srcs=['main.cpp'],
+    deps=[
+        ':libpzstd',
+        ':options',
+    ],
+)
+
+# Must run "make googletest" first
+cxx_library(
+    name='gtest',
+    srcs=glob([
+        'googletest/googletest/src/gtest-all.cc',
+        'googletest/googlemock/src/gmock-all.cc',
+        'googletest/googlemock/src/gmock_main.cc',
+    ]),
+    header_namespace='',
+    exported_headers=subdir_glob([
+        ('googletest/googletest/include', '**/*.h'),
+        ('googletest/googlemock/include', '**/*.h'),
+    ]),
+    headers=subdir_glob([
+        ('googletest/googletest', 'src/*.cc'),
+        ('googletest/googletest', 'src/*.h'),
+        ('googletest/googlemock', 'src/*.cc'),
+        ('googletest/googlemock', 'src/*.h'),
+    ]),
+    platform_linker_flags=[
+        ('android', []),
+        ('', ['-lpthread']),
+    ],
+    visibility=['PUBLIC'],
+)

contrib/pzstd/Makefile

@@ -26,7 +26,7 @@ POSTCOMPILE = mv -f $*.Td $*.d
 
 # CFLAGS, CXXFLAGS, CPPFLAGS, and LDFLAGS are for the users to override
 CFLAGS   ?= -O3 -Wall -Wextra
-CXXFLAGS ?= -O3 -Wall -Wextra -pedantic -std=c++11
+CXXFLAGS ?= -O3 -Wall -Wextra -pedantic
 CPPFLAGS ?=
 LDFLAGS  ?=
 
@@ -37,7 +37,7 @@ GTEST_INC  = -isystem googletest/googletest/include
 PZSTD_CPPFLAGS  = $(PZSTD_INC)
 PZSTD_CCXXFLAGS =
 PZSTD_CFLAGS    = $(PZSTD_CCXXFLAGS)
-PZSTD_CXXFLAGS  = $(PZSTD_CCXXFLAGS)
+PZSTD_CXXFLAGS  = $(PZSTD_CCXXFLAGS) -std=c++11
 PZSTD_LDFLAGS   =
 EXTRA_FLAGS     =
 ALL_CFLAGS      = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS)   $(PZSTD_CFLAGS)
@@ -85,6 +85,23 @@ endif
 .PHONY: default
 default: all
 
+.PHONY: test-pzstd
+test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd: clean googletest pzstd tests check
+
+.PHONY: test-pzstd32
+test-pzstd32: clean googletest32 all32 check
+
+.PHONY: test-pzstd-tsan
+test-pzstd-tsan: LDFLAGS=-fuse-ld=gold
+test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-tsan: clean googletest tsan check
+
+.PHONY: test-pzstd-asan
+test-pzstd-asan: LDFLAGS=-fuse-ld=gold
+test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-asan: clean asan check
+
 .PHONY: check
 check:
 	$(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS)
@@ -117,7 +134,7 @@ debug: pzstd$(EXT) tests roundtrip
 
 .PHONY: tsan
 tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC
-tsan: PZSTD_LDFLAGS   += -fsanitize=thread -pie
+tsan: PZSTD_LDFLAGS   += -fsanitize=thread
 tsan: debug
 
 .PHONY: asan

contrib/pzstd/Options.cpp

@@ -7,6 +7,7 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 #include "Options.h"
+#include "util.h"
 #include "utils/ScopeGuard.h"
 
 #include <algorithm>
@@ -15,7 +16,6 @@
 #include <cstring>
 #include <iterator>
 #include <thread>
-#include <util.h>
 #include <vector>
 
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) ||     \

contrib/pzstd/Options.h

@@ -54,7 +54,7 @@ struct Options {
 
   ZSTD_parameters determineParameters() const {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
-    params.fParams.contentSizeFlag = 1;
+    params.fParams.contentSizeFlag = 0;
     params.fParams.checksumFlag = checksum;
     if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) {
       params.cParams.windowLog = maxWindowLog;

contrib/pzstd/Pzstd.cpp

@@ -410,7 +410,7 @@ std::uint64_t asyncCompressChunks(
     });
     // Pass the output queue to the writer thread.
     chunks.push(std::move(out));
-    state.log(VERBOSE, "Starting a new frame\n");
+    state.log(VERBOSE, "%s\n", "Starting a new frame");
     // Fill the input queue for the compression job we just started
     status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead);
   }
@@ -547,8 +547,8 @@ std::uint64_t asyncDecompressFrames(
     if (frameSize == 0) {
       // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted
       // Pass the rest of the source to this decompression task
-      state.log(VERBOSE,
-          "Input not in pzstd format, falling back to serial decompression\n");
+      state.log(VERBOSE, "%s\n",
+          "Input not in pzstd format, falling back to serial decompression");
       while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
         status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead);
       }

contrib/pzstd/Pzstd.h

@@ -41,7 +41,7 @@ class SharedState {
       auto parameters = options.determineParameters();
       cStreamPool.reset(new ResourcePool<ZSTD_CStream>{
           [this, parameters]() -> ZSTD_CStream* {
-            this->log(VERBOSE, "Creating new ZSTD_CStream\n");
+            this->log(VERBOSE, "%s\n", "Creating new ZSTD_CStream");
             auto zcs = ZSTD_createCStream();
             if (zcs) {
               auto err = ZSTD_initCStream_advanced(
@@ -59,7 +59,7 @@ class SharedState {
     } else {
       dStreamPool.reset(new ResourcePool<ZSTD_DStream>{
           [this]() -> ZSTD_DStream* {
-            this->log(VERBOSE, "Creating new ZSTD_DStream\n");
+            this->log(VERBOSE, "%s\n", "Creating new ZSTD_DStream");
             auto zds = ZSTD_createDStream();
             if (zds) {
               auto err = ZSTD_initDStream(zds);

contrib/pzstd/main.cpp

@@ -9,11 +9,6 @@
 #include "ErrorHolder.h"
 #include "Options.h"
 #include "Pzstd.h"
-#include "utils/FileSystem.h"
-#include "utils/Range.h"
-#include "utils/ScopeGuard.h"
-#include "utils/ThreadPool.h"
-#include "utils/WorkQueue.h"
 
 using namespace pzstd;
 

contrib/pzstd/test/BUCK

@@ -0,0 +1,37 @@
+cxx_test(
+    name='options_test',
+    srcs=['OptionsTest.cpp'],
+    deps=['//contrib/pzstd:options'],
+)
+
+cxx_test(
+    name='pzstd_test',
+    srcs=['PzstdTest.cpp'],
+    deps=[
+        ':round_trip',
+        '//contrib/pzstd:libpzstd',
+        '//contrib/pzstd/utils:scope_guard',
+        '//programs:datagen',
+    ],
+)
+
+cxx_binary(
+    name='round_trip_test',
+    srcs=['RoundTripTest.cpp'],
+    deps=[
+        ':round_trip',
+        '//contrib/pzstd/utils:scope_guard',
+        '//programs:datagen',
+    ]
+)
+
+cxx_library(
+    name='round_trip',
+    header_namespace='test',
+    exported_headers=['RoundTrip.h'],
+    deps=[
+        '//contrib/pzstd:libpzstd',
+        '//contrib/pzstd:options',
+        '//contrib/pzstd/utils:scope_guard',
+    ]
+)

contrib/pzstd/test/PzstdTest.cpp

@@ -41,23 +41,20 @@ TEST(Pzstd, SmallSizes) {
       std::fclose(fd);
       ASSERT_EQ(written, len);
     }
-    for (unsigned headers = 0; headers <= 1; ++headers) {
-      for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) {
-        for (unsigned level = 1; level <= 4; level *= 4) {
-          auto errorGuard = makeScopeGuard([&] {
-            std::fprintf(stderr, "pzstd headers: %u\n", headers);
-            std::fprintf(stderr, "# threads: %u\n", numThreads);
-            std::fprintf(stderr, "compression level: %u\n", level);
-          });
-          Options options;
-          options.overwrite = true;
-          options.inputFiles = {inputFile};
-          options.numThreads = numThreads;
-          options.compressionLevel = level;
-          options.verbosity = 1;
-          ASSERT_TRUE(roundTrip(options));
-          errorGuard.dismiss();
-        }
+    for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) {
+      for (unsigned level = 1; level <= 4; level *= 4) {
+        auto errorGuard = makeScopeGuard([&] {
+          std::fprintf(stderr, "# threads: %u\n", numThreads);
+          std::fprintf(stderr, "compression level: %u\n", level);
+        });
+        Options options;
+        options.overwrite = true;
+        options.inputFiles = {inputFile};
+        options.numThreads = numThreads;
+        options.compressionLevel = level;
+        options.verbosity = 1;
+        ASSERT_TRUE(roundTrip(options));
+        errorGuard.dismiss();
       }
     }
   }
@@ -79,29 +76,26 @@ TEST(Pzstd, LargeSizes) {
       std::fclose(fd);
       ASSERT_EQ(written, len);
     }
-    for (unsigned headers = 0; headers <= 1; ++headers) {
-      for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
-        for (unsigned level = 1; level <= 4; level *= 2) {
-          auto errorGuard = makeScopeGuard([&] {
-            std::fprintf(stderr, "pzstd headers: %u\n", headers);
-            std::fprintf(stderr, "# threads: %u\n", numThreads);
-            std::fprintf(stderr, "compression level: %u\n", level);
-          });
-          Options options;
-          options.overwrite = true;
-          options.inputFiles = {inputFile};
-          options.numThreads = std::min(numThreads, options.numThreads);
-          options.compressionLevel = level;
-          options.verbosity = 1;
-          ASSERT_TRUE(roundTrip(options));
-          errorGuard.dismiss();
-        }
+    for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
+      for (unsigned level = 1; level <= 4; level *= 4) {
+        auto errorGuard = makeScopeGuard([&] {
+          std::fprintf(stderr, "# threads: %u\n", numThreads);
+          std::fprintf(stderr, "compression level: %u\n", level);
+        });
+        Options options;
+        options.overwrite = true;
+        options.inputFiles = {inputFile};
+        options.numThreads = std::min(numThreads, options.numThreads);
+        options.compressionLevel = level;
+        options.verbosity = 1;
+        ASSERT_TRUE(roundTrip(options));
+        errorGuard.dismiss();
       }
     }
   }
 }
 
-TEST(Pzstd, ExtremelyLargeSize) {
+TEST(Pzstd, DISABLED_ExtremelyLargeSize) {
   unsigned seed = std::random_device{}();
   std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed);
   std::mt19937 gen(seed);

contrib/pzstd/utils/BUCK

@@ -0,0 +1,75 @@
+cxx_library(
+    name='buffer',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['Buffer.h'],
+    deps=[':range'],
+)
+
+cxx_library(
+    name='file_system',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['FileSystem.h'],
+    deps=[':range'],
+)
+
+cxx_library(
+    name='likely',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['Likely.h'],
+)
+
+cxx_library(
+    name='range',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['Range.h'],
+    deps=[':likely'],
+)
+
+cxx_library(
+    name='resource_pool',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['ResourcePool.h'],
+)
+
+cxx_library(
+    name='scope_guard',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['ScopeGuard.h'],
+)
+
+cxx_library(
+    name='thread_pool',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['ThreadPool.h'],
+    deps=[':work_queue'],
+)
+
+cxx_library(
+    name='work_queue',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['WorkQueue.h'],
+    deps=[':buffer'],
+)
+
+cxx_library(
+    name='utils',
+    visibility=['PUBLIC'],
+    deps=[
+        ':buffer',
+        ':file_system',
+        ':likely',
+        ':range',
+        ':resource_pool',
+        ':scope_guard',
+        ':thread_pool',
+        ':work_queue',
+    ],
+)

contrib/pzstd/utils/test/BUCK

@@ -0,0 +1,35 @@
+cxx_test(
+    name='buffer_test',
+    srcs=['BufferTest.cpp'],
+    deps=['//contrib/pzstd/utils:buffer'],
+)
+
+cxx_test(
+    name='range_test',
+    srcs=['RangeTest.cpp'],
+    deps=['//contrib/pzstd/utils:range'],
+)
+
+cxx_test(
+    name='resource_pool_test',
+    srcs=['ResourcePoolTest.cpp'],
+    deps=['//contrib/pzstd/utils:resource_pool'],
+)
+
+cxx_test(
+    name='scope_guard_test',
+    srcs=['ScopeGuardTest.cpp'],
+    deps=['//contrib/pzstd/utils:scope_guard'],
+)
+
+cxx_test(
+    name='thread_pool_test',
+    srcs=['ThreadPoolTest.cpp'],
+    deps=['//contrib/pzstd/utils:thread_pool'],
+)
+
+cxx_test(
+    name='work_queue_test',
+    srcs=['RangeTest.cpp'],
+    deps=['//contrib/pzstd/utils:work_queue'],
+)

doc/README.md

@@ -0,0 +1,20 @@
+Zstandard Documentation
+=======================
+
+This directory contains material defining the Zstandard format,
+as well as for help using the `zstd` library.
+
+__`zstd_compression_format.md`__ : This document defines the Zstandard compression format.
+Compliant decoders must adhere to this document,
+and compliant encoders must generate data that follows it.
+
+__`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
+compliant with the Zstandard compression format.
+It can be used, for example, to better understand the format,
+or as the basis for a separate implementation a Zstandard decoder/encoder.
+
+__`zstd_manual.html`__ : Documentation on the functions found in `zstd.h`.
+See [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html) for
+the manual released with the latest official `zstd` release.
+
+

doc/educational_decoder/README.md

@@ -0,0 +1,29 @@
+Educational Decoder
+===================
+
+`zstd_decompress.c` is a self-contained implementation in C99 of a decoder,
+according to the [Zstandard format specification].
+While it does not implement as many features as the reference decoder,
+such as the streaming API or content checksums, it is written to be easy to
+follow and understand, to help understand how the Zstandard format works.
+It's laid out to match the [format specification],
+so it can be used to understand how complex segments could be implemented.
+It also contains implementations of Huffman and FSE table decoding.
+
+[Zstandard format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+[format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+
+`harness.c` provides a simple test harness around the decoder:
+
+    harness <input-file> <output-file> [dictionary]
+
+As an additional resource to be used with this decoder,
+see the `decodecorpus` tool in the [tests] directory.
+It generates valid Zstandard frames that can be used to verify
+a Zstandard decoder implementation.
+Note that to use the tool to verify this decoder implementation,
+the --content-size flag should be set,
+as this decoder does not handle streaming decoding,
+and so it must know the decompressed size in advance.
+
+[tests]: https://github.com/facebook/zstd/blob/dev/tests/

doc/educational_decoder/harness.c

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "zstd_decompress.h"
+
+typedef unsigned char u8;
+
+// If the data doesn't have decompressed size with it, fallback on assuming the
+// compression ratio is at most 16
+#define MAX_COMPRESSION_RATIO (16)
+
+// Protect against allocating too much memory for output
+#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
+
+u8 *input;
+u8 *output;
+u8 *dict;
+
+size_t read_file(const char *path, u8 **ptr) {
+    FILE *f = fopen(path, "rb");
+    if (!f) {
+        fprintf(stderr, "failed to open file %s\n", path);
+        exit(1);
+    }
+
+    fseek(f, 0L, SEEK_END);
+    size_t size = ftell(f);
+    rewind(f);
+
+    *ptr = malloc(size);
+    if (!ptr) {
+        fprintf(stderr, "failed to allocate memory to hold %s\n", path);
+        exit(1);
+    }
+
+    size_t pos = 0;
+    while (!feof(f)) {
+        size_t read = fread(&(*ptr)[pos], 1, size, f);
+        if (ferror(f)) {
+            fprintf(stderr, "error while reading file %s\n", path);
+            exit(1);
+        }
+        pos += read;
+    }
+
+    fclose(f);
+
+    return pos;
+}
+
+void write_file(const char *path, const u8 *ptr, size_t size) {
+    FILE *f = fopen(path, "wb");
+
+    size_t written = 0;
+    while (written < size) {
+        written += fwrite(&ptr[written], 1, size, f);
+        if (ferror(f)) {
+            fprintf(stderr, "error while writing file %s\n", path);
+            exit(1);
+        }
+    }
+
+    fclose(f);
+}
+
+int main(int argc, char **argv) {
+    if (argc < 3) {
+        fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary]\n",
+                argv[0]);
+
+        return 1;
+    }
+
+    size_t input_size = read_file(argv[1], &input);
+    size_t dict_size = 0;
+    if (argc >= 4) {
+        dict_size = read_file(argv[3], &dict);
+    }
+
+    size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size);
+    if (decompressed_size == -1) {
+        decompressed_size = MAX_COMPRESSION_RATIO * input_size;
+        fprintf(stderr, "WARNING: Compressed data does not contain "
+                        "decompressed size, going to assume the compression "
+                        "ratio is at most %d (decompressed size of at most "
+                        "%zu)\n",
+                MAX_COMPRESSION_RATIO, decompressed_size);
+    }
+    if (decompressed_size > MAX_OUTPUT_SIZE) {
+        fprintf(stderr,
+                "Required output size too large for this implementation\n");
+        return 1;
+    }
+    output = malloc(decompressed_size);
+    if (!output) {
+        fprintf(stderr, "failed to allocate memory\n");
+        return 1;
+    }
+
+    size_t decompressed =
+        ZSTD_decompress_with_dict(output, decompressed_size,
+                                  input, input_size, dict, dict_size);
+
+    write_file(argv[2], output, decompressed);
+
+    free(input);
+    free(output);
+    free(dict);
+    input = output = dict = NULL;
+}
+

doc/educational_decoder/zstd_decompress.c

@@ -0,0 +1,2345 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/// Zstandard educational decoder implementation
+/// See https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/// Zstandard decompression functions.
+/// `dst` must point to a space at least as large as the reconstructed output.
+size_t ZSTD_decompress(void *const dst, const size_t dst_len,
+                       const void *const src, const size_t src_len);
+/// If `dict != NULL` and `dict_len >= 8`, does the same thing as
+/// `ZSTD_decompress` but uses the provided dict
+size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
+                                 const void *const src, const size_t src_len,
+                                 const void *const dict, const size_t dict_len);
+
+/// Get the decompressed size of an input stream so memory can be allocated in
+/// advance
+size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
+
+/******* UTILITY MACROS AND TYPES *********************************************/
+// Max block size decompressed size is 128 KB and literal blocks must be smaller
+// than that
+#define MAX_LITERALS_SIZE ((size_t)128 * 1024)
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+#define ERROR(s)                                                               \
+    do {                                                                       \
+        fprintf(stderr, "Error: %s\n", s);                                     \
+        exit(1);                                                               \
+    } while (0)
+#define INP_SIZE()                                                             \
+    ERROR("Input buffer smaller than it should be or input is "                \
+          "corrupted")
+#define OUT_SIZE() ERROR("Output buffer too small for output")
+#define CORRUPTION() ERROR("Corruption detected while decompressing")
+#define BAD_ALLOC() ERROR("Memory allocation error")
+#define IMPOSSIBLE() ERROR("An impossibility has occurred")
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+
+typedef int8_t i8;
+typedef int16_t i16;
+typedef int32_t i32;
+typedef int64_t i64;
+/******* END UTILITY MACROS AND TYPES *****************************************/
+
+/******* IMPLEMENTATION PRIMITIVE PROTOTYPES **********************************/
+/// The implementations for these functions can be found at the bottom of this
+/// file.  They implement low-level functionality needed for the higher level
+/// decompression functions.
+
+/*** IO STREAM OPERATIONS *************/
+/// These structs are the interface for IO, and do bounds checking on all
+/// operations.  They should be used opaquely to ensure safety.
+
+/// Output is always done byte-by-byte
+typedef struct {
+    u8 *ptr;
+    size_t len;
+} ostream_t;
+
+/// Input often reads a few bits at a time, so maintain an internal offset
+typedef struct {
+    const u8 *ptr;
+    int bit_offset;
+    size_t len;
+} istream_t;
+
+/// The following two functions are the only ones that allow the istream to be
+/// non-byte aligned
+
+/// Reads `num` bits from a bitstream, and updates the internal offset
+static inline u64 IO_read_bits(istream_t *const in, const int num);
+/// Rewinds the stream by `num` bits
+static inline void IO_rewind_bits(istream_t *const in, const int num);
+/// If the remaining bits in a byte will be unused, advance to the end of the
+/// byte
+static inline void IO_align_stream(istream_t *const in);
+
+/// Write the given byte into the output stream
+static inline void IO_write_byte(ostream_t *const out, u8 symb);
+
+/// Returns the number of bytes left to be read in this stream.  The stream must
+/// be byte aligned.
+static inline size_t IO_istream_len(const istream_t *const in);
+
+/// Returns a pointer where `len` bytes can be read, and advances the internal
+/// state.  The stream must be byte aligned.
+static inline const u8 *IO_read_bytes(istream_t *const in, size_t len);
+/// Returns a pointer where `len` bytes can be written, and advances the internal
+/// state.  The stream must be byte aligned.
+static inline u8 *IO_write_bytes(ostream_t *const out, size_t len);
+
+/// Advance the inner state by `len` bytes.  The stream must be byte aligned.
+static inline void IO_advance_input(istream_t *const in, size_t len);
+
+/// Returns an `ostream_t` constructed from the given pointer and length
+static inline ostream_t IO_make_ostream(u8 *out, size_t len);
+/// Returns an `istream_t` constructed from the given pointer and length
+static inline istream_t IO_make_istream(const u8 *in, size_t len);
+
+/// Returns an `istream_t` with the same base as `in`, and length `len`
+/// Then, advance `in` to account for the consumed bytes
+/// `in` must be byte aligned
+static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len);
+/*** END IO STREAM OPERATIONS *********/
+
+/*** BITSTREAM OPERATIONS *************/
+/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits
+static inline u64 read_bits_LE(const u8 *src, const int num,
+                               const size_t offset);
+
+/// Read bits from the end of a HUF or FSE bitstream.  `offset` is in bits, so
+/// it updates `offset` to `offset - bits`, and then reads `bits` bits from
+/// `src + offset`.  If the offset becomes negative, the extra bits at the
+/// bottom are filled in with `0` bits instead of reading from before `src`.
+static inline u64 STREAM_read_bits(const u8 *src, const int bits,
+                                   i64 *const offset);
+/*** END BITSTREAM OPERATIONS *********/
+
+/*** BIT COUNTING OPERATIONS **********/
+/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to
+/// `num`, or `-1` if `num == 0`.
+static inline int log2inf(const u64 num);
+/*** END BIT COUNTING OPERATIONS ******/
+
+/*** HUFFMAN PRIMITIVES ***************/
+// Table decode method uses exponential memory, so we need to limit depth
+#define HUF_MAX_BITS (16)
+
+// Limit the maximum number of symbols to 256 so we can store a symbol in a byte
+#define HUF_MAX_SYMBS (256)
+
+/// Structure containing all tables necessary for efficient Huffman decoding
+typedef struct {
+    u8 *symbols;
+    u8 *num_bits;
+    int max_bits;
+} HUF_dtable;
+
+/// Decode a single symbol and read in enough bits to refresh the state
+static inline u8 HUF_decode_symbol(const HUF_dtable *const dtable,
+                                   u16 *const state, const u8 *const src,
+                                   i64 *const offset);
+/// Read in a full state's worth of bits to initialize it
+static inline void HUF_init_state(const HUF_dtable *const dtable,
+                                  u16 *const state, const u8 *const src,
+                                  i64 *const offset);
+
+/// Decompresses a single Huffman stream, returns the number of bytes decoded.
+/// `src_len` must be the exact length of the Huffman-coded block.
+static size_t HUF_decompress_1stream(const HUF_dtable *const dtable,
+                                     ostream_t *const out, istream_t *const in);
+/// Same as previous but decodes 4 streams, formatted as in the Zstandard
+/// specification.
+/// `src_len` must be the exact length of the Huffman-coded block.
+static size_t HUF_decompress_4stream(const HUF_dtable *const dtable,
+                                     ostream_t *const out, istream_t *const in);
+
+/// Initialize a Huffman decoding table using the table of bit counts provided
+static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits,
+                            const int num_symbs);
+/// Initialize a Huffman decoding table using the table of weights provided
+/// Weights follow the definition provided in the Zstandard specification
+static void HUF_init_dtable_usingweights(HUF_dtable *const table,
+                                         const u8 *const weights,
+                                         const int num_symbs);
+
+/// Free the malloc'ed parts of a decoding table
+static void HUF_free_dtable(HUF_dtable *const dtable);
+
+/// Deep copy a decoding table, so that it can be used and free'd without
+/// impacting the source table.
+static void HUF_copy_dtable(HUF_dtable *const dst, const HUF_dtable *const src);
+/*** END HUFFMAN PRIMITIVES ***********/
+
+/*** FSE PRIMITIVES *******************/
+/// For more description of FSE see
+/// https://github.com/Cyan4973/FiniteStateEntropy/
+
+// FSE table decoding uses exponential memory, so limit the maximum accuracy
+#define FSE_MAX_ACCURACY_LOG (15)
+// Limit the maximum number of symbols so they can be stored in a single byte
+#define FSE_MAX_SYMBS (256)
+
+/// The tables needed to decode FSE encoded streams
+typedef struct {
+    u8 *symbols;
+    u8 *num_bits;
+    u16 *new_state_base;
+    int accuracy_log;
+} FSE_dtable;
+
+/// Return the symbol for the current state
+static inline u8 FSE_peek_symbol(const FSE_dtable *const dtable,
+                                 const u16 state);
+/// Read the number of bits necessary to update state, update, and shift offset
+/// back to reflect the bits read
+static inline void FSE_update_state(const FSE_dtable *const dtable,
+                                    u16 *const state, const u8 *const src,
+                                    i64 *const offset);
+
+/// Combine peek and update: decode a symbol and update the state
+static inline u8 FSE_decode_symbol(const FSE_dtable *const dtable,
+                                   u16 *const state, const u8 *const src,
+                                   i64 *const offset);
+
+/// Read bits from the stream to initialize the state and shift offset back
+static inline void FSE_init_state(const FSE_dtable *const dtable,
+                                  u16 *const state, const u8 *const src,
+                                  i64 *const offset);
+
+/// Decompress two interleaved bitstreams (e.g. compressed Huffman weights)
+/// using an FSE decoding table.  `src_len` must be the exact length of the
+/// block.
+static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable,
+                                          ostream_t *const out,
+                                          istream_t *const in);
+
+/// Initialize a decoding table using normalized frequencies.
+static void FSE_init_dtable(FSE_dtable *const dtable,
+                            const i16 *const norm_freqs, const int num_symbs,
+                            const int accuracy_log);
+
+/// Decode an FSE header as defined in the Zstandard format specification and
+/// use the decoded frequencies to initialize a decoding table.
+static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in,
+                                const int max_accuracy_log);
+
+/// Initialize an FSE table that will always return the same symbol and consume
+/// 0 bits per symbol, to be used for RLE mode in sequence commands
+static void FSE_init_dtable_rle(FSE_dtable *const dtable, const u8 symb);
+
+/// Free the malloc'ed parts of a decoding table
+static void FSE_free_dtable(FSE_dtable *const dtable);
+
+/// Deep copy a decoding table, so that it can be used and free'd without
+/// impacting the source table.
+static void FSE_copy_dtable(FSE_dtable *const dst, const FSE_dtable *const src);
+/*** END FSE PRIMITIVES ***************/
+
+/******* END IMPLEMENTATION PRIMITIVE PROTOTYPES ******************************/
+
+/******* ZSTD HELPER STRUCTS AND PROTOTYPES ***********************************/
+
+/// A small structure that can be reused in various places that need to access
+/// frame header information
+typedef struct {
+    // The size of window that we need to be able to contiguously store for
+    // references
+    size_t window_size;
+    // The total output size of this compressed frame
+    size_t frame_content_size;
+
+    // The dictionary id if this frame uses one
+    u32 dictionary_id;
+
+    // Whether or not the content of this frame has a checksum
+    int content_checksum_flag;
+    // Whether or not the output for this frame is in a single segment
+    int single_segment_flag;
+} frame_header_t;
+
+/// The context needed to decode blocks in a frame
+typedef struct {
+    frame_header_t header;
+
+    // The total amount of data available for backreferences, to determine if an
+    // offset too large to be correct
+    size_t current_total_output;
+
+    const u8 *dict_content;
+    size_t dict_content_len;
+
+    // Entropy encoding tables so they can be repeated by future blocks instead
+    // of retransmitting
+    HUF_dtable literals_dtable;
+    FSE_dtable ll_dtable;
+    FSE_dtable ml_dtable;
+    FSE_dtable of_dtable;
+
+    // The last 3 offsets for the special "repeat offsets".
+    u64 previous_offsets[3];
+} frame_context_t;
+
+/// The decoded contents of a dictionary so that it doesn't have to be repeated
+/// for each frame that uses it
+typedef struct {
+    // Entropy tables
+    HUF_dtable literals_dtable;
+    FSE_dtable ll_dtable;
+    FSE_dtable ml_dtable;
+    FSE_dtable of_dtable;
+
+    // Raw content for backreferences
+    u8 *content;
+    size_t content_size;
+
+    // Offset history to prepopulate the frame's history
+    u64 previous_offsets[3];
+
+    u32 dictionary_id;
+} dictionary_t;
+
+/// A tuple containing the parts necessary to decode and execute a ZSTD sequence
+/// command
+typedef struct {
+    u32 literal_length;
+    u32 match_length;
+    u32 offset;
+} sequence_command_t;
+
+/// The decoder works top-down, starting at the high level like Zstd frames, and
+/// working down to lower more technical levels such as blocks, literals, and
+/// sequences.  The high-level functions roughly follow the outline of the
+/// format specification:
+/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+
+/// Before the implementation of each high-level function declared here, the
+/// prototypes for their helper functions are defined and explained
+
+/// Decode a single Zstd frame, or error if the input is not a valid frame.
+/// Accepts a dict argument, which may be NULL indicating no dictionary.
+/// See
+/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame-concatenation
+static void decode_frame(ostream_t *const out, istream_t *const in,
+                         const dictionary_t *const dict);
+
+// Decode data in a compressed block
+static void decompress_block(frame_context_t *const ctx, ostream_t *const out,
+                             istream_t *const in);
+
+// Decode the literals section of a block
+static size_t decode_literals(frame_context_t *const ctx, istream_t *const in,
+                              u8 **const literals);
+
+// Decode the sequences part of a block
+static size_t decode_sequences(frame_context_t *const ctx, istream_t *const in,
+                               sequence_command_t **const sequences);
+
+// Execute the decoded sequences on the literals block
+static void execute_sequences(frame_context_t *const ctx, ostream_t *const out,
+                              const u8 *const literals,
+                              const size_t literals_len,
+                              const sequence_command_t *const sequences,
+                              const size_t num_sequences);
+
+// Parse a provided dictionary blob for use in decompression
+static void parse_dictionary(dictionary_t *const dict, const u8 *src,
+                             size_t src_len);
+static void free_dictionary(dictionary_t *const dict);
+/******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/
+
+size_t ZSTD_decompress(void *const dst, const size_t dst_len,
+                       const void *const src, const size_t src_len) {
+    return ZSTD_decompress_with_dict(dst, dst_len, src, src_len, NULL, 0);
+}
+
+size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
+                                 const void *const src, const size_t src_len,
+                                 const void *const dict,
+                                 const size_t dict_len) {
+    dictionary_t parsed_dict;
+    memset(&parsed_dict, 0, sizeof(dictionary_t));
+    // dict_len < 8 is not a valid dictionary
+    if (dict && dict_len > 8) {
+        parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len);
+    }
+
+    istream_t in = {(const u8 *)src, 0, src_len};
+    ostream_t out = {(u8 *)dst, dst_len};
+
+    // "A content compressed by Zstandard is transformed into a Zstandard frame.
+    // Multiple frames can be appended into a single file or stream. A frame is
+    // totally independent, has a defined beginning and end, and a set of
+    // parameters which tells the decoder how to decompress it."
+    while (IO_istream_len(&in) > 0) {
+        decode_frame(&out, &in, &parsed_dict);
+    }
+
+    free_dictionary(&parsed_dict);
+
+    return out.ptr - (u8 *)dst;
+}
+
+/******* FRAME DECODING ******************************************************/
+
+static void decode_data_frame(ostream_t *const out, istream_t *const in,
+                              const dictionary_t *const dict);
+static void init_frame_context(frame_context_t *const context,
+                               istream_t *const in,
+                               const dictionary_t *const dict);
+static void free_frame_context(frame_context_t *const context);
+static void parse_frame_header(frame_header_t *const header,
+                               istream_t *const in);
+static void frame_context_apply_dict(frame_context_t *const ctx,
+                                     const dictionary_t *const dict);
+
+static void decompress_data(frame_context_t *const ctx, ostream_t *const out,
+                            istream_t *const in);
+
+static void decode_frame(ostream_t *const out, istream_t *const in,
+                         const dictionary_t *const dict) {
+    const u32 magic_number = IO_read_bits(in, 32);
+
+    // Skippable frame
+    //
+    // "Magic_Number
+    //
+    // 4 Bytes, little-endian format. Value : 0x184D2A5?, which means any value
+    // from 0x184D2A50 to 0x184D2A5F. All 16 values are valid to identify a
+    // skippable frame."
+    if ((magic_number & ~0xFU) == 0x184D2A50U) {
+        // "Skippable frames allow the insertion of user-defined data into a
+        // flow of concatenated frames. Its design is pretty straightforward,
+        // with the sole objective to allow the decoder to quickly skip over
+        // user-defined data and continue decoding.
+        //
+        // Skippable frames defined in this specification are compatible with
+        // LZ4 ones."
+        const size_t frame_size = IO_read_bits(in, 32);
+
+        // skip over frame
+        IO_advance_input(in, frame_size);
+
+        return;
+    }
+
+    // Zstandard frame
+    //
+    // "Magic_Number
+    //
+    // 4 Bytes, little-endian format. Value : 0xFD2FB528"
+    if (magic_number == 0xFD2FB528U) {
+        // ZSTD frame
+        decode_data_frame(out, in, dict);
+
+        return;
+    }
+
+    // not a real frame
+    ERROR("Invalid magic number");
+}
+
+/// Decode a frame that contains compressed data.  Not all frames do as there
+/// are skippable frames.
+/// See
+/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#general-structure-of-zstandard-frame-format
+static void decode_data_frame(ostream_t *const out, istream_t *const in,
+                              const dictionary_t *const dict) {
+    frame_context_t ctx;
+
+    // Initialize the context that needs to be carried from block to block
+    init_frame_context(&ctx, in, dict);
+
+    if (ctx.header.frame_content_size != 0 &&
+        ctx.header.frame_content_size > out->len) {
+        OUT_SIZE();
+    }
+
+    decompress_data(&ctx, out, in);
+
+    free_frame_context(&ctx);
+}
+
+/// Takes the information provided in the header and dictionary, and initializes
+/// the context for this frame
+static void init_frame_context(frame_context_t *const context,
+                               istream_t *const in,
+                               const dictionary_t *const dict) {
+    // Most fields in context are correct when initialized to 0
+    memset(context, 0, sizeof(frame_context_t));
+
+    // Parse data from the frame header
+    parse_frame_header(&context->header, in);
+
+    // Set up the offset history for the repeat offset commands
+    context->previous_offsets[0] = 1;
+    context->previous_offsets[1] = 4;
+    context->previous_offsets[2] = 8;
+
+    // Apply details from the dict if it exists
+    frame_context_apply_dict(context, dict);
+}
+
+static void free_frame_context(frame_context_t *const context) {
+    HUF_free_dtable(&context->literals_dtable);
+
+    FSE_free_dtable(&context->ll_dtable);
+    FSE_free_dtable(&context->ml_dtable);
+    FSE_free_dtable(&context->of_dtable);
+
+    memset(context, 0, sizeof(frame_context_t));
+}
+
+static void parse_frame_header(frame_header_t *const header,
+                               istream_t *const in) {
+    // "The first header's byte is called the Frame_Header_Descriptor. It tells
+    // which other fields are present. Decoding this byte is enough to tell the
+    // size of Frame_Header.
+    //
+    // Bit number   Field name
+    // 7-6  Frame_Content_Size_flag
+    // 5    Single_Segment_flag
+    // 4    Unused_bit
+    // 3    Reserved_bit
+    // 2    Content_Checksum_flag
+    // 1-0  Dictionary_ID_flag"
+    const u8 descriptor = IO_read_bits(in, 8);
+
+    // decode frame header descriptor into flags
+    const u8 frame_content_size_flag = descriptor >> 6;
+    const u8 single_segment_flag = (descriptor >> 5) & 1;
+    const u8 reserved_bit = (descriptor >> 3) & 1;
+    const u8 content_checksum_flag = (descriptor >> 2) & 1;
+    const u8 dictionary_id_flag = descriptor & 3;
+
+    if (reserved_bit != 0) {
+        CORRUPTION();
+    }
+
+    header->single_segment_flag = single_segment_flag;
+    header->content_checksum_flag = content_checksum_flag;
+
+    // decode window size
+    if (!single_segment_flag) {
+        // "Provides guarantees on maximum back-reference distance that will be
+        // used within compressed data. This information is important for
+        // decoders to allocate enough memory.
+        //
+        // Bit numbers  7-3         2-0
+        // Field name   Exponent    Mantissa"
+        u8 window_descriptor = IO_read_bits(in, 8);
+        u8 exponent = window_descriptor >> 3;
+        u8 mantissa = window_descriptor & 7;
+
+        // Use the algorithm from the specification to compute window size
+        // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor
+        size_t window_base = (size_t)1 << (10 + exponent);
+        size_t window_add = (window_base / 8) * mantissa;
+        header->window_size = window_base + window_add;
+    }
+
+    // decode dictionary id if it exists
+    if (dictionary_id_flag) {
+        // "This is a variable size field, which contains the ID of the
+        // dictionary required to properly decode the frame. Note that this
+        // field is optional. When it's not present, it's up to the caller to
+        // make sure it uses the correct dictionary. Format is little-endian."
+        const int bytes_array[] = {0, 1, 2, 4};
+        const int bytes = bytes_array[dictionary_id_flag];
+
+        header->dictionary_id = IO_read_bits(in, bytes * 8);
+    } else {
+        header->dictionary_id = 0;
+    }
+
+    // decode frame content size if it exists
+    if (single_segment_flag || frame_content_size_flag) {
+        // "This is the original (uncompressed) size. This information is
+        // optional. The Field_Size is provided according to value of
+        // Frame_Content_Size_flag. The Field_Size can be equal to 0 (not
+        // present), 1, 2, 4 or 8 bytes. Format is little-endian."
+        //
+        // if frame_content_size_flag == 0 but single_segment_flag is set, we
+        // still have a 1 byte field
+        const int bytes_array[] = {1, 2, 4, 8};
+        const int bytes = bytes_array[frame_content_size_flag];
+
+        header->frame_content_size = IO_read_bits(in, bytes * 8);
+        if (bytes == 2) {
+            // "When Field_Size is 2, the offset of 256 is added."
+            header->frame_content_size += 256;
+        }
+    } else {
+        header->frame_content_size = 0;
+    }
+
+    if (single_segment_flag) {
+        // "The Window_Descriptor byte is optional. It is absent when
+        // Single_Segment_flag is set. In this case, the maximum back-reference
+        // distance is the content size itself, which can be any value from 1 to
+        // 2^64-1 bytes (16 EB)."
+        header->window_size = header->frame_content_size;
+    }
+}
+
+/// A dictionary acts as initializing values for the frame context before
+/// decompression, so we implement it by applying it's predetermined
+/// tables and content to the context before beginning decompression
+static void frame_context_apply_dict(frame_context_t *const ctx,
+                                     const dictionary_t *const dict) {
+    // If the content pointer is NULL then it must be an empty dict
+    if (!dict || !dict->content)
+        return;
+
+    // If the requested dictionary_id is non-zero, the correct dictionary must
+    // be present
+    if (ctx->header.dictionary_id != 0 &&
+        ctx->header.dictionary_id != dict->dictionary_id) {
+        ERROR("Wrong dictionary provided");
+    }
+
+    // Copy the dict content to the context for references during sequence
+    // execution
+    ctx->dict_content = dict->content;
+    ctx->dict_content_len = dict->content_size;
+
+    // If it's a formatted dict copy the precomputed tables in so they can
+    // be used in the table repeat modes
+    if (dict->dictionary_id != 0) {
+        // Deep copy the entropy tables so they can be freed independently of
+        // the dictionary struct
+        HUF_copy_dtable(&ctx->literals_dtable, &dict->literals_dtable);
+        FSE_copy_dtable(&ctx->ll_dtable, &dict->ll_dtable);
+        FSE_copy_dtable(&ctx->of_dtable, &dict->of_dtable);
+        FSE_copy_dtable(&ctx->ml_dtable, &dict->ml_dtable);
+
+        memcpy(ctx->previous_offsets, dict->previous_offsets,
+               sizeof(ctx->previous_offsets));
+    }
+}
+
+/// Decompress the data from a frame block by block
+static void decompress_data(frame_context_t *const ctx, ostream_t *const out,
+                            istream_t *const in) {
+    // "A frame encapsulates one or multiple blocks. Each block can be
+    // compressed or not, and has a guaranteed maximum content size, which
+    // depends on frame parameters. Unlike frames, each block depends on
+    // previous blocks for proper decoding. However, each block can be
+    // decompressed without waiting for its successor, allowing streaming
+    // operations."
+    int last_block = 0;
+    do {
+        // "Last_Block
+        //
+        // The lowest bit signals if this block is the last one. Frame ends
+        // right after this block.
+        //
+        // Block_Type and Block_Size
+        //
+        // The next 2 bits represent the Block_Type, while the remaining 21 bits
+        // represent the Block_Size. Format is little-endian."
+        last_block = IO_read_bits(in, 1);
+        const int block_type = IO_read_bits(in, 2);
+        const size_t block_len = IO_read_bits(in, 21);
+
+        switch (block_type) {
+        case 0: {
+            // "Raw_Block - this is an uncompressed block. Block_Size is the
+            // number of bytes to read and copy."
+            const u8 *const read_ptr = IO_read_bytes(in, block_len);
+            u8 *const write_ptr = IO_write_bytes(out, block_len);
+            //
+            // Copy the raw data into the output
+            memcpy(write_ptr, read_ptr, block_len);
+
+            ctx->current_total_output += block_len;
+            break;
+        }
+        case 1: {
+            // "RLE_Block - this is a single byte, repeated N times. In which
+            // case, Block_Size is the size to regenerate, while the
+            // "compressed" block is just 1 byte (the byte to repeat)."
+            const u8 *const read_ptr = IO_read_bytes(in, 1);
+            u8 *const write_ptr = IO_write_bytes(out, block_len);
+
+            // Copy `block_len` copies of `streams->src[0]` to the output
+            memset(write_ptr, read_ptr[0], block_len);
+
+            ctx->current_total_output += block_len;
+            break;
+        }
+        case 2: {
+            // "Compressed_Block - this is a Zstandard compressed block,
+            // detailed in another section of this specification. Block_Size is
+            // the compressed size.
+
+            // Create a sub-stream for the block
+            istream_t block_stream = IO_make_sub_istream(in, block_len);
+            decompress_block(ctx, out, &block_stream);
+            break;
+        }
+        case 3:
+            // "Reserved - this is not a block. This value cannot be used with
+            // current version of this specification."
+            CORRUPTION();
+            break;
+        default:
+            IMPOSSIBLE();
+        }
+    } while (!last_block);
+
+    if (ctx->header.content_checksum_flag) {
+        // This program does not support checking the checksum, so skip over it
+        // if it's present
+        IO_advance_input(in, 4);
+    }
+}
+/******* END FRAME DECODING ***************************************************/
+
+/******* BLOCK DECOMPRESSION **************************************************/
+static void decompress_block(frame_context_t *const ctx, ostream_t *const out,
+                             istream_t *const in) {
+    // "A compressed block consists of 2 sections :
+    //
+    // Literals_Section
+    // Sequences_Section"
+
+
+    // Part 1: decode the literals block
+    u8 *literals = NULL;
+    const size_t literals_size = decode_literals(ctx, in, &literals);
+
+    // Part 2: decode the sequences block
+    sequence_command_t *sequences = NULL;
+    const size_t num_sequences =
+        decode_sequences(ctx, in, &sequences);
+
+    // Part 3: combine literals and sequence commands to generate output
+    execute_sequences(ctx, out, literals, literals_size, sequences,
+                      num_sequences);
+    free(literals);
+    free(sequences);
+}
+/******* END BLOCK DECOMPRESSION **********************************************/
+
+/******* LITERALS DECODING ****************************************************/
+static size_t decode_literals_simple(istream_t *const in, u8 **const literals,
+                                     const int block_type,
+                                     const int size_format);
+static size_t decode_literals_compressed(frame_context_t *const ctx,
+                                         istream_t *const in,
+                                         u8 **const literals,
+                                         const int block_type,
+                                         const int size_format);
+static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable);
+static void fse_decode_hufweights(ostream_t *weights, istream_t *const in,
+                                    int *const num_symbs);
+
+static size_t decode_literals(frame_context_t *const ctx, istream_t *const in,
+                              u8 **const literals) {
+    // "Literals can be stored uncompressed or compressed using Huffman prefix
+    // codes. When compressed, an optional tree description can be present,
+    // followed by 1 or 4 streams."
+    //
+    // "Literals_Section_Header
+    //
+    // Header is in charge of describing how literals are packed. It's a
+    // byte-aligned variable-size bitfield, ranging from 1 to 5 bytes, using
+    // little-endian convention."
+    //
+    // "Literals_Block_Type
+    //
+    // This field uses 2 lowest bits of first byte, describing 4 different block
+    // types"
+    //
+    // size_format takes between 1 and 2 bits
+    int block_type = IO_read_bits(in, 2);
+    int size_format = IO_read_bits(in, 2);
+
+    if (block_type <= 1) {
+        // Raw or RLE literals block
+        return decode_literals_simple(in, literals, block_type,
+                                      size_format);
+    } else {
+        // Huffman compressed literals
+        return decode_literals_compressed(ctx, in, literals, block_type,
+                                          size_format);
+    }
+}
+
+/// Decodes literals blocks in raw or RLE form
+static size_t decode_literals_simple(istream_t *const in, u8 **const literals,
+                                     const int block_type,
+                                     const int size_format) {
+    size_t size;
+    switch (size_format) {
+    // These cases are in the form ?0
+    // In this case, the ? bit is actually part of the size field
+    case 0:
+    case 2:
+        // "Size_Format uses 1 bit. Regenerated_Size uses 5 bits (0-31)."
+        IO_rewind_bits(in, 1);
+        size = IO_read_bits(in, 5);
+        break;
+    case 1:
+        // "Size_Format uses 2 bits. Regenerated_Size uses 12 bits (0-4095)."
+        size = IO_read_bits(in, 12);
+        break;
+    case 3:
+        // "Size_Format uses 2 bits. Regenerated_Size uses 20 bits (0-1048575)."
+        size = IO_read_bits(in, 20);
+        break;
+    default:
+        // Size format is in range 0-3
+        IMPOSSIBLE();
+    }
+
+    if (size > MAX_LITERALS_SIZE) {
+        CORRUPTION();
+    }
+
+    *literals = malloc(size);
+    if (!*literals) {
+        BAD_ALLOC();
+    }
+
+    switch (block_type) {
+    case 0: {
+        // "Raw_Literals_Block - Literals are stored uncompressed."
+        const u8 *const read_ptr = IO_read_bytes(in, size);
+        memcpy(*literals, read_ptr, size);
+        break;
+    }
+    case 1: {
+        // "RLE_Literals_Block - Literals consist of a single byte value repeated N times."
+        const u8 *const read_ptr = IO_read_bytes(in, 1);
+        memset(*literals, read_ptr[0], size);
+        break;
+    }
+    default:
+        IMPOSSIBLE();
+    }
+
+    return size;
+}
+
+/// Decodes Huffman compressed literals
+static size_t decode_literals_compressed(frame_context_t *const ctx,
+                                         istream_t *const in,
+                                         u8 **const literals,
+                                         const int block_type,
+                                         const int size_format) {
+    size_t regenerated_size, compressed_size;
+    // Only size_format=0 has 1 stream, so default to 4
+    int num_streams = 4;
+    switch (size_format) {
+    case 0:
+        // "A single stream. Both Compressed_Size and Regenerated_Size use 10
+        // bits (0-1023)."
+        num_streams = 1;
+    // Fall through as it has the same size format
+    case 1:
+        // "4 streams. Both Compressed_Size and Regenerated_Size use 10 bits
+        // (0-1023)."
+        regenerated_size = IO_read_bits(in, 10);
+        compressed_size = IO_read_bits(in, 10);
+        break;
+    case 2:
+        // "4 streams. Both Compressed_Size and Regenerated_Size use 14 bits
+        // (0-16383)."
+        regenerated_size = IO_read_bits(in, 14);
+        compressed_size = IO_read_bits(in, 14);
+        break;
+    case 3:
+        // "4 streams. Both Compressed_Size and Regenerated_Size use 18 bits
+        // (0-262143)."
+        regenerated_size = IO_read_bits(in, 18);
+        compressed_size = IO_read_bits(in, 18);
+        break;
+    default:
+        // Impossible
+        IMPOSSIBLE();
+    }
+    if (regenerated_size > MAX_LITERALS_SIZE ||
+        compressed_size >= regenerated_size) {
+        CORRUPTION();
+    }
+
+    *literals = malloc(regenerated_size);
+    if (!*literals) {
+        BAD_ALLOC();
+    }
+
+    ostream_t lit_stream = IO_make_ostream(*literals, regenerated_size);
+    istream_t huf_stream = IO_make_sub_istream(in, compressed_size);
+
+    if (block_type == 2) {
+        // Decode provided Huffman table
+        // "This section is only present when Literals_Block_Type type is
+        // Compressed_Literals_Block (2)."
+
+        HUF_free_dtable(&ctx->literals_dtable);
+        decode_huf_table(&huf_stream, &ctx->literals_dtable);
+    } else {
+        // If the previous Huffman table is being repeated, ensure it exists
+        if (!ctx->literals_dtable.symbols) {
+            CORRUPTION();
+        }
+    }
+
+    size_t symbols_decoded;
+    if (num_streams == 1) {
+        symbols_decoded = HUF_decompress_1stream(&ctx->literals_dtable, &lit_stream, &huf_stream);
+    } else {
+        symbols_decoded = HUF_decompress_4stream(&ctx->literals_dtable, &lit_stream, &huf_stream);
+    }
+
+    if (symbols_decoded != regenerated_size) {
+        CORRUPTION();
+    }
+
+    return regenerated_size;
+}
+
+// Decode the Huffman table description
+static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable) {
+    const u8 header = IO_read_bits(in, 8);
+
+    // "All literal values from zero (included) to last present one (excluded)
+    // are represented by Weight with values from 0 to Max_Number_of_Bits."
+
+    // "This is a single byte value (0-255), which describes how to decode the list of weights."
+    u8 weights[HUF_MAX_SYMBS];
+    memset(weights, 0, sizeof(weights));
+
+    int num_symbs;
+
+    if (header >= 128) {
+        // "This is a direct representation, where each Weight is written
+        // directly as a 4 bits field (0-15). The full representation occupies
+        // ((Number_of_Symbols+1)/2) bytes, meaning it uses a last full byte
+        // even if Number_of_Symbols is odd. Number_of_Symbols = headerByte -
+        // 127"
+        num_symbs = header - 127;
+        const size_t bytes = (num_symbs + 1) / 2;
+
+        const u8 *const weight_src = IO_read_bytes(in, bytes);
+
+        for (int i = 0; i < num_symbs; i++) {
+            // "They are encoded forward, 2
+            // weights to a byte with the first weight taking the top four bits
+            // and the second taking the bottom four (e.g. the following
+            // operations could be used to read the weights: Weight[0] =
+            // (Byte[0] >> 4), Weight[1] = (Byte[0] & 0xf), etc.)."
+            if (i % 2 == 0) {
+                weights[i] = weight_src[i / 2] >> 4;
+            } else {
+                weights[i] = weight_src[i / 2] & 0xf;
+            }
+        }
+    } else {
+        // The weights are FSE encoded, decode them before we can construct the
+        // table
+        istream_t fse_stream = IO_make_sub_istream(in, header);
+        ostream_t weight_stream = IO_make_ostream(weights, HUF_MAX_SYMBS);
+        fse_decode_hufweights(&weight_stream, &fse_stream, &num_symbs);
+    }
+
+    // Construct the table using the decoded weights
+    HUF_init_dtable_usingweights(dtable, weights, num_symbs);
+}
+
+static void fse_decode_hufweights(ostream_t *weights, istream_t *const in,
+                                    int *const num_symbs) {
+    const int MAX_ACCURACY_LOG = 7;
+
+    FSE_dtable dtable;
+
+    // "An FSE bitstream starts by a header, describing probabilities
+    // distribution. It will create a Decoding Table. For a list of Huffman
+    // weights, maximum accuracy is 7 bits."
+    FSE_decode_header(&dtable, in, MAX_ACCURACY_LOG);
+
+    // Decode the weights
+    *num_symbs = FSE_decompress_interleaved2(&dtable, weights, in);
+
+    FSE_free_dtable(&dtable);
+}
+/******* END LITERALS DECODING ************************************************/
+
+/******* SEQUENCE DECODING ****************************************************/
+/// The combination of FSE states needed to decode sequences
+typedef struct {
+    FSE_dtable ll_table;
+    FSE_dtable of_table;
+    FSE_dtable ml_table;
+
+    u16 ll_state;
+    u16 of_state;
+    u16 ml_state;
+} sequence_state_t;
+
+/// Different modes to signal to decode_seq_tables what to do
+typedef enum {
+    seq_literal_length = 0,
+    seq_offset = 1,
+    seq_match_length = 2,
+} seq_part_t;
+
+typedef enum {
+    seq_predefined = 0,
+    seq_rle = 1,
+    seq_fse = 2,
+    seq_repeat = 3,
+} seq_mode_t;
+
+/// The predefined FSE distribution tables for `seq_predefined` mode
+static const i16 SEQ_LITERAL_LENGTH_DEFAULT_DIST[36] = {
+    4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1,  1,  2,  2,
+    2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1};
+static const i16 SEQ_OFFSET_DEFAULT_DIST[29] = {
+    1, 1, 1, 1, 1, 1, 2, 2, 2, 1,  1,  1,  1,  1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1};
+static const i16 SEQ_MATCH_LENGTH_DEFAULT_DIST[53] = {
+    1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1,  1,  1,  1,  1,  1,  1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  1,  1,  1,  1,  1,  1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1};
+
+/// The sequence decoding baseline and number of additional bits to read/add
+/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets
+static const u32 SEQ_LITERAL_LENGTH_BASELINES[36] = {
+    0,  1,  2,   3,   4,   5,    6,    7,    8,    9,     10,    11,
+    12, 13, 14,  15,  16,  18,   20,   22,   24,   28,    32,    40,
+    48, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65538};
+static const u8 SEQ_LITERAL_LENGTH_EXTRA_BITS[36] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0,  0,  0,  0,  1,  1,
+    1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+
+static const u32 SEQ_MATCH_LENGTH_BASELINES[53] = {
+    3,  4,   5,   6,   7,    8,    9,    10,   11,    12,    13,   14, 15, 16,
+    17, 18,  19,  20,  21,   22,   23,   24,   25,    26,    27,   28, 29, 30,
+    31, 32,  33,  34,  35,   37,   39,   41,   43,    47,    51,   59, 67, 83,
+    99, 131, 259, 515, 1027, 2051, 4099, 8195, 16387, 32771, 65539};
+static const u8 SEQ_MATCH_LENGTH_EXTRA_BITS[53] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0,  0,  0,  0,  0,  0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0,  0,  0,  1,  1,  1, 1,
+    2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+
+/// Offset decoding is simpler so we just need a maximum code value
+static const u8 SEQ_MAX_CODES[3] = {35, -1, 52};
+
+static void decompress_sequences(frame_context_t *const ctx,
+                                 istream_t *const in,
+                                 sequence_command_t *const sequences,
+                                 const size_t num_sequences);
+static sequence_command_t decode_sequence(sequence_state_t *const state,
+                                          const u8 *const src,
+                                          i64 *const offset);
+static void decode_seq_table(istream_t *const in, FSE_dtable *const table,
+                               const seq_part_t type, const seq_mode_t mode);
+
+static size_t decode_sequences(frame_context_t *const ctx, istream_t *in,
+                               sequence_command_t **const sequences) {
+    // "A compressed block is a succession of sequences . A sequence is a
+    // literal copy command, followed by a match copy command. A literal copy
+    // command specifies a length. It is the number of bytes to be copied (or
+    // extracted) from the literal section. A match copy command specifies an
+    // offset and a length. The offset gives the position to copy from, which
+    // can be within a previous block."
+
+    size_t num_sequences;
+
+    // "Number_of_Sequences
+    //
+    // This is a variable size field using between 1 and 3 bytes. Let's call its
+    // first byte byte0."
+    u8 header = IO_read_bits(in, 8);
+    if (header == 0) {
+        // "There are no sequences. The sequence section stops there.
+        // Regenerated content is defined entirely by literals section."
+        *sequences = NULL;
+        return 0;
+    } else if (header < 128) {
+        // "Number_of_Sequences = byte0 . Uses 1 byte."
+        num_sequences = header;
+    } else if (header < 255) {
+        // "Number_of_Sequences = ((byte0-128) << 8) + byte1 . Uses 2 bytes."
+        num_sequences = ((header - 128) << 8) + IO_read_bits(in, 8);
+    } else {
+        // "Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00 . Uses 3 bytes."
+        num_sequences = IO_read_bits(in, 16) + 0x7F00;
+    }
+
+    *sequences = malloc(num_sequences * sizeof(sequence_command_t));
+    if (!*sequences) {
+        BAD_ALLOC();
+    }
+
+    decompress_sequences(ctx, in, *sequences, num_sequences);
+    return num_sequences;
+}
+
+/// Decompress the FSE encoded sequence commands
+static void decompress_sequences(frame_context_t *const ctx, istream_t *in,
+                                 sequence_command_t *const sequences,
+                                 const size_t num_sequences) {
+    // "The Sequences_Section regroup all symbols required to decode commands.
+    // There are 3 symbol types : literals lengths, offsets and match lengths.
+    // They are encoded together, interleaved, in a single bitstream."
+
+    // "Symbol compression modes
+    //
+    // This is a single byte, defining the compression mode of each symbol
+    // type."
+    //
+    // Bit number : Field name
+    // 7-6        : Literals_Lengths_Mode
+    // 5-4        : Offsets_Mode
+    // 3-2        : Match_Lengths_Mode
+    // 1-0        : Reserved
+    u8 compression_modes = IO_read_bits(in, 8);
+
+    if ((compression_modes & 3) != 0) {
+        // Reserved bits set
+        CORRUPTION();
+    }
+
+    // "Following the header, up to 3 distribution tables can be described. When
+    // present, they are in this order :
+    //
+    // Literals lengths
+    // Offsets
+    // Match Lengths"
+    // Update the tables we have stored in the context
+    decode_seq_table(in, &ctx->ll_dtable, seq_literal_length,
+                     (compression_modes >> 6) & 3);
+
+    decode_seq_table(in, &ctx->of_dtable, seq_offset,
+                     (compression_modes >> 4) & 3);
+
+    decode_seq_table(in, &ctx->ml_dtable, seq_match_length,
+                     (compression_modes >> 2) & 3);
+
+    // Check to make sure none of the tables are uninitialized
+    if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols ||
+        !ctx->ml_dtable.symbols) {
+        CORRUPTION();
+    }
+
+    sequence_state_t state;
+    // Copy the context's tables into the local state
+    memcpy(&state.ll_table, &ctx->ll_dtable, sizeof(FSE_dtable));
+    memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable));
+    memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable));
+
+    size_t len = IO_istream_len(in);
+    const u8 *const src = IO_read_bytes(in, len);
+
+    // "After writing the last bit containing information, the compressor writes
+    // a single 1-bit and then fills the byte with 0-7 0 bits of padding."
+    const int padding = 8 - log2inf(src[len - 1]);
+    i64 offset = len * 8 - padding;
+
+    // "The bitstream starts with initial state values, each using the required
+    // number of bits in their respective accuracy, decoded previously from
+    // their normalized distribution.
+    //
+    // It starts by Literals_Length_State, followed by Offset_State, and finally
+    // Match_Length_State."
+    FSE_init_state(&state.ll_table, &state.ll_state, src, &offset);
+    FSE_init_state(&state.of_table, &state.of_state, src, &offset);
+    FSE_init_state(&state.ml_table, &state.ml_state, src, &offset);
+
+    for (size_t i = 0; i < num_sequences; i++) {
+        // Decode sequences one by one
+        sequences[i] = decode_sequence(&state, src, &offset);
+    }
+
+    if (offset != 0) {
+        CORRUPTION();
+    }
+
+    // Don't free tables so they can be used in the next block
+}
+
+// Decode a single sequence and update the state
+static sequence_command_t decode_sequence(sequence_state_t *const state,
+                                          const u8 *const src,
+                                          i64 *const offset) {
+    // "Each symbol is a code in its own context, which specifies Baseline and
+    // Number_of_Bits to add. Codes are FSE compressed, and interleaved with raw
+    // additional bits in the same bitstream."
+
+    // Decode symbols, but don't update states
+    const u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state);
+    const u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state);
+    const u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state);
+
+    // Offset doesn't need a max value as it's not decoded using a table
+    if (ll_code > SEQ_MAX_CODES[seq_literal_length] ||
+        ml_code > SEQ_MAX_CODES[seq_match_length]) {
+        CORRUPTION();
+    }
+
+    // Read the interleaved bits
+    sequence_command_t seq;
+    // "Decoding starts by reading the Number_of_Bits required to decode Offset.
+    // It then does the same for Match_Length, and then for Literals_Length."
+    seq.offset = ((u32)1 << of_code) + STREAM_read_bits(src, of_code, offset);
+
+    seq.match_length =
+        SEQ_MATCH_LENGTH_BASELINES[ml_code] +
+        STREAM_read_bits(src, SEQ_MATCH_LENGTH_EXTRA_BITS[ml_code], offset);
+
+    seq.literal_length =
+        SEQ_LITERAL_LENGTH_BASELINES[ll_code] +
+        STREAM_read_bits(src, SEQ_LITERAL_LENGTH_EXTRA_BITS[ll_code], offset);
+
+    // "If it is not the last sequence in the block, the next operation is to
+    // update states. Using the rules pre-calculated in the decoding tables,
+    // Literals_Length_State is updated, followed by Match_Length_State, and
+    // then Offset_State."
+    // If the stream is complete don't read bits to update state
+    if (*offset != 0) {
+        FSE_update_state(&state->ll_table, &state->ll_state, src, offset);
+        FSE_update_state(&state->ml_table, &state->ml_state, src, offset);
+        FSE_update_state(&state->of_table, &state->of_state, src, offset);
+    }
+
+    return seq;
+}
+
+/// Given a sequence part and table mode, decode the FSE distribution
+static void decode_seq_table(istream_t *const in, FSE_dtable *const table,
+                               const seq_part_t type, const seq_mode_t mode) {
+    // Constant arrays indexed by seq_part_t
+    const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST,
+                                                SEQ_OFFSET_DEFAULT_DIST,
+                                                SEQ_MATCH_LENGTH_DEFAULT_DIST};
+    const size_t default_distribution_lengths[] = {36, 29, 53};
+    const size_t default_distribution_accuracies[] = {6, 5, 6};
+
+    const size_t max_accuracies[] = {9, 8, 9};
+
+    if (mode != seq_repeat) {
+        // Free old one before overwriting
+        FSE_free_dtable(table);
+    }
+
+    switch (mode) {
+    case seq_predefined: {
+        // "Predefined_Mode : uses a predefined distribution table."
+        const i16 *distribution = default_distributions[type];
+        const size_t symbs = default_distribution_lengths[type];
+        const size_t accuracy_log = default_distribution_accuracies[type];
+
+        FSE_init_dtable(table, distribution, symbs, accuracy_log);
+        break;
+    }
+    case seq_rle: {
+        // "RLE_Mode : it's a single code, repeated Number_of_Sequences times."
+        const u8 symb = IO_read_bytes(in, 1)[0];
+        FSE_init_dtable_rle(table, symb);
+        break;
+    }
+    case seq_fse: {
+        // "FSE_Compressed_Mode : standard FSE compression. A distribution table
+        // will be present "
+        FSE_decode_header(table, in, max_accuracies[type]);
+        break;
+    }
+    case seq_repeat:
+        // "Repeat_Mode : re-use distribution table from previous compressed
+        // block."
+        // Nothing to do here, table will be unchanged
+        break;
+    default:
+        // Impossible, as mode is from 0-3
+        IMPOSSIBLE();
+        break;
+    }
+}
+/******* END SEQUENCE DECODING ************************************************/
+
+/******* SEQUENCE EXECUTION ***************************************************/
+static void execute_sequences(frame_context_t *const ctx, ostream_t *const out,
+                              const u8 *const literals,
+                              const size_t literals_len,
+                              const sequence_command_t *const sequences,
+                              const size_t num_sequences) {
+    istream_t litstream = IO_make_istream(literals, literals_len);
+
+    u64 *const offset_hist = ctx->previous_offsets;
+    size_t total_output = ctx->current_total_output;
+
+    for (size_t i = 0; i < num_sequences; i++) {
+        const sequence_command_t seq = sequences[i];
+
+        {
+            if (seq.literal_length > IO_istream_len(&litstream)) {
+                CORRUPTION();
+            }
+
+            u8 *const write_ptr = IO_write_bytes(out, seq.literal_length);
+            const u8 *const read_ptr =
+                    IO_read_bytes(&litstream, seq.literal_length);
+            // Copy literals to output
+            memcpy(write_ptr, read_ptr, seq.literal_length);
+
+            total_output += seq.literal_length;
+        }
+
+        size_t offset;
+
+        // Offsets are special, we need to handle the repeat offsets
+        if (seq.offset <= 3) {
+            // "The first 3 values define a repeated offset and we will call
+            // them Repeated_Offset1, Repeated_Offset2, and Repeated_Offset3.
+            // They are sorted in recency order, with Repeated_Offset1 meaning
+            // 'most recent one'".
+
+            // Use 0 indexing for the array
+            u32 idx = seq.offset - 1;
+            if (seq.literal_length == 0) {
+                // "There is an exception though, when current sequence's
+                // literals length is 0. In this case, repeated offsets are
+                // shifted by one, so Repeated_Offset1 becomes Repeated_Offset2,
+                // Repeated_Offset2 becomes Repeated_Offset3, and
+                // Repeated_Offset3 becomes Repeated_Offset1 - 1_byte."
+                idx++;
+            }
+
+            if (idx == 0) {
+                offset = offset_hist[0];
+            } else {
+                // If idx == 3 then literal length was 0 and the offset was 3,
+                // as per the exception listed above
+                offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1;
+
+                // If idx == 1 we don't need to modify offset_hist[2]
+                if (idx > 1) {
+                    offset_hist[2] = offset_hist[1];
+                }
+                offset_hist[1] = offset_hist[0];
+                offset_hist[0] = offset;
+            }
+        } else {
+            offset = seq.offset - 3;
+
+            // Shift back history
+            offset_hist[2] = offset_hist[1];
+            offset_hist[1] = offset_hist[0];
+            offset_hist[0] = offset;
+        }
+
+        size_t match_length = seq.match_length;
+
+        u8 *write_ptr = IO_write_bytes(out, match_length);
+        if (total_output <= ctx->header.window_size) {
+            // In this case offset might go back into the dictionary
+            if (offset > total_output + ctx->dict_content_len) {
+                // The offset goes beyond even the dictionary
+                CORRUPTION();
+            }
+
+            if (offset > total_output) {
+                // "The rest of the dictionary is its content. The content act
+                // as a "past" in front of data to compress or decompress, so it
+                // can be referenced in sequence commands."
+                const size_t dict_copy =
+                    MIN(offset - total_output, match_length);
+                const size_t dict_offset =
+                    ctx->dict_content_len - (offset - total_output);
+
+                memcpy(write_ptr, ctx->dict_content + dict_offset, dict_copy);
+                write_ptr += dict_copy;
+                match_length -= dict_copy;
+            }
+        } else if (offset > ctx->header.window_size) {
+            CORRUPTION();
+        }
+
+        // We must copy byte by byte because the match length might be larger
+        // than the offset
+        // ex: if the output so far was "abc", a command with offset=3 and
+        // match_length=6 would produce "abcabcabc" as the new output
+        for (size_t i = 0; i < match_length; i++) {
+            *write_ptr = *(write_ptr - offset);
+            write_ptr++;
+        }
+
+        total_output += seq.match_length;
+    }
+
+    {
+        size_t len = IO_istream_len(&litstream);
+        u8 *const write_ptr = IO_write_bytes(out, len);
+        const u8 *const read_ptr = IO_read_bytes(&litstream, len);
+        // Copy any leftover literals
+        memcpy(write_ptr, read_ptr, len);
+
+        total_output += len;
+    }
+
+    ctx->current_total_output = total_output;
+}
+/******* END SEQUENCE EXECUTION ***********************************************/
+
+/******* OUTPUT SIZE COUNTING *************************************************/
+static void traverse_frame(const frame_header_t *const header, istream_t *const in);
+
+/// Get the decompressed size of an input stream so memory can be allocated in
+/// advance.
+/// This is more complex than the implementation in the reference
+/// implementation, as this API allows for the decompression of multiple
+/// concatenated frames.
+size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) {
+    istream_t in = IO_make_istream(src, src_len);
+    size_t dst_size = 0;
+
+    // Each frame header only gives us the size of its frame, so iterate over
+    // all
+    // frames
+    while (IO_istream_len(&in) > 0) {
+        const u32 magic_number = IO_read_bits(&in, 32);
+
+        if ((magic_number & ~0xFU) == 0x184D2A50U) {
+            // skippable frame, this has no impact on output size
+            const size_t frame_size = IO_read_bits(&in, 32);
+            IO_advance_input(&in, frame_size);
+        } else if (magic_number == 0xFD2FB528U) {
+            // ZSTD frame
+            frame_header_t header;
+            parse_frame_header(&header, &in);
+
+            if (header.frame_content_size == 0 && !header.single_segment_flag) {
+                // Content size not provided, we can't tell
+                return -1;
+            }
+
+            dst_size += header.frame_content_size;
+
+            // Consume the input from the frame to reach the start of the next
+            traverse_frame(&header, &in);
+        } else {
+            // not a real frame
+            ERROR("Invalid magic number");
+        }
+    }
+
+    return dst_size;
+}
+
+/// Iterate over each block in a frame to find the end of it, to get to the
+/// start of the next frame
+static void traverse_frame(const frame_header_t *const header, istream_t *const in) {
+    int last_block = 0;
+
+    do {
+        // Parse the block header
+        last_block = IO_read_bits(in, 1);
+        const int block_type = IO_read_bits(in, 2);
+        const size_t block_len = IO_read_bits(in, 21);
+
+        switch (block_type) {
+        case 0: // Raw block, block_len bytes
+            IO_advance_input(in, block_len);
+            break;
+        case 1: // RLE block, 1 byte
+            IO_advance_input(in, 1);
+            break;
+        case 2: // Compressed block, compressed size is block_len
+            IO_advance_input(in, block_len);
+            break;
+        case 3:
+            // Reserved block type
+            CORRUPTION();
+            break;
+        default:
+            IMPOSSIBLE();
+        }
+    } while (!last_block);
+
+    if (header->content_checksum_flag) {
+        IO_advance_input(in, 4);
+    }
+}
+
+/******* END OUTPUT SIZE COUNTING *********************************************/
+
+/******* DICTIONARY PARSING ***************************************************/
+static void init_dictionary_content(dictionary_t *const dict,
+                                    istream_t *const in);
+
+static void parse_dictionary(dictionary_t *const dict, const u8 *src,
+                             size_t src_len) {
+    memset(dict, 0, sizeof(dictionary_t));
+    if (src_len < 8) {
+        INP_SIZE();
+    }
+
+    istream_t in = IO_make_istream(src, src_len);
+
+    const u32 magic_number = IO_read_bits(&in, 32);
+    if (magic_number != 0xEC30A437) {
+        // raw content dict
+        IO_rewind_bits(&in, 32);
+        init_dictionary_content(dict, &in);
+        return;
+    }
+
+    dict->dictionary_id = IO_read_bits(&in, 32);
+
+    // "Entropy_Tables : following the same format as the tables in compressed
+    // blocks. They are stored in following order : Huffman tables for literals,
+    // FSE table for offsets, FSE table for match lengths, and FSE table for
+    // literals lengths. It's finally followed by 3 offset values, populating
+    // recent offsets (instead of using {1,4,8}), stored in order, 4-bytes
+    // little-endian each, for a total of 12 bytes. Each recent offset must have
+    // a value < dictionary size."
+    decode_huf_table(&in, &dict->literals_dtable);
+    decode_seq_table(&in, &dict->of_dtable, seq_offset, seq_fse);
+    decode_seq_table(&in, &dict->ml_dtable, seq_match_length, seq_fse);
+    decode_seq_table(&in, &dict->ll_dtable, seq_literal_length, seq_fse);
+
+    // Read in the previous offset history
+    dict->previous_offsets[0] = IO_read_bits(&in, 32);
+    dict->previous_offsets[1] = IO_read_bits(&in, 32);
+    dict->previous_offsets[2] = IO_read_bits(&in, 32);
+
+    // Ensure the provided offsets aren't too large
+    // "Each recent offset must have a value < dictionary size."
+    for (int i = 0; i < 3; i++) {
+        if (dict->previous_offsets[i] > src_len) {
+            ERROR("Dictionary corrupted");
+        }
+    }
+
+    // "Content : The rest of the dictionary is its content. The content act as
+    // a "past" in front of data to compress or decompress, so it can be
+    // referenced in sequence commands."
+    init_dictionary_content(dict, &in);
+}
+
+static void init_dictionary_content(dictionary_t *const dict,
+                                    istream_t *const in) {
+    // Copy in the content
+    dict->content_size = IO_istream_len(in);
+    dict->content = malloc(dict->content_size);
+    if (!dict->content) {
+        BAD_ALLOC();
+    }
+
+    const u8 *const content = IO_read_bytes(in, dict->content_size);
+
+    memcpy(dict->content, content, dict->content_size);
+}
+
+/// Free an allocated dictionary
+static void free_dictionary(dictionary_t *const dict) {
+    HUF_free_dtable(&dict->literals_dtable);
+    FSE_free_dtable(&dict->ll_dtable);
+    FSE_free_dtable(&dict->of_dtable);
+    FSE_free_dtable(&dict->ml_dtable);
+
+    free(dict->content);
+
+    memset(dict, 0, sizeof(dictionary_t));
+}
+/******* END DICTIONARY PARSING ***********************************************/
+
+/******* IO STREAM OPERATIONS *************************************************/
+#define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream")
+/// Reads `num` bits from a bitstream, and updates the internal offset
+static inline u64 IO_read_bits(istream_t *const in, const int num) {
+    if (num > 64 || num <= 0) {
+        ERROR("Attempt to read an invalid number of bits");
+    }
+
+    const size_t bytes = (num + in->bit_offset + 7) / 8;
+    const size_t full_bytes = (num + in->bit_offset) / 8;
+    if (bytes > in->len) {
+        INP_SIZE();
+    }
+
+    const u64 result = read_bits_LE(in->ptr, num, in->bit_offset);
+
+    in->bit_offset = (num + in->bit_offset) % 8;
+    in->ptr += full_bytes;
+    in->len -= full_bytes;
+
+    return result;
+}
+
+/// If a non-zero number of bits have been read from the current byte, advance
+/// the offset to the next byte
+static inline void IO_rewind_bits(istream_t *const in, int num) {
+    if (num < 0) {
+        ERROR("Attempting to rewind stream by a negative number of bits");
+    }
+
+    const int new_offset = in->bit_offset - num;
+    const i64 bytes = (new_offset - 7) / 8;
+
+    in->ptr += bytes;
+    in->len -= bytes;
+    in->bit_offset = ((new_offset % 8) + 8) % 8;
+}
+
+/// If the remaining bits in a byte will be unused, advance to the end of the
+/// byte
+static inline void IO_align_stream(istream_t *const in) {
+    if (in->bit_offset != 0) {
+        if (in->len == 0) {
+            INP_SIZE();
+        }
+        in->ptr++;
+        in->len--;
+        in->bit_offset = 0;
+    }
+}
+
+/// Write the given byte into the output stream
+static inline void IO_write_byte(ostream_t *const out, u8 symb) {
+    if (out->len == 0) {
+        OUT_SIZE();
+    }
+
+    out->ptr[0] = symb;
+    out->ptr++;
+    out->len--;
+}
+
+/// Returns the number of bytes left to be read in this stream.  The stream must
+/// be byte aligned.
+static inline size_t IO_istream_len(const istream_t *const in) {
+    return in->len;
+}
+
+/// Returns a pointer where `len` bytes can be read, and advances the internal
+/// state.  The stream must be byte aligned.
+static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) {
+    if (len > in->len) {
+        INP_SIZE();
+    }
+    if (in->bit_offset != 0) {
+        UNALIGNED();
+    }
+    const u8 *const ptr = in->ptr;
+    in->ptr += len;
+    in->len -= len;
+
+    return ptr;
+}
+/// Returns a pointer to write `len` bytes to, and advances the internal state
+static inline u8 *IO_write_bytes(ostream_t *const out, size_t len) {
+    if (len > out->len) {
+        OUT_SIZE();
+    }
+    u8 *const ptr = out->ptr;
+    out->ptr += len;
+    out->len -= len;
+
+    return ptr;
+}
+
+/// Advance the inner state by `len` bytes
+static inline void IO_advance_input(istream_t *const in, size_t len) {
+    if (len > in->len) {
+         INP_SIZE();
+    }
+    if (in->bit_offset != 0) {
+        UNALIGNED();
+    }
+
+    in->ptr += len;
+    in->len -= len;
+}
+
+/// Returns an `ostream_t` constructed from the given pointer and length
+static inline ostream_t IO_make_ostream(u8 *out, size_t len) {
+    return (ostream_t) { out, len };
+}
+
+/// Returns an `istream_t` constructed from the given pointer and length
+static inline istream_t IO_make_istream(const u8 *in, size_t len) {
+    return (istream_t) { in, 0, len };
+}
+
+/// Returns an `istream_t` with the same base as `in`, and length `len`
+/// Then, advance `in` to account for the consumed bytes
+/// `in` must be byte aligned
+static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len) {
+    if (len > in->len) {
+        INP_SIZE();
+    }
+    if (in->bit_offset != 0) {
+        UNALIGNED();
+    }
+    const istream_t sub = { in->ptr, in->bit_offset, len };
+
+    in->ptr += len;
+    in->len -= len;
+
+    return sub;
+}
+/******* END IO STREAM OPERATIONS *********************************************/
+
+/******* BITSTREAM OPERATIONS *************************************************/
+/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits
+static inline u64 read_bits_LE(const u8 *src, const int num,
+                               const size_t offset) {
+    if (num > 64) {
+        ERROR("Attempt to read an invalid number of bits");
+    }
+
+    // Skip over bytes that aren't in range
+    src += offset / 8;
+    size_t bit_offset = offset % 8;
+    u64 res = 0;
+
+    int shift = 0;
+    int left = num;
+    while (left > 0) {
+        u64 mask = left >= 8 ? 0xff : (((u64)1 << left) - 1);
+        // Dead the next byte, shift it to account for the offset, and then mask
+        // out the top part if we don't need all the bits
+        res += (((u64)*src++ >> bit_offset) & mask) << shift;
+        shift += 8 - bit_offset;
+        left -= 8 - bit_offset;
+        bit_offset = 0;
+    }
+
+    return res;
+}
+
+/// Read bits from the end of a HUF or FSE bitstream.  `offset` is in bits, so
+/// it updates `offset` to `offset - bits`, and then reads `bits` bits from
+/// `src + offset`.  If the offset becomes negative, the extra bits at the
+/// bottom are filled in with `0` bits instead of reading from before `src`.
+static inline u64 STREAM_read_bits(const u8 *const src, const int bits,
+                                   i64 *const offset) {
+    *offset = *offset - bits;
+    size_t actual_off = *offset;
+    size_t actual_bits = bits;
+    // Don't actually read bits from before the start of src, so if `*offset <
+    // 0` fix actual_off and actual_bits to reflect the quantity to read
+    if (*offset < 0) {
+        actual_bits += *offset;
+        actual_off = 0;
+    }
+    u64 res = read_bits_LE(src, actual_bits, actual_off);
+
+    if (*offset < 0) {
+        // Fill in the bottom "overflowed" bits with 0's
+        res = -*offset >= 64 ? 0 : (res << -*offset);
+    }
+    return res;
+}
+/******* END BITSTREAM OPERATIONS *********************************************/
+
+/******* BIT COUNTING OPERATIONS **********************************************/
+/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to
+/// `num`, or `-1` if `num == 0`.
+static inline int log2inf(const u64 num) {
+    for (int i = 63; i >= 0; i--) {
+        if (((u64)1 << i) <= num) {
+            return i;
+        }
+    }
+    return -1;
+}
+/******* END BIT COUNTING OPERATIONS ******************************************/
+
+/******* HUFFMAN PRIMITIVES ***************************************************/
+static inline u8 HUF_decode_symbol(const HUF_dtable *const dtable,
+                                   u16 *const state, const u8 *const src,
+                                   i64 *const offset) {
+    // Look up the symbol and number of bits to read
+    const u8 symb = dtable->symbols[*state];
+    const u8 bits = dtable->num_bits[*state];
+    const u16 rest = STREAM_read_bits(src, bits, offset);
+    // Shift `bits` bits out of the state, keeping the low order bits that
+    // weren't necessary to determine this symbol.  Then add in the new bits
+    // read from the stream.
+    *state = ((*state << bits) + rest) & (((u16)1 << dtable->max_bits) - 1);
+
+    return symb;
+}
+
+static inline void HUF_init_state(const HUF_dtable *const dtable,
+                                  u16 *const state, const u8 *const src,
+                                  i64 *const offset) {
+    // Read in a full `dtable->max_bits` bits to initialize the state
+    const u8 bits = dtable->max_bits;
+    *state = STREAM_read_bits(src, bits, offset);
+}
+
+static size_t HUF_decompress_1stream(const HUF_dtable *const dtable,
+                                     ostream_t *const out,
+                                     istream_t *const in) {
+    const size_t len = IO_istream_len(in);
+    if (len == 0) {
+        INP_SIZE();
+    }
+    const u8 *const src = IO_read_bytes(in, len);
+
+    // "Each bitstream must be read backward, that is starting from the end down
+    // to the beginning. Therefore it's necessary to know the size of each
+    // bitstream.
+    //
+    // It's also necessary to know exactly which bit is the latest. This is
+    // detected by a final bit flag : the highest bit of latest byte is a
+    // final-bit-flag. Consequently, a last byte of 0 is not possible. And the
+    // final-bit-flag itself is not part of the useful bitstream. Hence, the
+    // last byte contains between 0 and 7 useful bits."
+    const int padding = 8 - log2inf(src[len - 1]);
+
+    i64 offset = len * 8 - padding;
+    u16 state;
+
+    HUF_init_state(dtable, &state, src, &offset);
+
+    size_t symbols_written = 0;
+    while (offset > -dtable->max_bits) {
+        // Iterate over the stream, decoding one symbol at a time
+        IO_write_byte(out, HUF_decode_symbol(dtable, &state, src, &offset));
+        symbols_written++;
+    }
+    // "The process continues up to reading the required number of symbols per
+    // stream. If a bitstream is not entirely and exactly consumed, hence
+    // reaching exactly its beginning position with all bits consumed, the
+    // decoding process is considered faulty."
+
+    // When all symbols have been decoded, the final state value shouldn't have
+    // any data from the stream, so it should have "read" dtable->max_bits from
+    // before the start of `src`
+    // Therefore `offset`, the edge to start reading new bits at, should be
+    // dtable->max_bits before the start of the stream
+    if (offset != -dtable->max_bits) {
+        CORRUPTION();
+    }
+
+    return symbols_written;
+}
+
+static size_t HUF_decompress_4stream(const HUF_dtable *const dtable,
+                                     ostream_t *const out, istream_t *const in) {
+    // "Compressed size is provided explicitly : in the 4-streams variant,
+    // bitstreams are preceded by 3 unsigned little-endian 16-bits values. Each
+    // value represents the compressed size of one stream, in order. The last
+    // stream size is deducted from total compressed size and from previously
+    // decoded stream sizes"
+    const size_t csize1 = IO_read_bits(in, 16);
+    const size_t csize2 = IO_read_bits(in, 16);
+    const size_t csize3 = IO_read_bits(in, 16);
+
+    istream_t in1 = IO_make_sub_istream(in, csize1);
+    istream_t in2 = IO_make_sub_istream(in, csize2);
+    istream_t in3 = IO_make_sub_istream(in, csize3);
+    istream_t in4 = IO_make_sub_istream(in, IO_istream_len(in));
+
+    size_t total_output = 0;
+    // Decode each stream independently for simplicity
+    // If we wanted to we could decode all 4 at the same time for speed,
+    // utilizing more execution units
+    total_output += HUF_decompress_1stream(dtable, out, &in1);
+    total_output += HUF_decompress_1stream(dtable, out, &in2);
+    total_output += HUF_decompress_1stream(dtable, out, &in3);
+    total_output += HUF_decompress_1stream(dtable, out, &in4);
+
+    return total_output;
+}
+
+/// Initializes a Huffman table using canonical Huffman codes
+/// For more explanation on canonical Huffman codes see
+/// http://www.cs.uofs.edu/~mccloske/courses/cmps340/huff_canonical_dec2015.html
+/// Codes within a level are allocated in symbol order (i.e. smaller symbols get
+/// earlier codes)
+static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits,
+                            const int num_symbs) {
+    memset(table, 0, sizeof(HUF_dtable));
+    if (num_symbs > HUF_MAX_SYMBS) {
+        ERROR("Too many symbols for Huffman");
+    }
+
+    u8 max_bits = 0;
+    u16 rank_count[HUF_MAX_BITS + 1];
+    memset(rank_count, 0, sizeof(rank_count));
+
+    // Count the number of symbols for each number of bits, and determine the
+    // depth of the tree
+    for (int i = 0; i < num_symbs; i++) {
+        if (bits[i] > HUF_MAX_BITS) {
+            ERROR("Huffman table depth too large");
+        }
+        max_bits = MAX(max_bits, bits[i]);
+        rank_count[bits[i]]++;
+    }
+
+    const size_t table_size = 1 << max_bits;
+    table->max_bits = max_bits;
+    table->symbols = malloc(table_size);
+    table->num_bits = malloc(table_size);
+
+    if (!table->symbols || !table->num_bits) {
+        free(table->symbols);
+        free(table->num_bits);
+        BAD_ALLOC();
+    }
+
+    // "Symbols are sorted by Weight. Within same Weight, symbols keep natural
+    // order. Symbols with a Weight of zero are removed. Then, starting from
+    // lowest weight, prefix codes are distributed in order."
+
+    u32 rank_idx[HUF_MAX_BITS + 1];
+    // Initialize the starting codes for each rank (number of bits)
+    rank_idx[max_bits] = 0;
+    for (int i = max_bits; i >= 1; i--) {
+        rank_idx[i - 1] = rank_idx[i] + rank_count[i] * (1 << (max_bits - i));
+        // The entire range takes the same number of bits so we can memset it
+        memset(&table->num_bits[rank_idx[i]], i, rank_idx[i - 1] - rank_idx[i]);
+    }
+
+    if (rank_idx[0] != table_size) {
+        CORRUPTION();
+    }
+
+    // Allocate codes and fill in the table
+    for (int i = 0; i < num_symbs; i++) {
+        if (bits[i] != 0) {
+            // Allocate a code for this symbol and set its range in the table
+            const u16 code = rank_idx[bits[i]];
+            // Since the code doesn't care about the bottom `max_bits - bits[i]`
+            // bits of state, it gets a range that spans all possible values of
+            // the lower bits
+            const u16 len = 1 << (max_bits - bits[i]);
+            memset(&table->symbols[code], i, len);
+            rank_idx[bits[i]] += len;
+        }
+    }
+}
+
+static void HUF_init_dtable_usingweights(HUF_dtable *const table,
+                                         const u8 *const weights,
+                                         const int num_symbs) {
+    // +1 because the last weight is not transmitted in the header
+    if (num_symbs + 1 > HUF_MAX_SYMBS) {
+        ERROR("Too many symbols for Huffman");
+    }
+
+    u8 bits[HUF_MAX_SYMBS];
+
+    u64 weight_sum = 0;
+    for (int i = 0; i < num_symbs; i++) {
+        // Weights are in the same range as bit count
+        if (weights[i] > HUF_MAX_BITS) {
+            CORRUPTION();
+        }
+        weight_sum += weights[i] > 0 ? (u64)1 << (weights[i] - 1) : 0;
+    }
+
+    // Find the first power of 2 larger than the sum
+    const int max_bits = log2inf(weight_sum) + 1;
+    const u64 left_over = ((u64)1 << max_bits) - weight_sum;
+    // If the left over isn't a power of 2, the weights are invalid
+    if (left_over & (left_over - 1)) {
+        CORRUPTION();
+    }
+
+    // left_over is used to find the last weight as it's not transmitted
+    // by inverting 2^(weight - 1) we can determine the value of last_weight
+    const int last_weight = log2inf(left_over) + 1;
+
+    for (int i = 0; i < num_symbs; i++) {
+        // "Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0"
+        bits[i] = weights[i] > 0 ? (max_bits + 1 - weights[i]) : 0;
+    }
+    bits[num_symbs] =
+        max_bits + 1 - last_weight; // Last weight is always non-zero
+
+    HUF_init_dtable(table, bits, num_symbs + 1);
+}
+
+static void HUF_free_dtable(HUF_dtable *const dtable) {
+    free(dtable->symbols);
+    free(dtable->num_bits);
+    memset(dtable, 0, sizeof(HUF_dtable));
+}
+
+static void HUF_copy_dtable(HUF_dtable *const dst,
+                            const HUF_dtable *const src) {
+    if (src->max_bits == 0) {
+        memset(dst, 0, sizeof(HUF_dtable));
+        return;
+    }
+
+    const size_t size = (size_t)1 << src->max_bits;
+    dst->max_bits = src->max_bits;
+
+    dst->symbols = malloc(size);
+    dst->num_bits = malloc(size);
+    if (!dst->symbols || !dst->num_bits) {
+        BAD_ALLOC();
+    }
+
+    memcpy(dst->symbols, src->symbols, size);
+    memcpy(dst->num_bits, src->num_bits, size);
+}
+/******* END HUFFMAN PRIMITIVES ***********************************************/
+
+/******* FSE PRIMITIVES *******************************************************/
+/// For more description of FSE see
+/// https://github.com/Cyan4973/FiniteStateEntropy/
+
+/// Allow a symbol to be decoded without updating state
+static inline u8 FSE_peek_symbol(const FSE_dtable *const dtable,
+                                 const u16 state) {
+    return dtable->symbols[state];
+}
+
+/// Consumes bits from the input and uses the current state to determine the
+/// next state
+static inline void FSE_update_state(const FSE_dtable *const dtable,
+                                    u16 *const state, const u8 *const src,
+                                    i64 *const offset) {
+    const u8 bits = dtable->num_bits[*state];
+    const u16 rest = STREAM_read_bits(src, bits, offset);
+    *state = dtable->new_state_base[*state] + rest;
+}
+
+/// Decodes a single FSE symbol and updates the offset
+static inline u8 FSE_decode_symbol(const FSE_dtable *const dtable,
+                                   u16 *const state, const u8 *const src,
+                                   i64 *const offset) {
+    const u8 symb = FSE_peek_symbol(dtable, *state);
+    FSE_update_state(dtable, state, src, offset);
+    return symb;
+}
+
+static inline void FSE_init_state(const FSE_dtable *const dtable,
+                                  u16 *const state, const u8 *const src,
+                                  i64 *const offset) {
+    // Read in a full `accuracy_log` bits to initialize the state
+    const u8 bits = dtable->accuracy_log;
+    *state = STREAM_read_bits(src, bits, offset);
+}
+
+static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable,
+                                          ostream_t *const out,
+                                          istream_t *const in) {
+    const size_t len = IO_istream_len(in);
+    if (len == 0) {
+        INP_SIZE();
+    }
+    const u8 *const src = IO_read_bytes(in, len);
+
+    // "Each bitstream must be read backward, that is starting from the end down
+    // to the beginning. Therefore it's necessary to know the size of each
+    // bitstream.
+    //
+    // It's also necessary to know exactly which bit is the latest. This is
+    // detected by a final bit flag : the highest bit of latest byte is a
+    // final-bit-flag. Consequently, a last byte of 0 is not possible. And the
+    // final-bit-flag itself is not part of the useful bitstream. Hence, the
+    // last byte contains between 0 and 7 useful bits."
+    const int padding = 8 - log2inf(src[len - 1]);
+    i64 offset = len * 8 - padding;
+
+    u16 state1, state2;
+    // "The first state (State1) encodes the even indexed symbols, and the
+    // second (State2) encodes the odd indexes. State1 is initialized first, and
+    // then State2, and they take turns decoding a single symbol and updating
+    // their state."
+    FSE_init_state(dtable, &state1, src, &offset);
+    FSE_init_state(dtable, &state2, src, &offset);
+
+    // Decode until we overflow the stream
+    // Since we decode in reverse order, overflowing the stream is offset going
+    // negative
+    size_t symbols_written = 0;
+    while (1) {
+        // "The number of symbols to decode is determined by tracking bitStream
+        // overflow condition: If updating state after decoding a symbol would
+        // require more bits than remain in the stream, it is assumed the extra
+        // bits are 0. Then, the symbols for each of the final states are
+        // decoded and the process is complete."
+        IO_write_byte(out, FSE_decode_symbol(dtable, &state1, src, &offset));
+        symbols_written++;
+        if (offset < 0) {
+            // There's still a symbol to decode in state2
+            IO_write_byte(out, FSE_peek_symbol(dtable, state2));
+            symbols_written++;
+            break;
+        }
+
+        IO_write_byte(out, FSE_decode_symbol(dtable, &state2, src, &offset));
+        symbols_written++;
+        if (offset < 0) {
+            // There's still a symbol to decode in state1
+            IO_write_byte(out, FSE_peek_symbol(dtable, state1));
+            symbols_written++;
+            break;
+        }
+    }
+
+    return symbols_written;
+}
+
+static void FSE_init_dtable(FSE_dtable *const dtable,
+                            const i16 *const norm_freqs, const int num_symbs,
+                            const int accuracy_log) {
+    if (accuracy_log > FSE_MAX_ACCURACY_LOG) {
+        ERROR("FSE accuracy too large");
+    }
+    if (num_symbs > FSE_MAX_SYMBS) {
+        ERROR("Too many symbols for FSE");
+    }
+
+    dtable->accuracy_log = accuracy_log;
+
+    const size_t size = (size_t)1 << accuracy_log;
+    dtable->symbols = malloc(size * sizeof(u8));
+    dtable->num_bits = malloc(size * sizeof(u8));
+    dtable->new_state_base = malloc(size * sizeof(u16));
+
+    if (!dtable->symbols || !dtable->num_bits || !dtable->new_state_base) {
+        BAD_ALLOC();
+    }
+
+    // Used to determine how many bits need to be read for each state,
+    // and where the destination range should start
+    // Needs to be u16 because max value is 2 * max number of symbols,
+    // which can be larger than a byte can store
+    u16 state_desc[FSE_MAX_SYMBS];
+
+    // "Symbols are scanned in their natural order for "less than 1"
+    // probabilities. Symbols with this probability are being attributed a
+    // single cell, starting from the end of the table. These symbols define a
+    // full state reset, reading Accuracy_Log bits."
+    int high_threshold = size;
+    for (int s = 0; s < num_symbs; s++) {
+        // Scan for low probability symbols to put at the top
+        if (norm_freqs[s] == -1) {
+            dtable->symbols[--high_threshold] = s;
+            state_desc[s] = 1;
+        }
+    }
+
+    // "All remaining symbols are sorted in their natural order. Starting from
+    // symbol 0 and table position 0, each symbol gets attributed as many cells
+    // as its probability. Cell allocation is spreaded, not linear."
+    // Place the rest in the table
+    const u16 step = (size >> 1) + (size >> 3) + 3;
+    const u16 mask = size - 1;
+    u16 pos = 0;
+    for (int s = 0; s < num_symbs; s++) {
+        if (norm_freqs[s] <= 0) {
+            continue;
+        }
+
+        state_desc[s] = norm_freqs[s];
+
+        for (int i = 0; i < norm_freqs[s]; i++) {
+            // Give `norm_freqs[s]` states to symbol s
+            dtable->symbols[pos] = s;
+            // "A position is skipped if already occupied, typically by a "less
+            // than 1" probability symbol."
+            do {
+                pos = (pos + step) & mask;
+            } while (pos >=
+                     high_threshold);
+            // Note: no other collision checking is necessary as `step` is
+            // coprime to `size`, so the cycle will visit each position exactly
+            // once
+        }
+    }
+    if (pos != 0) {
+        CORRUPTION();
+    }
+
+    // Now we can fill baseline and num bits
+    for (int i = 0; i < size; i++) {
+        u8 symbol = dtable->symbols[i];
+        u16 next_state_desc = state_desc[symbol]++;
+        // Fills in the table appropriately, next_state_desc increases by symbol
+        // over time, decreasing number of bits
+        dtable->num_bits[i] = (u8)(accuracy_log - log2inf(next_state_desc));
+        // Baseline increases until the bit threshold is passed, at which point
+        // it resets to 0
+        dtable->new_state_base[i] =
+            ((u16)next_state_desc << dtable->num_bits[i]) - size;
+    }
+}
+
+/// Decode an FSE header as defined in the Zstandard format specification and
+/// use the decoded frequencies to initialize a decoding table.
+static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in,
+                                const int max_accuracy_log) {
+    // "An FSE distribution table describes the probabilities of all symbols
+    // from 0 to the last present one (included) on a normalized scale of 1 <<
+    // Accuracy_Log .
+    //
+    // It's a bitstream which is read forward, in little-endian fashion. It's
+    // not necessary to know its exact size, since it will be discovered and
+    // reported by the decoding process.
+    if (max_accuracy_log > FSE_MAX_ACCURACY_LOG) {
+        ERROR("FSE accuracy too large");
+    }
+
+    // The bitstream starts by reporting on which scale it operates.
+    // Accuracy_Log = low4bits + 5. Note that maximum Accuracy_Log for literal
+    // and match lengths is 9, and for offsets is 8. Higher values are
+    // considered errors."
+    const int accuracy_log = 5 + IO_read_bits(in, 4);
+    if (accuracy_log > max_accuracy_log) {
+        ERROR("FSE accuracy too large");
+    }
+
+    // "Then follows each symbol value, from 0 to last present one. The number
+    // of bits used by each field is variable. It depends on :
+    //
+    // Remaining probabilities + 1 : example : Presuming an Accuracy_Log of 8,
+    // and presuming 100 probabilities points have already been distributed, the
+    // decoder may read any value from 0 to 255 - 100 + 1 == 156 (inclusive).
+    // Therefore, it must read log2sup(156) == 8 bits.
+    //
+    // Value decoded : small values use 1 less bit : example : Presuming values
+    // from 0 to 156 (inclusive) are possible, 255-156 = 99 values are remaining
+    // in an 8-bits field. They are used this way : first 99 values (hence from
+    // 0 to 98) use only 7 bits, values from 99 to 156 use 8 bits. "
+
+    i32 remaining = 1 << accuracy_log;
+    i16 frequencies[FSE_MAX_SYMBS];
+
+    int symb = 0;
+    while (remaining > 0 && symb < FSE_MAX_SYMBS) {
+        // Log of the number of possible values we could read
+        int bits = log2inf(remaining + 1) + 1;
+
+        u16 val = IO_read_bits(in, bits);
+
+        // Try to mask out the lower bits to see if it qualifies for the "small
+        // value" threshold
+        const u16 lower_mask = ((u16)1 << (bits - 1)) - 1;
+        const u16 threshold = ((u16)1 << bits) - 1 - (remaining + 1);
+
+        if ((val & lower_mask) < threshold) {
+            IO_rewind_bits(in, 1);
+            val = val & lower_mask;
+        } else if (val > lower_mask) {
+            val = val - threshold;
+        }
+
+        // "Probability is obtained from Value decoded by following formula :
+        // Proba = value - 1"
+        const i16 proba = (i16)val - 1;
+
+        // "It means value 0 becomes negative probability -1. -1 is a special
+        // probability, which means "less than 1". Its effect on distribution
+        // table is described in next paragraph. For the purpose of calculating
+        // cumulated distribution, it counts as one."
+        remaining -= proba < 0 ? -proba : proba;
+
+        frequencies[symb] = proba;
+        symb++;
+
+        // "When a symbol has a probability of zero, it is followed by a 2-bits
+        // repeat flag. This repeat flag tells how many probabilities of zeroes
+        // follow the current one. It provides a number ranging from 0 to 3. If
+        // it is a 3, another 2-bits repeat flag follows, and so on."
+        if (proba == 0) {
+            // Read the next two bits to see how many more 0s
+            int repeat = IO_read_bits(in, 2);
+
+            while (1) {
+                for (int i = 0; i < repeat && symb < FSE_MAX_SYMBS; i++) {
+                    frequencies[symb++] = 0;
+                }
+                if (repeat == 3) {
+                    repeat = IO_read_bits(in, 2);
+                } else {
+                    break;
+                }
+            }
+        }
+    }
+    IO_align_stream(in);
+
+    // "When last symbol reaches cumulated total of 1 << Accuracy_Log, decoding
+    // is complete. If the last symbol makes cumulated total go above 1 <<
+    // Accuracy_Log, distribution is considered corrupted."
+    if (remaining != 0 || symb >= FSE_MAX_SYMBS) {
+        CORRUPTION();
+    }
+
+    // Initialize the decoding table using the determined weights
+    FSE_init_dtable(dtable, frequencies, symb, accuracy_log);
+}
+
+static void FSE_init_dtable_rle(FSE_dtable *const dtable, const u8 symb) {
+    dtable->symbols = malloc(sizeof(u8));
+    dtable->num_bits = malloc(sizeof(u8));
+    dtable->new_state_base = malloc(sizeof(u16));
+
+    if (!dtable->symbols || !dtable->num_bits || !dtable->new_state_base) {
+        BAD_ALLOC();
+    }
+
+    // This setup will always have a state of 0, always return symbol `symb`,
+    // and never consume any bits
+    dtable->symbols[0] = symb;
+    dtable->num_bits[0] = 0;
+    dtable->new_state_base[0] = 0;
+    dtable->accuracy_log = 0;
+}
+
+static void FSE_free_dtable(FSE_dtable *const dtable) {
+    free(dtable->symbols);
+    free(dtable->num_bits);
+    free(dtable->new_state_base);
+    memset(dtable, 0, sizeof(FSE_dtable));
+}
+
+static void FSE_copy_dtable(FSE_dtable *const dst, const FSE_dtable *const src) {
+    if (src->accuracy_log == 0) {
+        memset(dst, 0, sizeof(FSE_dtable));
+        return;
+    }
+
+    size_t size = (size_t)1 << src->accuracy_log;
+    dst->accuracy_log = src->accuracy_log;
+
+    dst->symbols = malloc(size);
+    dst->num_bits = malloc(size);
+    dst->new_state_base = malloc(size * sizeof(u16));
+    if (!dst->symbols || !dst->num_bits || !dst->new_state_base) {
+        BAD_ALLOC();
+    }
+
+    memcpy(dst->symbols, src->symbols, size);
+    memcpy(dst->num_bits, src->num_bits, size);
+    memcpy(dst->new_state_base, src->new_state_base, size * sizeof(u16));
+}
+/******* END FSE PRIMITIVES ***************************************************/
+

doc/educational_decoder/zstd_decompress.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+size_t ZSTD_decompress(void *const dst, const size_t dst_len,
+                       const void *const src, const size_t src_len);
+size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
+                                 const void *const src, const size_t src_len,
+                                 const void *const dict, const size_t dict_len);
+size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
+

doc/zstd_compression_format.md

@@ -3,7 +3,7 @@ Zstandard Compression Format
 
 ### Notices
 
-Copyright (c) 2016 Yann Collet
+Copyright (c) 2016-present Yann Collet, Facebook, Inc.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
@@ -16,8 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.2.2 (14/09/16)
-
+0.2.4 (17/02/17)
 
 Introduction
 ------------
@@ -57,18 +56,15 @@ Whenever it does not support a parameter defined in the compressed stream,
 it must produce a non-ambiguous error code and associated error message
 explaining which parameter is unsupported.
 
-
-Overall conventions
------------
+### Overall conventions
 In this document:
 - square brackets i.e. `[` and `]` are used to indicate optional fields or parameters.
-- a naming convention for identifiers is `Mixed_Case_With_Underscores`
+- the naming convention for identifiers is `Mixed_Case_With_Underscores`
 
-Definitions
------------
-A content compressed by Zstandard is transformed into a Zstandard __frame__.
+### Definitions
+Content compressed by Zstandard is transformed into a Zstandard __frame__.
 Multiple frames can be appended into a single file or stream.
-A frame is totally independent, has a defined beginning and end,
+A frame is completely independent, has a defined beginning and end,
 and a set of parameters which tells the decoder how to decompress it.
 
 A frame encapsulates one or multiple __blocks__.
@@ -78,63 +74,33 @@ Unlike frames, each block depends on previous blocks for proper decoding.
 However, each block can be decompressed without waiting for its successor,
 allowing streaming operations.
 
+Overview
+---------
+- [Frames](#frames)
+  - [Zstandard frames](#zstandard-frames)
+    - [Blocks](#blocks)
+      - [Literals Section](#literals-section)
+      - [Sequences Section](#sequences-section)
+      - [Sequence Execution](#sequence-execution)
+  - [Skippable frames](#skippable-frames)
+- [Entropy Encoding](#entropy-encoding)
+  - [FSE](#fse)
+  - [Huffman Coding](#huffman-coding)
+- [Dictionary Format](#dictionary-format)
 
-Frame Concatenation
--------------------
+Frames
+------
+Zstandard compressed data is made of up one or more __frames__.
+Each frame is independent and can be decompressed indepedently of other frames.
+The decompressed content of multiple concatenated frames is the concatenation of
+each frames decompressed content.
 
-In some circumstances, it may be required to append multiple frames,
-for example in order to add new data to an existing compressed file
-without re-framing it.
+There are two frame formats defined by Zstandard:
+  Zstandard frames and Skippable frames.
+Zstandard frames contain compressed data, while
+skippable frames contain no data and can be used for metadata.
 
-In such case, each frame brings its own set of descriptor flags.
-Each frame is considered independent.
-The only relation between frames is their sequential order.
-
-The ability to decode multiple concatenated frames
-within a single stream or file is left outside of this specification.
-As an example, the reference `zstd` command line utility is able
-to decode all concatenated frames in their sequential order,
-delivering the final decompressed result as if it was a single content.
-
-
-Skippable Frames
-----------------
-
-| `Magic_Number` | `Frame_Size` | `User_Data` |
-|:--------------:|:------------:|:-----------:|
-|   4 bytes      |  4 bytes     |   n bytes   |
-
-Skippable frames allow the insertion of user-defined data
-into a flow of concatenated frames.
-Its design is pretty straightforward,
-with the sole objective to allow the decoder to quickly skip
-over user-defined data and continue decoding.
-
-Skippable frames defined in this specification are compatible with [LZ4] ones.
-
-[LZ4]:http://www.lz4.org
-
-__`Magic_Number`__
-
-4 Bytes, little-endian format.
-Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F.
-All 16 values are valid to identify a skippable frame.
-
-__`Frame_Size`__
-
-This is the size, in bytes, of the following `User_Data`
-(without including the magic number nor the size field itself).
-This field is represented using 4 Bytes, little-endian format, unsigned 32-bits.
-This means `User_Data` can’t be bigger than (2^32-1) bytes.
-
-__`User_Data`__
-
-The `User_Data` can be anything. Data will just be skipped by the decoder.
-
-
-
-General Structure of Zstandard Frame format
--------------------------------------------
+## Zstandard frames
 The structure of a single Zstandard frame is following:
 
 | `Magic_Number` | `Frame_Header` |`Data_Block`| [More data blocks] | [`Content_Checksum`] |
@@ -148,11 +114,11 @@ Value : 0xFD2FB528
 
 __`Frame_Header`__
 
-2 to 14 Bytes, detailed in [next part](#the-structure-of-frame_header).
+2 to 14 Bytes, detailed in [`Frame_Header`](#frame_header).
 
 __`Data_Block`__
 
-Detailed in [next chapter](#the-structure-of-data_block).
+Detailed in [`Blocks`](#blocks).
 That’s where compressed data is stored.
 
 __`Content_Checksum`__
@@ -163,10 +129,9 @@ of [xxh64() hash function](http://www.xxhash.org)
 digesting the original (decoded) data as input, and a seed of zero.
 The low 4 bytes of the checksum are stored in little endian format.
 
+### `Frame_Header`
 
-The structure of `Frame_Header`
--------------------------------
-The `Frame_Header` has a variable size, which uses a minimum of 2 bytes,
+The `Frame_Header` has a variable size, with a minimum of 2 bytes,
 and up to 14 bytes depending on optional parameters.
 The structure of `Frame_Header` is following:
 
@@ -174,10 +139,10 @@ The structure of `Frame_Header` is following:
 | ------------------------- | --------------------- | ----------------- | ---------------------- |
 | 1 byte                    | 0-1 byte              | 0-4 bytes         | 0-8 bytes              |
 
-### `Frame_Header_Descriptor`
+#### `Frame_Header_Descriptor`
 
 The first header's byte is called the `Frame_Header_Descriptor`.
-It tells which other fields are present.
+It describes which other fields are present.
 Decoding this byte is enough to tell the size of `Frame_Header`.
 
 | Bit number | Field name                |
@@ -189,7 +154,7 @@ Decoding this byte is enough to tell the size of `Frame_Header`.
 | 2          | `Content_Checksum_flag`   |
 | 1-0        | `Dictionary_ID_flag`      |
 
-In this table, bit 7 is highest bit, while bit 0 is lowest.
+In this table, bit 7 the is highest bit, while bit 0 the is lowest.
 
 __`Frame_Content_Size_flag`__
 
@@ -217,7 +182,7 @@ but `Window_Descriptor` byte is skipped.
 As a consequence, the decoder must allocate a memory segment
 of size equal or bigger than `Frame_Content_Size`.
 
-In order to preserve the decoder from unreasonable memory requirement,
+In order to preserve the decoder from unreasonable memory requirements,
 a decoder can reject a compressed frame
 which requests a memory size beyond decoder's authorized range.
 
@@ -257,7 +222,7 @@ It also specifies the size of this field as `Field_Size`.
 | ---------- | --- | --- | --- | --- |
 |`Field_Size`|  0  |  1  |  2  |  4  |
 
-### `Window_Descriptor`
+#### `Window_Descriptor`
 
 Provides guarantees on maximum back-reference distance
 that will be used within compressed data.
@@ -267,7 +232,7 @@ The `Window_Descriptor` byte is optional. It is absent when `Single_Segment_flag
 In this case, the maximum back-reference distance is the content size itself,
 which can be any value from 1 to 2^64-1 bytes (16 EB).
 
-| Bit numbers |     7-3    |     0-2    |
+| Bit numbers |     7-3    |     2-0    |
 | ----------- | ---------- | ---------- |
 | Field name  | `Exponent` | `Mantissa` |
 
@@ -295,12 +260,12 @@ It's merely a recommendation though,
 decoders are free to support larger or lower limits,
 depending on local limitations.
 
-### `Dictionary_ID`
+#### `Dictionary_ID`
 
 This is a variable size field, which contains
 the ID of the dictionary required to properly decode the frame.
 Note that this field is optional. When it's not present,
-it's up to the caller to make sure it uses the correct dictionary.
+it's up to the decoder to make sure it uses the correct dictionary.
 Format is little-endian.
 
 Field size depends on `Dictionary_ID_flag`.
@@ -320,7 +285,7 @@ the following ranges are reserved for future use and should not be used :
 - high range : >= (2^31)
 
 
-### `Frame_Content_Size`
+#### `Frame_Content_Size`
 
 This is the original (uncompressed) size. This information is optional.
 The `Field_Size` is provided according to value of `Frame_Content_Size_flag`.
@@ -338,10 +303,14 @@ When `Field_Size` is 1, 4 or 8 bytes, the value is read directly.
 When `Field_Size` is 2, _the offset of 256 is added_.
 It's allowed to represent a small size (for example `18`) using any compatible variant.
 
+Blocks
+-------
+After the magic number and header of each block,
+there are some number of blocks.
+Each frame must have at least one block  but there is no upper limit
+on the number of blocks per frame.
 
-The structure of `Data_Block`
------------------------------
-The structure of `Data_Block` is following:
+The structure of a block is as follows:
 
 | `Last_Block` | `Block_Type` | `Block_Size` | `Block_Content` |
 |:------------:|:------------:|:------------:|:---------------:|
@@ -352,8 +321,9 @@ The block header (`Last_Block`, `Block_Type`, and `Block_Size`) uses 3-bytes.
 __`Last_Block`__
 
 The lowest bit signals if this block is the last one.
-Frame ends right after this block.
-It may be followed by an optional `Content_Checksum` .
+The frame will end after this one.
+It may be followed by an optional `Content_Checksum`
+(see [Zstandard Frames](#zstandard-frames)).
 
 __`Block_Type` and `Block_Size`__
 
@@ -368,59 +338,57 @@ There are 4 block types :
 | `Block_Type` | `Raw_Block` | `RLE_Block` | `Compressed_Block` | `Reserved`|
 
 - `Raw_Block` - this is an uncompressed block.
-  `Block_Size` is the number of bytes to read and copy.
+  `Block_Content` contains `Block_Size` bytes to read and copy
+  as decoded data.
+
 - `RLE_Block` - this is a single byte, repeated N times.
-  In which case, `Block_Size` is the size to regenerate,
-  while the "compressed" block is just 1 byte (the byte to repeat).
-- `Compressed_Block` - this is a [Zstandard compressed block](#the-format-of-compressed_block),
-  detailed in another section of this specification.
-  `Block_Size` is the compressed size.
-  Decompressed size is unknown,
+  `Block_Content` consists of a single byte,
+  and `Block_Size` is the number of times this byte should be repeated.
+
+- `Compressed_Block` - this is a [Zstandard compressed block](#compressed-blocks),
+  explained later on.
+  `Block_Size` is the length of `Block_Content`, the compressed data.
+  The decompressed size is unknown,
   but its maximum possible value is guaranteed (see below)
+
 - `Reserved` - this is not a block.
   This value cannot be used with current version of this specification.
 
 Block sizes must respect a few rules :
-- In compressed mode, compressed size if always strictly `< decompressed size`.
-- Block decompressed size is always <= maximum back-reference distance .
-- Block decompressed size is always <= 128 KB
+- In compressed mode, compressed size is always strictly less than decompressed size.
+- Block decompressed size is always <= maximum back-reference distance.
+- Block decompressed size is always <= 128 KB.
 
-
-__`Block_Content`__
-
-The `Block_Content` is where the actual data to decode stands.
-It might be compressed or not, depending on previous field indications.
 A data block is not necessarily "full" :
 since an arbitrary “flush” may happen anytime,
-block decompressed content can be any size,
+block decompressed content can be any size (even empty),
 up to `Block_Maximum_Decompressed_Size`, which is the smallest of :
 - Maximum back-reference distance
 - 128 KB
 
-
-
-The format of `Compressed_Block`
---------------------------------
-The size of `Compressed_Block` must be provided using `Block_Size` field from `Data_Block`.
-The `Compressed_Block` has a guaranteed maximum regenerated size,
-in order to properly allocate destination buffer.
-See [`Data_Block`](#the-structure-of-data_block) for more details.
+Compressed Blocks
+-----------------
+To decompress a compressed block, the compressed size must be provided from
+`Block_Size` field in the block header.
 
 A compressed block consists of 2 sections :
-- [`Literals_Section`](#literals_section)
-- [`Sequences_Section`](#sequences_section)
+- [Literals Section](#literals-section)
+- [Sequences Section](#sequences-section)
 
-### Prerequisites
+The results of the two sections are then combined to produce the decompressed
+data in [Sequence Execution](#sequence-execution)
+
+#### Prerequisites
 To decode a compressed block, the following elements are necessary :
-- Previous decoded blocks, up to a distance of `Window_Size`,
-  or all previous blocks when `Single_Segment_flag` is set.
-- List of "recent offsets" from previous compressed block.
-- Decoding tables of previous compressed block for each symbol type
+- Previous decoded data, up to a distance of `Window_Size`,
+  or all previous data when `Single_Segment_flag` is set.
+- List of "recent offsets" from the previous compressed block.
+- Decoding tables of the previous compressed block for each symbol type
   (literals, literals lengths, match lengths, offsets).
 
-
-### `Literals_Section`
-
+Literals Section
+----------------
+During sequence execution, symbols from the literals section
 During sequence phase, literals will be entangled with match copy operations.
 All literals are regrouped in the first part of the block.
 They can be decoded first, and then copied during sequence operations,
@@ -444,7 +412,7 @@ using little-endian convention.
 | --------------------- | ------------- | ------------------ | ----------------- |
 |   2 bits              |  1 - 2 bits   |    5 - 20 bits     |    0 - 18 bits    |
 
-In this representation, bits on the left are smallest bits.
+In this representation, bits on the left are the lowest bits.
 
 __`Literals_Block_Type`__
 
@@ -465,256 +433,118 @@ This field uses 2 lowest bits of first byte, describing 4 different block types
 - `Repeat_Stats_Literals_Block` - This is a Huffman-compressed block,
         using Huffman tree _from previous Huffman-compressed literals block_.
         Huffman tree description will be skipped.
+        Note: If this mode is used without any previous Huffman-table in the frame
+        (or [dictionary](#dictionary-format)), this should be treated as corruption.
 
 __`Size_Format`__
 
 `Size_Format` is divided into 2 families :
 
-- For `Compressed_Block`, it requires to decode both `Compressed_Size`
-  and `Regenerated_Size` (the decompressed size). It will also decode the number of streams.
 - For `Raw_Literals_Block` and `RLE_Literals_Block` it's enough to decode `Regenerated_Size`.
+- For `Compressed_Block`, its required to decode both `Compressed_Size`
+  and `Regenerated_Size` (the decompressed size). It will also decode the number of streams.
 
 For values spanning several bytes, convention is little-endian.
 
 __`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ :
 
-- Value x0 : `Regenerated_Size` uses 5 bits (0-31).
+- Value ?0 : `Size_Format` uses 1 bit.
+               `Regenerated_Size` uses 5 bits (0-31).
                `Literals_Section_Header` has 1 byte.
                `Regenerated_Size = Header[0]>>3`
-- Value 01 : `Regenerated_Size` uses 12 bits (0-4095).
+- Value 01 : `Size_Format` uses 2 bits.
+               `Regenerated_Size` uses 12 bits (0-4095).
                `Literals_Section_Header` has 2 bytes.
                `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)`
-- Value 11 : `Regenerated_Size` uses 20 bits (0-1048575).
+- Value 11 : `Size_Format` uses 2 bits.
+               `Regenerated_Size` uses 20 bits (0-1048575).
                `Literals_Section_Header` has 3 bytes.
                `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)`
 
+Only Stream1 is present for these cases.
 Note : it's allowed to represent a short value (for example `13`)
 using a long format, accepting the increased compressed data size.
 
 __`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ :
 
 - Value 00 : _A single stream_.
-               Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023).
+               Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023).
                `Literals_Section_Header` has 3 bytes.
 - Value 01 : 4 streams.
-               Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023).
+               Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023).
                `Literals_Section_Header` has 3 bytes.
 - Value 10 : 4 streams.
-               Both `Compressed_Size` and `Regenerated_Size` use 14 bits (0-16383).
+               Both `Regenerated_Size` and `Compressed_Size` use 14 bits (0-16383).
                `Literals_Section_Header` has 4 bytes.
 - Value 11 : 4 streams.
-               Both `Compressed_Size` and `Regenerated_Size` use 18 bits (0-262143).
+               Both `Regenerated_Size` and `Compressed_Size` use 18 bits (0-262143).
                `Literals_Section_Header` has 5 bytes.
 
 Both `Compressed_Size` and `Regenerated_Size` fields follow little-endian convention.
+Note: `Compressed_Size` __includes__ the size of the Huffman Tree description if it
+is present.
 
+### Raw Literals Block
+The data in Stream1 is `Regenerated_Size` bytes long, and contains the raw literals data
+to be used in sequence execution.
+
+### RLE Literals Block
+Stream1 consists of a single byte which should be repeated `Regenerated_Size` times
+to generate the decoded literals.
+
+### Compressed Literals Block and Repeat Stats Literals Block
+Both of these modes contain Huffman encoded data 
 
 #### `Huffman_Tree_Description`
-
 This section is only present when `Literals_Block_Type` type is `Compressed_Literals_Block` (`2`).
+The format of the Huffman tree description can be found at [Huffman Tree description](#huffman-tree-description).
+The size Huffman Tree description will be determined during the decoding process,
+and must be used to determine where the compressed Huffman streams begin.
 
-Prefix coding represents symbols from an a priori known alphabet
-by bit sequences (codewords), one codeword for each symbol,
-in a manner such that different symbols may be represented
-by bit sequences of different lengths,
-but a parser can always parse an encoded string
-unambiguously symbol-by-symbol.
+If repeat stats mode is used, the Huffman table used in the previous compressed block will
+be used to decompress this block as well.
 
-Given an alphabet with known symbol frequencies,
-the Huffman algorithm allows the construction of an optimal prefix code
-using the fewest bits of any possible prefix codes for that alphabet.
+Huffman compressed data consists either 1 or 4 Huffman-coded streams.
 
-Prefix code must not exceed a maximum code length.
-More bits improve accuracy but cost more header size,
-and require more memory or more complex decoding operations.
-This specification limits maximum code length to 11 bits.
+If only one stream is present, it is a single bitstream occupying the entire
+remaining portion of the literals block, encoded as described at
+[Huffman-Coded Streams](#huffman-coded-streams).
 
+If there are four streams, the literals section header only provides enough
+information to know the regenerated and compressed sizes of all four streams combined.
+The regenerated size of each stream is equal to `(totalSize+3)/4`, except for the last stream,
+which may be up to 3 bytes smaller, to reach a total decompressed size match that described
+in the literals header.
 
-##### Representation
-
-All literal values from zero (included) to last present one (excluded)
-are represented by `Weight` with values from `0` to `Max_Number_of_Bits`.
-Transformation from `Weight` to `Number_of_Bits` follows this formula :
-```
-Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
-```
-The last symbol's `Weight` is deduced from previously decoded ones,
-by completing to the nearest power of 2.
-This power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
-
-__Example__ :
-Let's presume the following Huffman tree must be described :
-
-|     literal      |  0  |  1  |  2  |  3  |  4  |  5  |
-| ---------------- | --- | --- | --- | --- | --- | --- |
-| `Number_of_Bits` |  1  |  2  |  3  |  0  |  4  |  4  |
-
-The tree depth is 4, since its smallest element uses 4 bits.
-Value `5` will not be listed, nor will values above `5`.
-Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
-Weight formula is :
-```
-Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0
-```
-It gives the following serie of weights :
-
-| `Weight` |  4  |  3  |  2  |  0  |  1  |
-| -------- | --- | --- | --- | --- | --- |
-| literal  |  0  |  1  |  2  |  3  |  4  |
-
-The decoder will do the inverse operation :
-having collected weights of literals from `0` to `4`,
-it knows the last literal, `5`, is present with a non-zero weight.
-The weight of `5` can be deducted by joining to the nearest power of 2.
-Sum of `2^(Weight-1)` (excluding 0) is :
-`8 + 4 + 2 + 0 + 1 = 15`.
-Nearest power of 2 is 16.
-Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = 1`.
-
-##### Huffman Tree header
-
-This is a single byte value (0-255),
-which tells how to decode the list of weights.
-
-- if `headerByte` >= 128 : this is a direct representation,
-  where each `Weight` is written directly as a 4 bits field (0-15).
-  The full representation occupies `((Number_of_Symbols+1)/2)` bytes,
-  meaning it uses a last full byte even if `Number_of_Symbols` is odd.
-  `Number_of_Symbols = headerByte - 127`.
-  Note that maximum `Number_of_Symbols` is 255-127 = 128.
-  A larger serie must necessarily use FSE compression.
-
-- if `headerByte` < 128 :
-  the serie of weights is compressed by FSE.
-  The length of the FSE-compressed serie is equal to `headerByte` (0-127).
-
-##### Finite State Entropy (FSE) compression of Huffman weights
-
-The serie of weights is compressed using FSE compression.
-It's a single bitstream with 2 interleaved states,
-sharing a single distribution table.
-
-To decode an FSE bitstream, it is necessary to know its compressed size.
-Compressed size is provided by `headerByte`.
-It's also necessary to know its _maximum possible_ decompressed size,
-which is `255`, since literal values span from `0` to `255`,
-and last symbol value is not represented.
-
-An FSE bitstream starts by a header, describing probabilities distribution.
-It will create a Decoding Table.
-Table must be pre-allocated, which requires to support a maximum accuracy.
-For a list of Huffman weights, maximum accuracy is 7 bits.
-
-FSE header is [described in relevant chapter](#fse-distribution-table--condensed-format),
-and so is [FSE bitstream](#bitstream).
-The main difference is that Huffman header compression uses 2 states,
-which share the same FSE distribution table.
-Bitstream contains only FSE symbols (no interleaved "raw bitfields").
-The number of symbols to decode is discovered
-by tracking bitStream overflow condition.
-When both states have overflowed the bitstream, end is reached.
-
-
-##### Conversion from weights to Huffman prefix codes
-
-All present symbols shall now have a `Weight` value.
-It is possible to transform weights into Number_of_Bits, using this formula:
-```
-Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0
-```
-Symbols are sorted by `Weight`. Within same `Weight`, symbols keep natural order.
-Symbols with a `Weight` of zero are removed.
-Then, starting from lowest weight, prefix codes are distributed in order.
-
-__Example__ :
-Let's presume the following list of weights has been decoded :
-
-| Literal  |  0  |  1  |  2  |  3  |  4  |  5  |
-| -------- | --- | --- | --- | --- | --- | --- |
-| `Weight` |  4  |  3  |  2  |  0  |  1  |  1  |
-
-Sorted by weight and then natural order,
-it gives the following distribution :
-
-| Literal          |  3  |  4  |  5  |  2  |  1  |   0  |
-| ---------------- | --- | --- | --- | --- | --- | ---- |
-| `Weight`         |  0  |  1  |  1  |  2  |  3  |   4  |
-| `Number_of_Bits` |  0  |  4  |  4  |  3  |  2  |   1  |
-| prefix codes     | N/A | 0000| 0001| 001 | 01  |   1  |
-
-
-#### The content of Huffman-compressed literal stream
-
-##### Bitstreams sizes
-
-As seen in a previous paragraph,
-there are 2 types of Huffman-compressed literals :
-a single stream and 4 streams.
-
-Encoding using 4 streams is useful for CPU with multiple execution units and out-of-order operations.
-Since each stream can be decoded independently,
-it's possible to decode them up to 4x faster than a single stream,
-presuming the CPU has enough parallelism available.
-
-For single stream, header provides both the compressed and regenerated size.
-For 4 streams though,
-header only provides compressed and regenerated size of all 4 streams combined.
-In order to properly decode the 4 streams,
-it's necessary to know the compressed and regenerated size of each stream.
-
-Regenerated size of each stream can be calculated by `(totalSize+3)/4`,
-except for last one, which can be up to 3 bytes smaller, to reach `totalSize`.
-
-Compressed size is provided explicitly : in the 4-streams variant,
-bitstreams are preceded by 3 unsigned little-endian 16-bits values.
-Each value represents the compressed size of one stream, in order.
-The last stream size is deducted from total compressed size
-and from previously decoded stream sizes :
+The compressed size of each stream is provided explicitly: the first 6 bytes of the compressed
+data consist of three 2-byte little endian fields, describing the compressed sizes
+of the first three streams.
+The last streams size is computed from the total compressed size and the size of the other
+three streams.
 
 `stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize`.
 
+Note: remember that totalCSize may be smaller than the `Compressed_Size` found in the literals
+block header as `Compressed_Size` also contains the size of the Huffman Tree description if it
+is present.
 
-##### Bitstreams read and decode
-
-Each bitstream must be read _backward_,
-that is starting from the end down to the beginning.
-Therefore it's necessary to know the size of each bitstream.
-
-It's also necessary to know exactly which _bit_ is the latest.
-This is detected by a final bit flag :
-the highest bit of latest byte is a final-bit-flag.
-Consequently, a last byte of `0` is not possible.
-And the final-bit-flag itself is not part of the useful bitstream.
-Hence, the last byte contains between 0 and 7 useful bits.
-
-Starting from the end,
-it's possible to read the bitstream in a little-endian fashion,
-keeping track of already used bits.
-
-Reading the last `Max_Number_of_Bits` bits,
-it's then possible to compare extracted value to decoding table,
-determining the symbol to decode and number of bits to discard.
-
-The process continues up to reading the required number of symbols per stream.
-If a bitstream is not entirely and exactly consumed,
-hence reaching exactly its beginning position with _all_ bits consumed,
-the decoding process is considered faulty.
-
-
-### `Sequences_Section`
+Each of these 4 bitstreams is then decoded independently as a Huffman-Coded stream,
+as described at [Huffman-Coded Streams](#huffman-coded-streams)
 
+Sequences Section
+-----------------
 A compressed block is a succession of _sequences_ .
 A sequence is a literal copy command, followed by a match copy command.
 A literal copy command specifies a length.
 It is the number of bytes to be copied (or extracted) from the literal section.
 A match copy command specifies an offset and a length.
-The offset gives the position to copy from,
-which can be within a previous block.
 
 When all _sequences_ are decoded,
-if there is any literal left in the _literal section_,
+if there is are any literals left in the _literal section_,
 these bytes are added at the end of the block.
 
+This is described in more detail in [Sequence Execution](#sequence-execution)
+
 The `Sequences_Section` regroup all symbols required to decode commands.
 There are 3 symbol types : literals lengths, offsets and match lengths.
 They are encoded together, interleaved, in a single _bitstream_.
@@ -727,7 +557,7 @@ followed by the bitstream.
 | -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- |
 
 To decode the `Sequences_Section`, it's required to know its size.
-This size is deducted from `blockSize - literalSectionSize`.
+This size is deduced from `blockSize - literalSectionSize`.
 
 
 #### `Sequences_Section_Header`
@@ -766,12 +596,21 @@ They follow the same enumeration :
 | ------------------ | ----------------- | ---------- | --------------------- | ------------- |
 | `Compression_Mode` | `Predefined_Mode` | `RLE_Mode` | `FSE_Compressed_Mode` | `Repeat_Mode` |
 
-- `Predefined_Mode` : uses a predefined distribution table.
-- `RLE_Mode` : it's a single code, repeated `Number_of_Sequences` times.
-- `Repeat_Mode` : re-use distribution table from previous compressed block.
+- `Predefined_Mode` : A predefined FSE distribution table is used, defined in
+          [default distributions](#default-distributions).
+          The table takes no space in the compressed data.
+- `RLE_Mode` : The table description consists of a single byte.
+          This code will be repeated for every sequence.
+- `Repeat_Mode` : The table used in the previous compressed block will be used again.
+          No distribution table will be present.
+          Note: this includes RLE mode, so if repeat_mode follows rle_mode the same symbol will be repeated.
+          If this mode is used without any previous sequence table in the frame
+          (or [dictionary](#dictionary-format)) to repeat, this should be treated as corruption.
 - `FSE_Compressed_Mode` : standard FSE compression.
           A distribution table will be present.
-          It will be described in [next part](#distribution-tables).
+          The format of this distribution table is described in (FSE Table Description)[#fse-table-description].
+          Note that the maximum allowed accuracy log for literals length and match length tables is 9,
+          and the maximum accuracy log for the offsets table is 8.
 
 #### The codes for literals lengths, match lengths, and offsets.
 
@@ -784,6 +623,9 @@ and interleaved with raw additional bits in the same bitstream.
 
 Literals length codes are values ranging from `0` to `35` included.
 They define lengths from 0 to 131071 bytes.
+The literals length is equal to the decoded `Baseline` plus
+the result of reading `Number_of_Bits` bits from the bitstream,
+as a little-endian value.
 
 | `Literals_Length_Code` |         0-15           |
 | ---------------------- | ---------------------- |
@@ -805,23 +647,14 @@ They define lengths from 0 to 131071 bytes.
 | `Baseline`             | 8192 |16384 |32768 |65536 |
 | `Number_of_Bits`       |  13  |  14  |  15  |  16  |
 
-##### Default distribution for literals length codes
-
-When `Compression_Mode` is `Predefined_Mode`,
-a predefined distribution is used for FSE compression.
-
-Below is its definition. It uses an accuracy of 6 bits (64 states).
-```
-short literalsLength_defaultDistribution[36] =
-        { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
-          2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
-         -1,-1,-1,-1 };
-```
 
 ##### Match length codes
 
 Match length codes are values ranging from `0` to `52` included.
 They define lengths from 3 to 131074 bytes.
+The match length is equal to the decoded `Baseline` plus
+the result of reading `Number_of_Bits` bits from the bitstream,
+as a little-endian value.
 
 | `Match_Length_Code` |         0-31            |
 | ------------------- | ----------------------- |
@@ -835,28 +668,14 @@ They define lengths from 3 to 131074 bytes.
 
 | `Match_Length_Code` |  40  |  41  |  42  |  43  |  44  |  45  |  46  |  47  |
 | ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
-| `Baseline`          |  67  |  83  |  99  |  131 |  258 |  514 | 1026 | 2050 |
+| `Baseline`          |  67  |  83  |  99  |  131 |  259 |  515 | 1027 | 2051 |
 | `Number_of_Bits`    |   4  |   4  |   5  |   7  |   8  |   9  |  10  |  11  |
 
 | `Match_Length_Code` |  48  |  49  |  50  |  51  |  52  |
 | ------------------- | ---- | ---- | ---- | ---- | ---- |
-| `Baseline`          | 4098 | 8194 |16486 |32770 |65538 |
+| `Baseline`          | 4099 | 8195 |16387 |32771 |65539 |
 | `Number_of_Bits`    |  12  |  13  |  14  |  15  |  16  |
 
-##### Default distribution for match length codes
-
-When `Compression_Mode` is defined as `Predefined_Mode`,
-a predefined distribution is used for FSE compression.
-
-Below is its definition. It uses an accuracy of 6 bits (64 states).
-```
-short matchLengths_defaultDistribution[53] =
-        { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
-          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
-         -1,-1,-1,-1,-1 };
-```
-
 ##### Offset codes
 
 Offset codes are values ranging from `0` to `N`.
@@ -866,7 +685,7 @@ Recommendation is to support at least up to `22`.
 For information, at the time of this writing.
 the reference decoder supports a maximum `N` value of `28` in 64-bits mode.
 
-An offset code is also the number of additional bits to read,
+An offset code is also the number of additional bits to read in little-endian fashion,
 and can be translated into an `Offset_Value` using the following formulas :
 
 ```
@@ -876,44 +695,235 @@ if (Offset_Value > 3) offset = Offset_Value - 3;
 It means that maximum `Offset_Value` is `(2^(N+1))-1` and it supports back-reference distance up to `(2^(N+1))-4`
 but is limited by [maximum back-reference distance](#window_descriptor).
 
-`Offset_Value` from 1 to 3 are special : they define "repeat codes",
-which means one of the previous offsets will be repeated.
-They are sorted in recency order, with 1 meaning the most recent one.
-See [Repeat offsets](#repeat-offsets) paragraph.
+`Offset_Value` from 1 to 3 are special : they define "repeat codes".
+This is described in more detail in [Repeat Offsets](#repeat-offsets).
 
+#### Decoding Sequences
+FSE bitstreams are read in reverse direction than written. In zstd,
+the compressor writes bits forward into a block and the decompressor
+must read the bitstream _backwards_.
 
-##### Default distribution for offset codes
+To find the start of the bitstream it is therefore necessary to
+know the offset of the last byte of the block which can be found
+by counting `Block_Size` bytes after the block header.
 
-When `Compression_Mode` is defined as `Predefined_Mode`,
-a predefined distribution is used for FSE compression.
+After writing the last bit containing information, the compressor
+writes a single `1`-bit and then fills the byte with 0-7 `0` bits of
+padding. The last byte of the compressed bitstream cannot be `0` for
+that reason.
 
-Below is its definition. It uses an accuracy of 5 bits (32 states),
-and supports a maximum `N` of 28, allowing offset values up to 536,870,908 .
+When decompressing, the last byte containing the padding is the first
+byte to read. The decompressor needs to skip 0-7 initial `0`-bits and
+the first `1`-bit it occurs. Afterwards, the useful part of the bitstream
+begins.
 
-If any sequence in the compressed block requires an offset larger than this,
+FSE decoding requires a 'state' to be carried from symbol to symbol.
+For more explanation on FSE decoding, see the [FSE section](#fse).
+
+For sequence decoding, a separate state must be kept track of for each of
+literal lengths, offsets, and match lengths.
+Some FSE primitives are also used.
+For more details on the operation of these primitives, see the [FSE section](#fse).
+
+##### Starting states
+The bitstream starts with initial FSE state values,
+each using the required number of bits in their respective _accuracy_,
+decoded previously from their normalized distribution.
+
+It starts by `Literals_Length_State`,
+followed by `Offset_State`,
+and finally `Match_Length_State`.
+
+Reminder : always keep in mind that all values are read _backward_,
+so the 'start' of the bitstream is at the highest position in memory,
+immediately before the last `1`-bit for padding.
+
+After decoding the starting states, a single sequence is decoded
+`Number_Of_Sequences` times.
+These sequences are decoded in order from first to last.
+Since the compressor writes the bitstream in the forward direction,
+this means the compressor must encode the sequences starting with the last
+one and ending with the first.
+
+##### Decoding a sequence
+For each of the symbol types, the FSE state can be used to determine the appropriate code.
+The code then defines the baseline and number of bits to read for each type.
+See the [description of the codes] for how to determine these values.
+
+[description of the codes]: #the-codes-for-literals-lengths-match-lengths-and-offsets
+
+Decoding starts by reading the `Number_of_Bits` required to decode `Offset`.
+It then does the same for `Match_Length`,
+and then for `Literals_Length`.
+This sequence is then used for [sequence execution](#sequence-execution).
+
+If it is not the last sequence in the block,
+the next operation is to update states.
+Using the rules pre-calculated in the decoding tables,
+`Literals_Length_State` is updated,
+followed by `Match_Length_State`,
+and then `Offset_State`.
+See the [FSE section](#fse) for details on how to update states from the bitstream.
+
+This operation will be repeated `Number_of_Sequences` times.
+At the end, the bitstream shall be entirely consumed,
+otherwise the bitstream is considered corrupted.
+
+#### Default Distributions
+If `Predefined_Mode` is selected for a symbol type,
+its FSE decoding table is generated from a predefined distribution table defined here.
+For details on how to convert this distribution into a decoding table, see the [FSE section].
+
+[FSE section]: #from-normalized-distribution-to-decoding-tables
+
+##### Literals Length
+The decoding table uses an accuracy log of 6 bits (64 states).
+```
+short literalsLength_defaultDistribution[36] =
+        { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
+          2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
+         -1,-1,-1,-1 };
+```
+
+##### Match Length
+The decoding table uses an accuracy log of 6 bits (64 states).
+```
+short matchLengths_defaultDistribution[53] =
+        { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
+         -1,-1,-1,-1,-1 };
+```
+
+##### Offset Codes
+The decoding table uses an accuracy log of 5 bits (32 states),
+and supports a maximum `N` value of 28, allowing offset values up to 536,870,908 .
+
+If any sequence in the compressed block requires a larger offset than this,
 it's not possible to use the default distribution to represent it.
-
 ```
 short offsetCodes_defaultDistribution[29] =
         { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
           1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
 ```
 
-#### Distribution tables
+Sequence Execution
+------------------
+Once literals and sequences have been decoded,
+they are combined to produce the decoded content of a block.
 
-Following the header, up to 3 distribution tables can be described.
-When present, they are in this order :
-- Literals lengths
-- Offsets
-- Match Lengths
+Each sequence consists of a tuple of (`literals_length`, `offset_value`, `match_length`),
+decoded as described in the [Sequences Section](#sequences-section).
+To execute a sequence, first copy `literals_length` bytes from the literals section
+to the output.
 
-The content to decode depends on their respective encoding mode :
-- `Predefined_Mode` : no content. Use predefined distribution table.
-- `RLE_Mode` : 1 byte. This is the only code to use across the whole compressed block.
-- `FSE_Compressed_Mode` : A distribution table is present.
-- `Repeat_Mode` : no content. Re-use distribution from previous compressed block.
+Then `match_length` bytes are copied from previous decoded data.
+The offset to copy from is determined by `offset_value`:
+if `offset_value > 3`, then the offset is `offset_value - 3`.
+If `offset_value` is from 1-3, the offset is a special repeat offset value.
+See the [repeat offset](#repeat-offsets) section for how the offset is determined
+in this case.
 
-##### FSE distribution table : condensed format
+The offset is defined as from the current position, so an offset of 6
+and a match length of 3 means that 3 bytes should be copied from 6 bytes back.
+Note that all offsets must be at most equal to the window size defined by the frame header.
+
+#### Repeat offsets
+As seen in [Sequence Execution](#sequence-execution),
+the first 3 values define a repeated offset and we will call them
+`Repeated_Offset1`, `Repeated_Offset2`, and `Repeated_Offset3`.
+They are sorted in recency order, with `Repeated_Offset1` meaning "most recent one".
+
+If `offset_value == 1`, then the offset used is `Repeated_Offset1`, etc.
+
+There is an exception though, when current sequence's `literals_length = 0`.
+In this case, repeated offsets are shifted by one,
+so an `offset_value` of 1 means `Repeated_Offset2`,
+an `offset_value` of 2 means `Repeated_Offset3`,
+and an `offset_value` of 3 means `Repeated_Offset1 - 1_byte`.
+
+In the first block, the offset history is populated with the following values : 1, 4 and 8 (in order).
+
+Then each block gets its starting offset history from the ending values of the most recent compressed block.
+Note that non-compressed blocks are skipped,
+they do not contribute to offset history.
+
+[Offset Codes]: #offset-codes
+
+###### Offset updates rules
+
+The newest offset takes the lead in offset history,
+shifting others back (up to its previous place if it was already present).
+
+This means that when `Repeated_Offset1` (most recent) is used, history is unmodified.
+When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
+If any other offset is used, it becomes `Repeated_Offset1` and the rest are shift back by one.
+
+Skippable Frames
+----------------
+
+| `Magic_Number` | `Frame_Size` | `User_Data` |
+|:--------------:|:------------:|:-----------:|
+|   4 bytes      |  4 bytes     |   n bytes   |
+
+Skippable frames allow the insertion of user-defined data
+into a flow of concatenated frames.
+Its design is pretty straightforward,
+with the sole objective to allow the decoder to quickly skip
+over user-defined data and continue decoding.
+
+Skippable frames defined in this specification are compatible with [LZ4] ones.
+
+[LZ4]:http://www.lz4.org
+
+__`Magic_Number`__
+
+4 Bytes, little-endian format.
+Value : 0x184D2A5?, which means any value from 0x184D2A50 to 0x184D2A5F.
+All 16 values are valid to identify a skippable frame.
+
+__`Frame_Size`__
+
+This is the size, in bytes, of the following `User_Data`
+(without including the magic number nor the size field itself).
+This field is represented using 4 Bytes, little-endian format, unsigned 32-bits.
+This means `User_Data` can’t be bigger than (2^32-1) bytes.
+
+__`User_Data`__
+
+The `User_Data` can be anything. Data will just be skipped by the decoder.
+
+Entropy Encoding
+----------------
+Two types of entropy encoding are used by the Zstandard format:
+FSE, and Huffman coding.
+
+FSE
+---
+FSE, or FiniteStateEntropy is an entropy coding based on [ANS].
+FSE encoding/decoding involves a state that is carried over between symbols,
+so decoding must be done in the opposite direction as encoding.
+Therefore, all FSE bitstreams are read from end to beginning.
+
+For additional details on FSE, see [Finite State Entropy].
+
+[Finite State Entropy]:https://github.com/Cyan4973/FiniteStateEntropy/
+
+FSE decoding involves a decoding table which has a power of 2 size and three elements:
+`Symbol`, `Num_Bits`, and `Baseline`.
+The `log2` of the table size is its `Accuracy_Log`.
+The FSE state represents an index in this table.
+The next symbol in the stream is the symbol indicated by the table value for that state.
+To obtain the next state value,
+the decoder should consume `Num_Bits` bits from the stream as a little endian value and add it to baseline.
+
+To obtain the initial state value, consume `Accuracy_Log` bits from the stream as a little endian value.
+
+[ANS]: https://en.wikipedia.org/wiki/Asymmetric_Numeral_Systems
+
+### FSE Table Description
+To decode FSE streams, it is necessary to construct the decoding table.
+The Zstandard format encodes FSE table descriptions as follows:
 
 An FSE distribution table describes the probabilities of all symbols
 from `0` to the last present one (included)
@@ -925,8 +935,6 @@ since it will be discovered and reported by the decoding process.
 
 The bitstream starts by reporting on which scale it operates.
 `Accuracy_Log = low4bits + 5`.
-Note that maximum `Accuracy_Log` for literal and match lengths is `9`,
-and for offsets is `8`. Higher values are considered errors.
 
 Then follows each symbol value, from `0` to last present one.
 The number of bits used by each field is variable.
@@ -936,12 +944,12 @@ It depends on :
   __example__ :
   Presuming an `Accuracy_Log` of 8,
   and presuming 100 probabilities points have already been distributed,
-  the decoder may read any value from `0` to `255 - 100 + 1 == 156` (included).
+  the decoder may read any value from `0` to `255 - 100 + 1 == 156` (inclusive).
   Therefore, it must read `log2sup(156) == 8` bits.
 
 - Value decoded : small values use 1 less bit :
   __example__ :
-  Presuming values from 0 to 156 (included) are possible,
+  Presuming values from 0 to 156 (inclusive) are possible,
   255-156 = 99 values are remaining in an 8-bits field.
   They are used this way :
   first 99 values (hence from 0 to 98) use only 7 bits,
@@ -962,12 +970,12 @@ Probability is obtained from Value decoded by following formula :
 
 It means value `0` becomes negative probability `-1`.
 `-1` is a special probability, which means "less than 1".
-Its effect on distribution table is described in [next paragraph].
-For the purpose of calculating cumulated distribution, it counts as one.
+Its effect on distribution table is described in the [next section].
+For the purpose of calculating total allocated probability points, it counts as one.
 
-[next paragraph]:#fse-decoding--from-normalized-distribution-to-decoding-tables
+[next section]:#from-normalized-distribution-to-decoding-tables
 
-When a symbol has a probability of `zero`,
+When a symbol has a __probability__ of `zero`,
 it is followed by a 2-bits repeat flag.
 This repeat flag tells how many probabilities of zeroes follow the current one.
 It provides a number ranging from 0 to 3.
@@ -983,14 +991,14 @@ and how many symbols are present.
 The bitstream consumes a round number of bytes.
 Any remaining bit within the last byte is just unused.
 
-##### FSE decoding : from normalized distribution to decoding tables
+##### From normalized distribution to decoding tables
 
 The distribution of normalized probabilities is enough
 to create a unique decoding table.
 
 It follows the following build rule :
 
-The table has a size of `tableSize = 1 << Accuracy_Log`.
+The table has a size of `Table_Size = 1 << Accuracy_Log`.
 Each cell describes the symbol decoded,
 and instructions to get the next state.
 
@@ -1010,8 +1018,10 @@ position += (tableSize>>1) + (tableSize>>3) + 3;
 position &= tableSize-1;
 ```
 
-A position is skipped if already occupied,
-typically by a "less than 1" probability symbol.
+A position is skipped if already occupied by a "less than 1" probability symbol.
+`position` does not reset between symbols, it simply iterates through
+each position in the table, switching to the next symbol when enough
+states have been allocated to the current one.
 
 The result is a list of state values.
 Each state will decode the current symbol.
@@ -1043,19 +1053,19 @@ Numbering starts from higher states using less bits.
 | `Baseline`       |  32   |  64   |   96   |   0  |  16   |
 | range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
 
-Next state is determined from current state
+The next state is determined from current state
 by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
 
+See [Appendix A] for the results of this process applied to the default distributions.
 
-#### Bitstream
+[Appendix A]: #appendix-a---decoding-tables-for-predefined-codes
 
-FSE bitstreams are read in reverse direction than written. In zstd,
-the compressor writes bits forward into a block and the decompressor
-must read the bitstream _backwards_.
-
-To find the start of the bitstream it is therefore necessary to
-know the offset of the last byte of the block which can be found
-by counting `Block_Size` bytes after the block header.
+Huffman Coding
+--------------
+Zstandard Huffman-coded streams are read backwards,
+similar to the FSE bitstreams.
+Therefore, to find the start of the bitstream it is therefore necessary to
+know the offset of the last byte of the Huffman-coded stream.
 
 After writing the last bit containing information, the compressor
 writes a single `1`-bit and then fills the byte with 0-7 `0` bits of
@@ -1067,76 +1077,195 @@ byte to read. The decompressor needs to skip 0-7 initial `0`-bits and
 the first `1`-bit it occurs. Afterwards, the useful part of the bitstream
 begins.
 
-##### Starting states
+The bitstream contains Huffman-coded symbols in little-endian order,
+with the codes defined by the method below.
 
-The bitstream starts with initial state values,
-each using the required number of bits in their respective _accuracy_,
-decoded previously from their normalized distribution.
+### Huffman Tree Description
+Prefix coding represents symbols from an a priori known alphabet
+by bit sequences (codewords), one codeword for each symbol,
+in a manner such that different symbols may be represented
+by bit sequences of different lengths,
+but a parser can always parse an encoded string
+unambiguously symbol-by-symbol.
 
-It starts by `Literals_Length_State`,
-followed by `Offset_State`,
-and finally `Match_Length_State`.
+Given an alphabet with known symbol frequencies,
+the Huffman algorithm allows the construction of an optimal prefix code
+using the fewest bits of any possible prefix codes for that alphabet.
 
-Reminder : always keep in mind that all values are read _backward_.
-
-##### Decoding a sequence
-
-A state gives a code.
-A code provides `Baseline` and `Number_of_Bits` to add.
-See [Symbol Decoding] section for details on each symbol.
-
-Decoding starts by reading the `Number_of_Bits` required to decode `Offset`.
-It then does the same for `Match_Length`,
-and then for `Literals_Length`.
-
-`Offset`, `Match_Length`, and `Literals_Length` define a sequence.
-It starts by inserting the number of literals defined by `Literals_Length`,
-then continue by copying `Match_Length` bytes from `currentPos - Offset`.
-
-The next operation is to update states.
-Using rules pre-calculated in the decoding tables,
-`Literals_Length_State` is updated,
-followed by `Match_Length_State`,
-and then `Offset_State`.
-
-This operation will be repeated `Number_of_Sequences` times.
-At the end, the bitstream shall be entirely consumed,
-otherwise bitstream is considered corrupted.
-
-[Symbol Decoding]:#the-codes-for-literals-lengths-match-lengths-and-offsets
-
-##### Repeat offsets
-
-As seen in [Offset Codes], the first 3 values define a repeated offset and we will call them `Repeated_Offset1`, `Repeated_Offset2`, and `Repeated_Offset3`.
-They are sorted in recency order, with `Repeated_Offset1` meaning "most recent one".
-
-There is an exception though, when current sequence's literals length is `0`.
-In which case, repeated offsets are "pushed by one",
-so `Repeated_Offset1` becomes `Repeated_Offset2`, `Repeated_Offset2` becomes `Repeated_Offset3`,
-and `Repeated_Offset3` becomes `Repeated_Offset1 - 1_byte`.
-
-On first block, offset history is populated by the following values : 1, 4 and 8 (in order).
-
-Then each block receives its start value from previous compressed block.
-Note that non-compressed blocks are skipped,
-they do not contribute to offset history.
-
-[Offset Codes]: #offset-codes
-
-###### Offset updates rules
-
-New offset take the lead in offset history,
-up to its previous place if it was already present.
-
-It means that when `Repeated_Offset1` (most recent) is used, history is unmodified.
-When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
+Prefix code must not exceed a maximum code length.
+More bits improve accuracy but cost more header size,
+and require more memory or more complex decoding operations.
+This specification limits maximum code length to 11 bits.
 
 
-Dictionary format
+##### Representation
+
+All literal values from zero (included) to last present one (excluded)
+are represented by `Weight` with values from `0` to `Max_Number_of_Bits`.
+Transformation from `Weight` to `Number_of_Bits` follows this formula :
+```
+Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
+```
+The last symbol's `Weight` is deduced from previously decoded ones,
+by completing to the nearest power of 2.
+This power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
+
+__Example__ :
+Let's presume the following Huffman tree must be described :
+
+|     literal      |  0  |  1  |  2  |  3  |  4  |  5  |
+| ---------------- | --- | --- | --- | --- | --- | --- |
+| `Number_of_Bits` |  1  |  2  |  3  |  0  |  4  |  4  |
+
+The tree depth is 4, since its smallest element uses 4 bits.
+Value `5` will not be listed as it can be determined from the values for 0-4,
+nor will values above `5` as they are all 0.
+Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
+Weight formula is :
+```
+Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0
+```
+It gives the following series of weights :
+
+| literal  |  0  |  1  |  2  |  3  |  4  |
+| -------- | --- | --- | --- | --- | --- |
+| `Weight` |  4  |  3  |  2  |  0  |  1  |
+
+The decoder will do the inverse operation :
+having collected weights of literals from `0` to `4`,
+it knows the last literal, `5`, is present with a non-zero weight.
+The weight of `5` can be determined by advancing to the next power of 2.
+The sum of `2^(Weight-1)` (excluding 0's) is :
+`8 + 4 + 2 + 0 + 1 = 15`.
+Nearest power of 2 is 16.
+Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = 1`.
+
+##### Huffman Tree header
+
+This is a single byte value (0-255),
+which describes how to decode the list of weights.
+
+- if `headerByte` >= 128 : this is a direct representation,
+  where each `Weight` is written directly as a 4 bits field (0-15).
+  They are encoded forward, 2 weights to a byte with the first weight taking
+  the top four bits and the second taking the bottom four (e.g. the following
+  operations could be used to read the weights:
+  `Weight[0] = (Byte[0] >> 4), Weight[1] = (Byte[0] & 0xf)`, etc.).
+  The full representation occupies `((Number_of_Symbols+1)/2)` bytes,
+  meaning it uses a last full byte even if `Number_of_Symbols` is odd.
+  `Number_of_Symbols = headerByte - 127`.
+  Note that maximum `Number_of_Symbols` is 255-127 = 128.
+  A larger series must necessarily use FSE compression.
+
+- if `headerByte` < 128 :
+  the series of weights is compressed by FSE.
+  The length of the FSE-compressed series is equal to `headerByte` (0-127).
+
+##### Finite State Entropy (FSE) compression of Huffman weights
+
+In this case, the series of Huffman weights is compressed using FSE compression.
+It's a single bitstream with 2 interleaved states,
+sharing a single distribution table.
+
+To decode an FSE bitstream, it is necessary to know its compressed size.
+Compressed size is provided by `headerByte`.
+It's also necessary to know its _maximum possible_ decompressed size,
+which is `255`, since literal values span from `0` to `255`,
+and last symbol's weight is not represented.
+
+An FSE bitstream starts by a header, describing probabilities distribution.
+It will create a Decoding Table.
+For a list of Huffman weights, the maximum accuracy log is 7 bits.
+For more description see the [FSE header description](#fse-table-description)
+
+The Huffman header compression uses 2 states,
+which share the same FSE distribution table.
+The first state (`State1`) encodes the even indexed symbols,
+and the second (`State2`) encodes the odd indexes.
+State1 is initialized first, and then State2, and they take turns decoding
+a single symbol and updating their state.
+For more details on these FSE operations, see the [FSE section](#fse).
+
+The number of symbols to decode is determined
+by tracking bitStream overflow condition:
+If updating state after decoding a symbol would require more bits than
+remain in the stream, it is assumed the extra bits are 0.  Then,
+the symbols for each of the final states are decoded and the process is complete.
+
+##### Conversion from weights to Huffman prefix codes
+
+All present symbols shall now have a `Weight` value.
+It is possible to transform weights into Number_of_Bits, using this formula:
+```
+Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0
+```
+Symbols are sorted by `Weight`. Within same `Weight`, symbols keep natural order.
+Symbols with a `Weight` of zero are removed.
+Then, starting from lowest weight, prefix codes are distributed in order.
+
+__Example__ :
+Let's presume the following list of weights has been decoded :
+
+| Literal  |  0  |  1  |  2  |  3  |  4  |  5  |
+| -------- | --- | --- | --- | --- | --- | --- |
+| `Weight` |  4  |  3  |  2  |  0  |  1  |  1  |
+
+Sorted by weight and then natural order,
+it gives the following distribution :
+
+| Literal          |  3  |  4  |  5  |  2  |  1  |   0  |
+| ---------------- | --- | --- | --- | --- | --- | ---- |
+| `Weight`         |  0  |  1  |  1  |  2  |  3  |   4  |
+| `Number_of_Bits` |  0  |  4  |  4  |  3  |  2  |   1  |
+| prefix codes     | N/A | 0000| 0001| 001 | 01  |   1  |
+
+### Huffman-coded Streams
+Given a Huffman decoding table,
+it's possible to decode a Huffman-coded stream.
+
+Each bitstream must be read _backward_,
+that is starting from the end down to the beginning.
+Therefore it's necessary to know the size of each bitstream.
+
+It's also necessary to know exactly which _bit_ is the latest.
+This is detected by a final bit flag :
+the highest bit of latest byte is a final-bit-flag.
+Consequently, a last byte of `0` is not possible.
+And the final-bit-flag itself is not part of the useful bitstream.
+Hence, the last byte contains between 0 and 7 useful bits.
+
+For example, if the literal sequence "0145" was encoded using the prefix codes above,
+it would be encoded as:
+```
+00000001 01110000
+```
+
+|Symbol  |   5  |   4  |  1 | 0 | Padding |
+|--------|------|------|----|---|---------|
+|Encoding|`0000`|`0001`|`01`|`1`| `10000` |
+
+Starting from the end,
+it's possible to read the bitstream in a little-endian fashion,
+keeping track of already used bits.  Since the bitstream is encoded in reverse
+order, by starting at the end the symbols can be read in forward order.
+
+Reading the last `Max_Number_of_Bits` bits,
+it's then possible to compare extracted value to decoding table,
+determining the symbol to decode and number of bits to discard.
+
+The process continues up to reading the required number of symbols per stream.
+If a bitstream is not entirely and exactly consumed,
+hence reaching exactly its beginning position with _all_ bits consumed,
+the decoding process is considered faulty.
+
+Dictionary Format
 -----------------
 
-`zstd` is compatible with "raw content" dictionaries, free of any format restriction,
+Zstandard is compatible with "raw content" dictionaries, free of any format restriction,
 except that they must be at least 8 bytes.
+These dictionaries function as if they were just the `Content` block of a formatted
+dictionary.
+
 But dictionaries created by `zstd --train` follow a format, described here.
 
 __Pre-requisites__ : a dictionary has a size,
@@ -1160,19 +1289,28 @@ _Reserved ranges :_
               - low range : 1 - 32767
               - high range : >= (2^31)
 
-__`Entropy_Tables`__ : following the same format as a [compressed blocks].
+__`Entropy_Tables`__ : following the same format as the tables in compressed blocks.
+              See the relevant [FSE](#fse-table-description)
+              and [Huffman](#huffman-tree-description) sections for how to decode these tables.
               They are stored in following order :
               Huffman tables for literals, FSE table for offsets,
               FSE table for match lengths, and FSE table for literals lengths.
-              It's finally followed by 3 offset values, populating recent offsets,
+              These tables populate the Repeat Stats literals mode and
+              Repeat distribution mode for sequence decoding.
+              It's finally followed by 3 offset values, populating recent offsets (instead of using `{1,4,8}`),
               stored in order, 4-bytes little-endian each, for a total of 12 bytes.
               Each recent offset must have a value < dictionary size.
 
 __`Content`__ : The rest of the dictionary is its content.
-              The content act as a "past" in front of data to compress or decompress.
+              The content act as a "past" in front of data to compress or decompress,
+              so it can be referenced in sequence commands.
+              As long as the amount of data decoded from this frame is less than or
+              equal to the window-size, sequence commands may specify offsets longer
+              than the lenght of total decoded output so far to reference back to the
+              dictionary.  After the total output has surpassed the window size however,
+              this is no longer allowed and the dictionary is no longer accessible.
 
 [compressed blocks]: #the-format-of-compressed_block
-
 Appendix A - Decoding tables for predefined codes
 -------------------------------------------------
 
@@ -1358,6 +1496,8 @@ to crosscheck that an implementation implements the decoding table generation al
 
 Version changes
 ---------------
+- 0.2.4 : section restructuring, by Sean Purcell
+- 0.2.3 : clarified several details, by Sean Purcell
 - 0.2.2 : added predefined codes, by Johannes Rudolph
 - 0.2.1 : clarify field names, by Przemyslaw Skibinski
 - 0.2.0 : numerous format adjustments for zstd v0.8

doc/zstd_manual.html

@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.1.2 Manual</title>
+<title>zstd 1.1.4 Manual</title>
 </head>
 <body>
-<h1>zstd 1.1.2 Manual</h1>
+<h1>zstd 1.1.4 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -19,20 +19,22 @@
 <li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
 <li><a href="#Chapter10">START OF ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
 <li><a href="#Chapter11">Advanced types</a></li>
-<li><a href="#Chapter12">Advanced compression functions</a></li>
-<li><a href="#Chapter13">Advanced decompression functions</a></li>
-<li><a href="#Chapter14">Advanced streaming functions</a></li>
-<li><a href="#Chapter15">Buffer-less and synchronous inner streaming functions</a></li>
-<li><a href="#Chapter16">Buffer-less streaming compression (synchronous mode)</a></li>
-<li><a href="#Chapter17">Buffer-less streaming decompression (synchronous mode)</a></li>
-<li><a href="#Chapter18">Block functions</a></li>
+<li><a href="#Chapter12">Compressed size functions</a></li>
+<li><a href="#Chapter13">Decompressed size functions</a></li>
+<li><a href="#Chapter14">Advanced compression functions</a></li>
+<li><a href="#Chapter15">Advanced decompression functions</a></li>
+<li><a href="#Chapter16">Advanced streaming functions</a></li>
+<li><a href="#Chapter17">Buffer-less and synchronous inner streaming functions</a></li>
+<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li>
+<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li>
+<li><a href="#Chapter20">Block functions</a></li>
 </ol>
 <hr>
 <a name="Chapter1"></a><h2>Introduction</h2><pre>
   zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
   at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
   decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
-  Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
@@ -53,8 +55,8 @@
 <a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
 
 <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
-                            const void* src, size_t srcSize,
-                                  int compressionLevel);
+                const void* src, size_t srcSize,
+                      int compressionLevel);
 </b><p>    Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
     Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
     @return : compressed size written into `dst` (<= `dstCapacity),
@@ -62,8 +64,8 @@
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
-                              const void* src, size_t compressedSize);
-</b><p>    `compressedSize` : must be the _exact_ size of a single compressed frame.
+                  const void* src, size_t compressedSize);
+</b><p>    `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
@@ -71,7 +73,16 @@
 </p></pre><BR>
 
 <pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-</b><p>   'src' is the start of a zstd compressed frame.
+</b><p>   NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
+   ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
+   frame, but distinguishes empty frames from frames with an unknown size, or errors.
+
+   Additionally, ZSTD_findDecompressedSize can be used instead.  It can handle multiple
+   concatenated frames in one buffer, and so is more general.
+   As a result however, it requires more computation and entire frames to be passed to it,
+   as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
+
+   'src' is the start of a zstd compressed frame.
    @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
     note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
              When `return==0`, data to decompress could be any size.
@@ -88,21 +99,33 @@
     note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. 
 </p></pre><BR>
 
-<h3>Helper functions</h3><pre><b>int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
+<h3>Helper functions</h3><pre></pre><b><pre>int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
 size_t      ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
 unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
 const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
-</b></pre><BR>
+</pre></b><BR>
 <a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
 
+<h3>Compression context</h3><pre>   When compressing many times,
+   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+   This will make workload friendlier for system's memory.
+   Use one context per thread for parallel execution in multi-threaded environments. 
+</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTD_CCtx* ZSTD_createCCtx(void);
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
+</pre></b><BR>
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 </b><p>    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 </p></pre><BR>
 
-<h3>Decompression context</h3><pre><b>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+<h3>Decompression context</h3><pre>   When decompressing many times,
+   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+   This will make workload friendlier for system's memory.
+   Use one context per thread for parallel execution in multi-threaded environments. 
+</pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-</b></pre><BR>
+</pre></b><BR>
 <pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 </b><p>   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
 </p></pre><BR>
@@ -110,19 +133,19 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 <a name="Chapter5"></a><h2>Simple dictionary API</h2><pre></pre>
 
 <pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const void* dict,size_t dictSize,
-                                           int compressionLevel);
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
+                               int compressionLevel);
 </b><p>   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
    Note : This function loads the dictionary, resulting in significant startup delay.
    Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
-                                             void* dst, size_t dstCapacity,
-                                       const void* src, size_t srcSize,
-                                       const void* dict,size_t dictSize);
+                                 void* dst, size_t dstCapacity,
+                           const void* src, size_t srcSize,
+                           const void* dict,size_t dictSize);
 </b><p>   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
    Dictionary must be identical to the one used during compression.
    Note : This function loads the dictionary, resulting in significant startup delay.
@@ -131,11 +154,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 <a name="Chapter6"></a><h2>Fast dictionary API</h2><pre></pre>
 
-<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
 </b><p>   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
    ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
    ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-   `dict` can be released after ZSTD_CDict creation. 
+   `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict 
 </p></pre><BR>
 
 <pre><b>size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -143,17 +166,17 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
-                                            void* dst, size_t dstCapacity,
-                                      const void* src, size_t srcSize,
-                                      const ZSTD_CDict* cdict);
+                                void* dst, size_t dstCapacity,
+                          const void* src, size_t srcSize,
+                          const ZSTD_CDict* cdict);
 </b><p>   Compression using a digested Dictionary.
    Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
    Note that compression level is decided during dictionary creation. 
 </p></pre><BR>
 
-<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
 </b><p>   Create a digested dictionary, ready to start decompression operation without startup delay.
-   `dict` can be released after creation. 
+   dictBuffer can be released after DDict creation, as its content is copied inside DDict 
 </p></pre><BR>
 
 <pre><b>size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
@@ -161,9 +184,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
-                                              void* dst, size_t dstCapacity,
-                                        const void* src, size_t srcSize,
-                                        const ZSTD_DDict* ddict);
+                                  void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                            const ZSTD_DDict* ddict);
 </b><p>   Decompression using a digested Dictionary.
    Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
 </p></pre><BR>
@@ -220,6 +243,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
  
 <BR></pre>
 
+<h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+</pre></b><BR>
+<h3>Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+</pre></b><BR>
 <pre><b>size_t ZSTD_CStreamInSize(void);    </b>/**< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_CStreamOutSize(void);   </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b>
@@ -245,6 +276,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
  
 <BR></pre>
 
+<h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+</pre></b><BR>
+<h3>Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+</pre></b><BR>
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_DStreamOutSize(void);   </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
@@ -271,9 +308,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 } ZSTD_compressionParameters;
 </b></pre><BR>
 <pre><b>typedef struct {
-    unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (if known). */<b>
-    unsigned checksumFlag;    </b>/**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */<b>
-    unsigned noDictIDFlag;    </b>/**< 1: no dict ID will be saved into frame header (if dictionary compression) */<b>
+    unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (when known) */<b>
+    unsigned checksumFlag;    </b>/**< 1: generate a 32-bits checksum at end of frame, for error detection */<b>
+    unsigned noDictIDFlag;    </b>/**< 1: no dictID will be saved into frame header (if dictionary compression) */<b>
 } ZSTD_frameParameters;
 </b></pre><BR>
 <pre><b>typedef struct {
@@ -281,11 +318,56 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     ZSTD_frameParameters fParams;
 } ZSTD_parameters;
 </b></pre><BR>
-<h3>Custom memory allocation functions</h3><pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-</b></pre><BR>
-<a name="Chapter12"></a><h2>Advanced compression functions</h2><pre></pre>
+</pre></b><BR>
+<a name="Chapter12"></a><h2>Compressed size functions</h2><pre></pre>
+
+<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+</b><p>  `src` should point to the start of a ZSTD encoded frame or skippable frame
+  `srcSize` must be at least as large as the frame
+  @return : the compressed size of the frame pointed to by `src`, suitable to pass to
+      `ZSTD_decompress` or similar, or an error code if given invalid input. 
+</p></pre><BR>
+
+<a name="Chapter13"></a><h2>Decompressed size functions</h2><pre></pre>
+
+<pre><b>unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+</b><p>   `src` should point to the start of a ZSTD encoded frame
+   `srcSize` must be at least as large as the frame header.  A value greater than or equal
+       to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
+   @return : decompressed size of the frame pointed to be `src` if known, otherwise
+             - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) 
+</p></pre><BR>
+
+<pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+</b><p>   `src` should point the start of a series of ZSTD encoded and/or skippable frames
+   `srcSize` must be the _exact_ size of this series
+       (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
+   @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
+             - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+             - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+
+    note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+             When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+             In which case, it's necessary to use streaming mode to decompress data.
+             Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+             (For example, data may be necessarily cut into blocks <= 16 KB).
+    note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+    note 3 : decompressed size can be very large (64-bits value),
+             potentially larger than what local system can handle as a single memory segment.
+             In which case, it's necessary to use streaming mode to decompress data.
+    note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+             Always ensure result fits within application's authorized limits.
+             Each application can set its own limits.
+    note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+             read each contained frame header.  This is efficient as most of the data is skipped,
+             however it does mean that all frame data must be present and valid. 
+</p></pre><BR>
+
+<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
 
 <pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
 </b><p>  Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
@@ -300,8 +382,24 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </b><p>  Gives the amount of memory used by a given ZSTD_CCtx 
 </p></pre><BR>
 
-<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
-                                                  ZSTD_parameters params, ZSTD_customMem customMem);
+<pre><b>typedef enum {
+    ZSTD_p_forceWindow,   </b>/* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */<b>
+    ZSTD_p_forceRawDict   </b>/* Force loading dictionary in "content-only" mode (no header analysis) */<b>
+} ZSTD_CCtxParameter;
+</b></pre><BR>
+<pre><b>size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
+</b><p>  Set advanced parameters, selected through enum ZSTD_CCtxParameter
+  @result : 0, or an error code (which can be tested with ZSTD_isError()) 
+</p></pre><BR>
+
+<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+</b><p>  Create a digested dictionary for compression
+  Dictionary content is simply referenced, and therefore stays in dictBuffer.
+  It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict 
+</p></pre><BR>
+
+<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
+                                      ZSTD_parameters params, ZSTD_customMem customMem);
 </b><p>  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 </p></pre><BR>
 
@@ -329,14 +427,14 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const void* dict,size_t dictSize,
-                                           ZSTD_parameters params);
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
+                               ZSTD_parameters params);
 </b><p>   Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter 
 </p></pre><BR>
 
-<a name="Chapter13"></a><h2>Advanced decompression functions</h2><pre></pre>
+<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre>
 
 <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
 </b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -357,6 +455,17 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </b><p>  Gives the amount of memory used by a given ZSTD_DCtx 
 </p></pre><BR>
 
+<pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+</b><p>  Create a digested dictionary, ready to start decompression operation without startup delay.
+  Dictionary content is simply referenced, and therefore stays in dictBuffer.
+  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict 
+</p></pre><BR>
+
+<pre><b>ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+                                      unsigned byReference, ZSTD_customMem customMem);
+</b><p>  Create a ZSTD_DDict using external alloc and free, optionally by reference 
+</p></pre><BR>
+
 <pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 </b><p>  Gives the amount of memory used by a given ZSTD_DDict 
 </p></pre><BR>
@@ -385,33 +494,33 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
   When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. 
 </p></pre><BR>
 
-<a name="Chapter14"></a><h2>Advanced streaming functions</h2><pre></pre>
+<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
 
-<h3>Advanced Streaming compression functions</h3><pre><b>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct */<b>
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */<b>
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize is optional and can be zero == unknown */<b>
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */<b>
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */<b>
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-</b></pre><BR>
-<h3>Advanced Streaming decompression functions</h3><pre><b>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+</pre></b><BR>
+<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-</b></pre><BR>
-<a name="Chapter15"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
+</pre></b><BR>
+<a name="Chapter17"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with many restrictions (documented below).
   Prefer using normal streaming API for an easier experience
  
 <BR></pre>
 
-<a name="Chapter16"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
+<a name="Chapter18"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -434,20 +543,21 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
     In which case, it will "discard" the relevant memory section from its history.
 
   Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
-  It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
-  Without last block mark, frames will be considered unfinished (broken) by decoders.
+  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+  Without last block mark, frames will be considered unfinished (corrupted) by decoders.
 
-  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
 <BR></pre>
 
-<h3>Buffer-less streaming compression functions</h3><pre><b>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
-size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */<b>
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */<b>
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-</b></pre><BR>
-<a name="Chapter17"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
+</pre></b><BR>
+<a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -490,7 +600,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames ==
+  == Special case : skippable frames 
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
@@ -499,6 +609,9 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   c) Frame Content - any content (User Data) of length equal to Frame Size
   For skippable frames ZSTD_decompressContinue() always returns 0.
   For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+    Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
+           For purposes of decompression, it is valid in both cases to skip the frame using
+           ZSTD_findFrameCompressedSize to find its size in bytes.
   It also returns Frame Size as fparamsPtr->frameContentSize.
 <BR></pre>
 
@@ -509,7 +622,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
     unsigned checksumFlag;
 } ZSTD_frameParams;
 </b></pre><BR>
-<h3>Buffer-less streaming decompression functions</h3><pre><b>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input, see details below */<b>
+<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input, see details below */<b>
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
@@ -517,8 +630,8 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-</b></pre><BR>
-<a name="Chapter18"></a><h2>Block functions</h2><pre>
+</pre></b><BR>
+<a name="Chapter20"></a><h2>Block functions</h2><pre>
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -542,10 +655,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
         Use ZSTD_insertBlock() in such a case.
 <BR></pre>
 
-<h3>Raw zstd block functions</h3><pre><b>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
 size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
-</b></pre><BR>
+</pre></b><BR>
 </html>
 </body>

examples/Makefile

@@ -9,7 +9,7 @@
 
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
-LDFLAGS+= -lzstd
+LDFLAGS += -lzstd
 
 .PHONY: default all clean test
 
@@ -52,16 +52,23 @@ clean:
 test: all
 	cp README.md tmp
 	cp Makefile tmp2
-	@echo starting simple compression
+	@echo -- Simple compression tests
 	./simple_compression tmp
 	./simple_decompression tmp.zst
 	./streaming_decompression tmp.zst > /dev/null
-	@echo starting streaming compression
+	@echo -- Streaming compression tests
 	./streaming_compression tmp
 	./streaming_decompression tmp.zst > /dev/null
-	@echo starting multiple streaming compression
+	@echo -- Edge cases detection
+	! ./streaming_decompression tmp    # invalid input, must fail
+	! ./simple_decompression tmp       # invalid input, must fail
+	! ./simple_decompression tmp.zst   # unknown input size, must fail
+	touch tmpNull                      # create 0-size file
+	./simple_compression tmpNull
+	./simple_decompression tmpNull.zst # 0-size frame : must work
+	@echo -- Multiple streaming tests
 	./multiple_streaming_compression *.c
-	@echo starting dictionary compression
+	@echo -- Dictionary compression tests
 	./dictionary_compression tmp2 tmp README.md
 	./dictionary_decompression tmp2.zst tmp.zst README.md
 	$(RM) tmp* *.zst

examples/dictionary_decompression.c

@@ -13,6 +13,7 @@
 #include <string.h>    // strerror
 #include <errno.h>     // errno
 #include <sys/stat.h>  // stat
+#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
 #include <zstd.h>      // presumes zstd library is installed
 
 
@@ -76,11 +77,15 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
 {
     size_t cSize;
     void* const cBuff = loadFile_orDie(fname, &cSize);
-    unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
-    if (rSize==0) {
+    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
+    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
+        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
+        exit(5);
+    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
         fprintf(stderr, "%s : original size unknown \n", fname);
         exit(6);
     }
+
     void* const rBuff = malloc_orDie((size_t)rSize);
 
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();

examples/simple_compression.c

@@ -102,7 +102,7 @@ static void compress_orDie(const char* fname, const char* oname)
 }
 
 
-static const char* createOutFilename_orDie(const char* filename)
+static char* createOutFilename_orDie(const char* filename)
 {
     size_t const inL = strlen(filename);
     size_t const outL = inL + 5;
@@ -110,7 +110,7 @@ static const char* createOutFilename_orDie(const char* filename)
     memset(outSpace, 0, outL);
     strcat(outSpace, filename);
     strcat(outSpace, ".zst");
-    return (const char*)outSpace;
+    return (char*)outSpace;
 }
 
 int main(int argc, const char** argv)
@@ -125,8 +125,8 @@ int main(int argc, const char** argv)
         return 1;
     }
 
-    const char* const outFilename = createOutFilename_orDie(inFilename);
+    char* const outFilename = createOutFilename_orDie(inFilename);
     compress_orDie(inFilename, outFilename);
-
+    free(outFilename);
     return 0;
 }

examples/simple_decompression.c

@@ -6,51 +6,50 @@
  * LICENSE-examples file in the root directory of this source tree.
  */
 
-
-
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf
 #include <string.h>    // strerror
 #include <errno.h>     // errno
 #include <sys/stat.h>  // stat
+#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
 #include <zstd.h>      // presumes zstd library is installed
 
 
-static off_t fsize_X(const char *filename)
+static off_t fsize_orDie(const char *filename)
 {
     struct stat st;
     if (stat(filename, &st) == 0) return st.st_size;
     /* error */
-    printf("stat: %s : %s \n", filename, strerror(errno));
+    fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno));
     exit(1);
 }
 
-static FILE* fopen_X(const char *filename, const char *instruction)
+static FILE* fopen_orDie(const char *filename, const char *instruction)
 {
     FILE* const inFile = fopen(filename, instruction);
     if (inFile) return inFile;
     /* error */
-    printf("fopen: %s : %s \n", filename, strerror(errno));
+    fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno));
     exit(2);
 }
 
-static void* malloc_X(size_t size)
+static void* malloc_orDie(size_t size)
 {
-    void* const buff = malloc(size);
+    void* const buff = malloc(size + !size);   /* avoid allocating size of 0 : may return NULL (implementation dependent) */
     if (buff) return buff;
     /* error */
-    printf("malloc: %s \n", strerror(errno));
+    fprintf(stderr, "malloc: %s \n", strerror(errno));
     exit(3);
 }
 
-static void* loadFile_X(const char* fileName, size_t* size)
+static void* loadFile_orDie(const char* fileName, size_t* size)
 {
-    off_t const buffSize = fsize_X(fileName);
-    FILE* const inFile = fopen_X(fileName, "rb");
-    void* const buffer = malloc_X(buffSize);
+    off_t const buffSize = fsize_orDie(fileName);
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
     size_t const readSize = fread(buffer, 1, buffSize, inFile);
     if (readSize != (size_t)buffSize) {
-        printf("fread: %s : %s \n", fileName, strerror(errno));
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
         exit(4);
     }
     fclose(inFile);   /* can't fail (read only) */
@@ -62,18 +61,23 @@ static void* loadFile_X(const char* fileName, size_t* size)
 static void decompress(const char* fname)
 {
     size_t cSize;
-    void* const cBuff = loadFile_X(fname, &cSize);
-    unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
-    if (rSize==0) {
-        printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
+    void* const cBuff = loadFile_orDie(fname, &cSize);
+    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
+    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
+        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
         exit(5);
+    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
+        fprintf(stderr,
+                "%s : original size unknown. Use streaming decompression instead.\n", fname);
+        exit(6);
     }
-    void* const rBuff = malloc_X((size_t)rSize);
+
+    void* const rBuff = malloc_orDie((size_t)rSize);
 
     size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
 
     if (dSize != rSize) {
-        printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
+        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
         exit(7);
     }
 

lib/BUCK

@@ -0,0 +1,186 @@
+cxx_library(
+    name='zstd',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    deps=[
+        ':common',
+        ':compress',
+        ':decompress',
+        ':deprecated',
+    ],
+)
+
+cxx_library(
+    name='compress',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('compress', 'zstdmt_compress.h'),
+    ]),
+    headers=subdir_glob([
+        ('compress', 'zstd_opt.h'),
+    ]),
+    srcs=[
+        'compress/zstd_compress.c',
+        'compress/zstdmt_compress.c',
+    ],
+    deps=[':common'],
+)
+
+cxx_library(
+    name='decompress',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    srcs=['decompress/zstd_decompress.c'],
+    deps=[
+        ':common',
+        ':legacy',
+    ],
+)
+
+cxx_library(
+    name='deprecated',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('decprecated', '*.h'),
+    ]),
+    srcs=glob(['deprecated/*.c']),
+    deps=[':common'],
+)
+
+cxx_library(
+    name='legacy',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('legacy', '*.h'),
+    ]),
+    srcs=glob(['legacy/*.c']),
+    deps=[':common'],
+)
+
+cxx_library(
+    name='zdict',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('dictBuilder', 'zdict.h'),
+    ]),
+    headers=subdir_glob([
+        ('dictBuilder', 'divsufsort.h'),
+    ]),
+    srcs=glob(['dictBuilder/*.c']),
+    deps=[':common'],
+)
+
+cxx_library(
+    name='bitstream',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'bitstream.h'),
+    ]),
+)
+
+cxx_library(
+    name='entropy',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'fse.h'),
+        ('common', 'huf.h'),
+    ]),
+    srcs=[
+        'common/entropy_common.c',
+        'common/fse_decompress.c',
+        'compress/fse_compress.c',
+        'compress/huf_compress.c',
+        'decompress/huf_decompress.c',
+    ],
+    deps=[
+        ':bitstream',
+        ':errors',
+        ':mem',
+    ],
+)
+
+cxx_library(
+    name='errors',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'error_private.h'),
+        ('common', 'zstd_errors.h'),
+    ]),
+    srcs=['common/error_private.c'],
+)
+
+cxx_library(
+    name='mem',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'mem.h'),
+    ]),
+)
+
+cxx_library(
+    name='pool',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'pool.h'),
+    ]),
+    srcs=['common/pool.c'],
+    deps=[':threading'],
+)
+
+cxx_library(
+    name='threading',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'threading.h'),
+    ]),
+    srcs=['common/threading.c'],
+)
+
+cxx_library(
+    name='xxhash',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'xxhash.h'),
+    ]),
+    srcs=['common/xxhash.c'],
+)
+
+cxx_library(
+    name='zstd_common',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('', 'zstd.h'),
+        ('common', 'zstd_internal.h'),
+    ]),
+    srcs=['common/zstd_common.c'],
+    deps=[
+        ':errors',
+        ':mem',
+    ],
+)
+
+cxx_library(
+    name='common',
+    deps=[
+        ':bitstream',
+        ':entropy',
+        ':errors',
+        ':mem',
+        ':pool',
+        ':threading',
+        ':xxhash',
+        ':zstd_common',
+    ]
+)

lib/Makefile

@@ -1,11 +1,13 @@
-# ################################################################
+# ##########################################################################
 # Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
+# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+#
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
+# ##########################################################################
 
 # Version numbers
 LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
@@ -18,28 +20,28 @@ LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER := $(shell echo $(LIBVER_SCRIPT))
 VERSION?= $(LIBVER)
 
-DESTDIR?=
-PREFIX ?= /usr/local
-LIBDIR ?= $(PREFIX)/lib
-INCLUDEDIR=$(PREFIX)/include
-
 CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
 CFLAGS  ?= -O3
-CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-           -Wpointer-arith
-CFLAGS  += $(MOREFLAGS)
+DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+           -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
 
 ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
 
-ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
-CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
-else
-CPPFLAGS  += -I./legacy -DZSTD_LEGACY_SUPPORT=1
-ZSTD_FILES+= $(wildcard legacy/*.c)
+ZSTD_LEGACY_SUPPORT ?= 4
+
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+	ZSTD_FILES += $(shell ls legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
+	CPPFLAGS += -I./legacy
+endif
+CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+
+ZSTD_OBJ   := $(patsubst %.c,%.o,$(ZSTD_FILES))
 
 # OS X linker doesn't support -soname, and use different extension
 # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
@@ -60,15 +62,14 @@ LIBZSTD = libzstd.$(SHARED_EXT_VER)
 
 .PHONY: default all clean install uninstall
 
-default: lib
+default: lib-release
 
 all: lib
 
 libzstd.a: ARFLAGS = rcs
-libzstd.a: $(ZSTD_FILES)
+libzstd.a: $(ZSTD_OBJ)
 	@echo compiling static library
-	@$(CC) $(FLAGS) -c $^
-	@$(AR) $(ARFLAGS) $@ *.o
+	@$(AR) $(ARFLAGS) $@ $^
 
 $(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
 $(LIBZSTD): $(ZSTD_FILES)
@@ -87,14 +88,41 @@ libzstd : $(LIBZSTD)
 
 lib: libzstd.a libzstd
 
+lib-release: DEBUGFLAGS :=
+lib-release: lib
+
 clean:
-	@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
-	@$(RM) decompress/*.o
+	@$(RM) -r *.dSYM   # Mac OS-X specific
+	@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
+	@$(RM) dll/libzstd.dll dll/libzstd.lib
+	@$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
 	@echo Cleaning library completed
 
-#------------------------------------------------------------------------
-#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+PREFIX     ?= /usr/local
+DESTDIR    ?=
+LIBDIR     ?= $(PREFIX)/lib
+INCLUDEDIR ?= $(PREFIX)/include
+
+ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
+PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
+else
+PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
+endif
+
+INSTALL_LIB  ?= $(INSTALL) -m 755
+INSTALL_DATA ?= $(INSTALL) -m 644
+
 
 libzstd.pc:
 libzstd.pc: libzstd.pc.in
@@ -106,16 +134,18 @@ libzstd.pc: libzstd.pc.in
              $< >$@
 
 install: libzstd.a libzstd libzstd.pc
-	@install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
-	@install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-	@cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
-	@cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
-	@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
-	@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)
-	@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)
-	@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
-	@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)     # prototypes generate deprecation warnings
-	@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/
+	@$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
+	@echo Installing libraries
+	@$(INSTALL_LIB) libzstd.a $(DESTDIR)$(LIBDIR)
+	@$(INSTALL_LIB) libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
+	@ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	@ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	@echo Installing includes
+	@$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
+	@$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
+	@$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)     # prototypes generate deprecation warnings
+	@$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
 	@echo zstd static and shared library installed
 
 uninstall:
@@ -123,7 +153,7 @@ uninstall:
 	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
 	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
 	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
-	@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
+	@$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h   # Deprecated streaming functions

lib/common/bitstream.h

@@ -60,6 +60,9 @@ extern "C" {
 #  include <immintrin.h>   /* support for bextr (experimental) */
 #endif
 
+#define STREAM_ACCUMULATOR_MIN_32  25
+#define STREAM_ACCUMULATOR_MIN_64  57
+#define STREAM_ACCUMULATOR_MIN    ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
 
 /*-******************************************
 *  bitStream encoding API (write forward)

lib/common/entropy_common.c

@@ -43,27 +43,21 @@
 #include "huf.h"
 
 
-/*-****************************************
-*  FSE Error Management
-******************************************/
-unsigned FSE_isError(size_t code) { return ERR_isError(code); }
+/*===   Version   ===*/
+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
 
+
+/*===   Error Management   ===*/
+unsigned FSE_isError(size_t code) { return ERR_isError(code); }
 const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
 
-
-/* **************************************************************
-*  HUF Error Management
-****************************************************************/
 unsigned HUF_isError(size_t code) { return ERR_isError(code); }
-
 const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
 
 
 /*-**************************************************************
 *  FSE NCount encoding-decoding
 ****************************************************************/
-static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
-
 size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
                  const void* headerBuffer, size_t hbSize)
 {
@@ -117,21 +111,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
             } else {
                 bitStream >>= 2;
         }   }
-        {   short const max = (short)((2*threshold-1)-remaining);
-            short count;
+        {   int const max = (2*threshold-1) - remaining;
+            int count;
 
             if ((bitStream & (threshold-1)) < (U32)max) {
-                count = (short)(bitStream & (threshold-1));
-                bitCount   += nbBits-1;
+                count = bitStream & (threshold-1);
+                bitCount += nbBits-1;
             } else {
-                count = (short)(bitStream & (2*threshold-1));
+                count = bitStream & (2*threshold-1);
                 if (count >= threshold) count -= max;
-                bitCount   += nbBits;
+                bitCount += nbBits;
             }
 
             count--;   /* extra accuracy */
-            remaining -= FSE_abs(count);
-            normalizedCounter[charnum++] = count;
+            remaining -= count < 0 ? -count : count;   /* -1 means +1 */
+            normalizedCounter[charnum++] = (short)count;
             previous0 = !count;
             while (remaining < threshold) {
                 nbBits--;

lib/common/fse.h

@@ -45,6 +45,32 @@ extern "C" {
 #include <stddef.h>    /* size_t, ptrdiff_t */
 
 
+/*-*****************************************
+*  FSE_PUBLIC_API : control library symbols visibility
+******************************************/
+#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
+#  define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
+#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
+#  define FSE_PUBLIC_API __declspec(dllexport)
+#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
+#  define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define FSE_PUBLIC_API
+#endif
+
+/*------   Version   ------*/
+#define FSE_VERSION_MAJOR    0
+#define FSE_VERSION_MINOR    9
+#define FSE_VERSION_RELEASE  0
+
+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
+#define FSE_QUOTE(str) #str
+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
+
+#define FSE_VERSION_NUMBER  (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
+FSE_PUBLIC_API unsigned FSE_versionNumber(void);   /**< library version number; to be used when checking dll version */
+
 /*-****************************************
 *  FSE simple functions
 ******************************************/
@@ -56,8 +82,8 @@ extern "C" {
                      if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
                      if FSE_isError(return), compression failed (more details using FSE_getErrorName())
 */
-size_t FSE_compress(void* dst, size_t dstCapacity,
-              const void* src, size_t srcSize);
+FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
+                             const void* src, size_t srcSize);
 
 /*! FSE_decompress():
     Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
@@ -69,18 +95,18 @@ size_t FSE_compress(void* dst, size_t dstCapacity,
     Why ? : making this distinction requires a header.
     Header management is intentionally delegated to the user layer, which can better manage special cases.
 */
-size_t FSE_decompress(void* dst,  size_t dstCapacity,
-                const void* cSrc, size_t cSrcSize);
+FSE_PUBLIC_API size_t FSE_decompress(void* dst,  size_t dstCapacity,
+                               const void* cSrc, size_t cSrcSize);
 
 
 /*-*****************************************
 *  Tool functions
 ******************************************/
-size_t FSE_compressBound(size_t size);       /* maximum compressed size */
+FSE_PUBLIC_API size_t FSE_compressBound(size_t size);       /* maximum compressed size */
 
 /* Error Management */
-unsigned    FSE_isError(size_t code);        /* tells if a return value is an error code */
-const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
+FSE_PUBLIC_API unsigned    FSE_isError(size_t code);        /* tells if a return value is an error code */
+FSE_PUBLIC_API const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
 
 
 /*-*****************************************
@@ -94,7 +120,7 @@ const char* FSE_getErrorName(size_t code);   /* provides error code string (usef
                      if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
                      if FSE_isError(return), it's an error code.
 */
-size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
 
 
 /*-*****************************************
@@ -127,50 +153,50 @@ or to save and provide normalized distribution using external method.
     @return : the count of the most frequent symbol (which is not identified).
               if return == srcSize, there is only one symbol.
               Can also return an error code, which can be tested with FSE_isError(). */
-size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
+FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
 
 /*! FSE_optimalTableLog():
     dynamically downsize 'tableLog' when conditions are met.
     It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
     @return : recommended tableLog (necessarily <= 'maxTableLog') */
-unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
 
 /*! FSE_normalizeCount():
     normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
     'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
     @return : tableLog,
               or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
+FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
 
 /*! FSE_NCountWriteBound():
     Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
     Typically useful for allocation purpose. */
-size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_writeNCount():
     Compactly save 'normalizedCounter' into 'buffer'.
     @return : size of the compressed table,
               or an errorCode, which can be tested using FSE_isError(). */
-size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 
 /*! Constructor and Destructor of FSE_CTable.
     Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
 typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
-FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
-void        FSE_freeCTable (FSE_CTable* ct);
+FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
+FSE_PUBLIC_API void        FSE_freeCTable (FSE_CTable* ct);
 
 /*! FSE_buildCTable():
     Builds `ct`, which must be already allocated, using FSE_createCTable().
     @return : 0, or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_compress_usingCTable():
     Compress `src` using `ct` into `dst` which must be already allocated.
     @return : size of compressed data (<= `dstCapacity`),
               or 0 if compressed data could not fit into `dst`,
               or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
+FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
 
 /*!
 Tutorial :
@@ -223,25 +249,25 @@ If there is an error, the function will return an ErrorCode (which can be tested
     @return : size read from 'rBuffer',
               or an errorCode, which can be tested using FSE_isError().
               maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
-size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
+FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
 
 /*! Constructor and Destructor of FSE_DTable.
     Note that its size depends on 'tableLog' */
 typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be more restrictive than void* */
-FSE_DTable* FSE_createDTable(unsigned tableLog);
-void        FSE_freeDTable(FSE_DTable* dt);
+FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
+FSE_PUBLIC_API void        FSE_freeDTable(FSE_DTable* dt);
 
 /*! FSE_buildDTable():
     Builds 'dt', which must be already allocated, using FSE_createDTable().
     return : 0, or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_decompress_usingDTable():
     Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
     into `dst` which must be already allocated.
     @return : size of regenerated data (necessarily <= `dstCapacity`),
               or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
+FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
 
 /*!
 Tutorial :

lib/common/fse_decompress.c

@@ -59,7 +59,6 @@
 ****************************************************************/
 #include <stdlib.h>     /* malloc, free, qsort */
 #include <string.h>     /* memcpy, memset */
-#include <stdio.h>      /* printf (debug) */
 #include "bitstream.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"

lib/common/huf.h

@@ -91,7 +91,7 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
 
 /** HUF_compress4X_wksp() :
 *   Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
-size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 
 
@@ -102,10 +102,11 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
 
 
 /* *** Constants *** */
-#define HUF_TABLELOG_ABSOLUTEMAX  15   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
-#define HUF_TABLELOG_MAX  12           /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
+#define HUF_TABLELOG_MAX      12       /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
 #define HUF_TABLELOG_DEFAULT  11       /* tableLog by default, when not specified */
-#define HUF_SYMBOLVALUE_MAX 255
+#define HUF_SYMBOLVALUE_MAX  255
+
+#define HUF_TABLELOG_ABSOLUTEMAX  15   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
 #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
 #  error "HUF_TABLELOG_MAX is too large !"
 #endif
@@ -133,6 +134,10 @@ typedef U32 HUF_DTable;
 #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
         HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
 
+/* The workspace must have alignment at least 4 and be at least this large */
+#define HUF_WORKSPACE_SIZE (6 << 10)
+#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
+
 
 /* ****************************************
 *  Advanced decompression functions
@@ -168,6 +173,17 @@ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSym
 size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
 size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 
+typedef enum {
+   HUF_repeat_none,  /**< Cannot use the previous table */
+   HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
+   HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+ } HUF_repeat;
+/** HUF_compress4X_repeat() :
+*   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+*   If it uses hufTable it does not modify hufTable or repeat.
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+*   If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@@ -214,8 +230,14 @@ size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* c
 /* single stream variants */
 
 size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
+/** HUF_compress1X_repeat() :
+*   Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+*   If it uses hufTable it does not modify hufTable or repeat.
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+*   If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */

lib/common/mem.h

@@ -39,7 +39,7 @@ extern "C" {
 #endif
 
 /* code only tested on 32 and 64 bits systems */
-#define MEM_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
+#define MEM_STATIC_ASSERT(c)   { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
 MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
 
 
@@ -48,14 +48,15 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include <stdint.h>
-  typedef  uint8_t BYTE;
-  typedef uint16_t U16;
-  typedef  int16_t S16;
-  typedef uint32_t U32;
-  typedef  int32_t S32;
-  typedef uint64_t U64;
-  typedef  int64_t S64;
-  typedef intptr_t iPtrDiff;
+  typedef   uint8_t BYTE;
+  typedef  uint16_t U16;
+  typedef   int16_t S16;
+  typedef  uint32_t U32;
+  typedef   int32_t S32;
+  typedef  uint64_t U64;
+  typedef   int64_t S64;
+  typedef  intptr_t iPtrDiff;
+  typedef uintptr_t uPtrDiff;
 #else
   typedef unsigned char      BYTE;
   typedef unsigned short      U16;
@@ -65,6 +66,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
   typedef unsigned long long  U64;
   typedef   signed long long  S64;
   typedef ptrdiff_t      iPtrDiff;
+  typedef size_t         uPtrDiff;
 #endif
 
 
@@ -76,11 +78,11 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
  * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
  * The below switch allow to select different access method for improved performance.
  * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ *            In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
  * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
@@ -182,7 +184,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_ulong(in);
-#elif defined (__GNUC__)
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
     return __builtin_bswap32(in);
 #else
     return  ((in << 24) & 0xff000000 ) |
@@ -196,7 +198,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_uint64(in);
-#elif defined (__GNUC__)
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
     return __builtin_bswap64(in);
 #else
     return  ((in << 56) & 0xff00000000000000ULL) |

lib/common/pool.c

@@ -0,0 +1,194 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* ======   Dependencies   ======= */
+#include <stddef.h>  /* size_t */
+#include <stdlib.h>  /* malloc, calloc, free */
+#include "pool.h"
+
+/* ======   Compiler specifics   ====== */
+#if defined(_MSC_VER)
+#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+#ifdef ZSTD_MULTITHREAD
+
+#include "threading.h"   /* pthread adaptation */
+
+/* A job is a function and an opaque argument */
+typedef struct POOL_job_s {
+  POOL_function function;
+  void *opaque;
+} POOL_job;
+
+struct POOL_ctx_s {
+    /* Keep track of the threads */
+    pthread_t *threads;
+    size_t numThreads;
+
+    /* The queue is a circular buffer */
+    POOL_job *queue;
+    size_t queueHead;
+    size_t queueTail;
+    size_t queueSize;
+    /* The mutex protects the queue */
+    pthread_mutex_t queueMutex;
+    /* Condition variable for pushers to wait on when the queue is full */
+    pthread_cond_t queuePushCond;
+    /* Condition variables for poppers to wait on when the queue is empty */
+    pthread_cond_t queuePopCond;
+    /* Indicates if the queue is shutting down */
+    int shutdown;
+};
+
+/* POOL_thread() :
+   Work thread for the thread pool.
+   Waits for jobs and executes them.
+   @returns : NULL on failure else non-null.
+*/
+static void* POOL_thread(void* opaque) {
+    POOL_ctx* const ctx = (POOL_ctx*)opaque;
+    if (!ctx) { return NULL; }
+    for (;;) {
+        /* Lock the mutex and wait for a non-empty queue or until shutdown */
+        pthread_mutex_lock(&ctx->queueMutex);
+        while (ctx->queueHead == ctx->queueTail && !ctx->shutdown) {
+            pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+        }
+        /* empty => shutting down: so stop */
+        if (ctx->queueHead == ctx->queueTail) {
+            pthread_mutex_unlock(&ctx->queueMutex);
+            return opaque;
+        }
+        /* Pop a job off the queue */
+        {   POOL_job const job = ctx->queue[ctx->queueHead];
+            ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
+            /* Unlock the mutex, signal a pusher, and run the job */
+            pthread_mutex_unlock(&ctx->queueMutex);
+            pthread_cond_signal(&ctx->queuePushCond);
+            job.function(job.opaque);
+        }
+    }
+    /* Unreachable */
+}
+
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+    POOL_ctx *ctx;
+    /* Check the parameters */
+    if (!numThreads || !queueSize) { return NULL; }
+    /* Allocate the context and zero initialize */
+    ctx = (POOL_ctx *)calloc(1, sizeof(POOL_ctx));
+    if (!ctx) { return NULL; }
+    /* Initialize the job queue.
+     * It needs one extra space since one space is wasted to differentiate empty
+     * and full queues.
+     */
+    ctx->queueSize = queueSize + 1;
+    ctx->queue = (POOL_job *)malloc(ctx->queueSize * sizeof(POOL_job));
+    ctx->queueHead = 0;
+    ctx->queueTail = 0;
+    pthread_mutex_init(&ctx->queueMutex, NULL);
+    pthread_cond_init(&ctx->queuePushCond, NULL);
+    pthread_cond_init(&ctx->queuePopCond, NULL);
+    ctx->shutdown = 0;
+    /* Allocate space for the thread handles */
+    ctx->threads = (pthread_t *)malloc(numThreads * sizeof(pthread_t));
+    ctx->numThreads = 0;
+    /* Check for errors */
+    if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
+    /* Initialize the threads */
+    {   size_t i;
+        for (i = 0; i < numThreads; ++i) {
+            if (pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+                ctx->numThreads = i;
+                POOL_free(ctx);
+                return NULL;
+        }   }
+        ctx->numThreads = numThreads;
+    }
+    return ctx;
+}
+
+/*! POOL_join() :
+    Shutdown the queue, wake any sleeping threads, and join all of the threads.
+*/
+static void POOL_join(POOL_ctx *ctx) {
+    /* Shut down the queue */
+    pthread_mutex_lock(&ctx->queueMutex);
+    ctx->shutdown = 1;
+    pthread_mutex_unlock(&ctx->queueMutex);
+    /* Wake up sleeping threads */
+    pthread_cond_broadcast(&ctx->queuePushCond);
+    pthread_cond_broadcast(&ctx->queuePopCond);
+    /* Join all of the threads */
+    {   size_t i;
+        for (i = 0; i < ctx->numThreads; ++i) {
+            pthread_join(ctx->threads[i], NULL);
+    }   }
+}
+
+void POOL_free(POOL_ctx *ctx) {
+    if (!ctx) { return; }
+    POOL_join(ctx);
+    pthread_mutex_destroy(&ctx->queueMutex);
+    pthread_cond_destroy(&ctx->queuePushCond);
+    pthread_cond_destroy(&ctx->queuePopCond);
+    if (ctx->queue) free(ctx->queue);
+    if (ctx->threads) free(ctx->threads);
+    free(ctx);
+}
+
+void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
+    POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
+    if (!ctx) { return; }
+
+    pthread_mutex_lock(&ctx->queueMutex);
+    {   POOL_job const job = {function, opaque};
+        /* Wait until there is space in the queue for the new job */
+        size_t newTail = (ctx->queueTail + 1) % ctx->queueSize;
+        while (ctx->queueHead == newTail && !ctx->shutdown) {
+          pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+          newTail = (ctx->queueTail + 1) % ctx->queueSize;
+        }
+        /* The queue is still going => there is space */
+        if (!ctx->shutdown) {
+            ctx->queue[ctx->queueTail] = job;
+            ctx->queueTail = newTail;
+        }
+    }
+    pthread_mutex_unlock(&ctx->queueMutex);
+    pthread_cond_signal(&ctx->queuePopCond);
+}
+
+#else  /* ZSTD_MULTITHREAD  not defined */
+/* No multi-threading support */
+
+/* We don't need any data, but if it is empty malloc() might return NULL. */
+struct POOL_ctx_s {
+  int data;
+};
+
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+  (void)numThreads;
+  (void)queueSize;
+  return (POOL_ctx *)malloc(sizeof(POOL_ctx));
+}
+
+void POOL_free(POOL_ctx *ctx) {
+  if (ctx) free(ctx);
+}
+
+void POOL_add(void *ctx, POOL_function function, void *opaque) {
+  (void)ctx;
+  function(opaque);
+}
+
+#endif  /* ZSTD_MULTITHREAD */

lib/common/pool.h

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#ifndef POOL_H
+#define POOL_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+#include <stddef.h>   /* size_t */
+
+typedef struct POOL_ctx_s POOL_ctx;
+
+/*! POOL_create() :
+    Create a thread pool with at most `numThreads` threads.
+    `numThreads` must be at least 1.
+    The maximum number of queued jobs before blocking is `queueSize`.
+    `queueSize` must be at least 1.
+    @return : The POOL_ctx pointer on success else NULL.
+*/
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
+
+/*! POOL_free() :
+    Free a thread pool returned by POOL_create().
+*/
+void POOL_free(POOL_ctx *ctx);
+
+/*! POOL_function :
+    The function type that can be added to a thread pool.
+*/
+typedef void (*POOL_function)(void *);
+/*! POOL_add_function :
+    The function type for a generic thread pool add function.
+*/
+typedef void (*POOL_add_function)(void *, POOL_function, void *);
+
+/*! POOL_add() :
+    Add the job `function(opaque)` to the thread pool.
+    Possibly blocks until there is room in the queue.
+    Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed.
+*/
+void POOL_add(void *ctx, POOL_function function, void *opaque);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif

lib/common/threading.c

@@ -0,0 +1,80 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+/**
+ * This file will hold wrapper for systems, which do not support pthreads
+ */
+
+/* When ZSTD_MULTITHREAD is not defined, this file would become an empty translation unit.
+* Include some ISO C header code to prevent this and portably avoid related warnings.
+* (Visual C++: C4206 / GCC: -Wpedantic / Clang: -Wempty-translation-unit)
+*/
+#include <stddef.h>
+
+
+#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
+
+/**
+ * Windows minimalist Pthread Wrapper, based on :
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+
+
+/* ===  Dependencies  === */
+#include <process.h>
+#include <errno.h>
+#include "threading.h"
+
+
+/* ===  Implementation  === */
+
+static unsigned __stdcall worker(void *arg)
+{
+    pthread_t* const thread = (pthread_t*) arg;
+    thread->arg = thread->start_routine(thread->arg);
+    return 0;
+}
+
+int pthread_create(pthread_t* thread, const void* unused,
+            void* (*start_routine) (void*), void* arg)
+{
+    (void)unused;
+    thread->arg = arg;
+    thread->start_routine = start_routine;
+    thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
+
+    if (!thread->handle)
+        return errno;
+    else
+        return 0;
+}
+
+int _pthread_join(pthread_t * thread, void **value_ptr)
+{
+    DWORD result;
+
+    if (!thread->handle) return 0;
+
+    result = WaitForSingleObject(thread->handle, INFINITE);
+    switch (result) {
+    case WAIT_OBJECT_0:
+        if (value_ptr) *value_ptr = thread->arg;
+        return 0;
+    case WAIT_ABANDONED:
+        return EINVAL;
+    default:
+        return GetLastError();
+    }
+}
+
+#endif   /* ZSTD_MULTITHREAD */

lib/common/threading.h

@@ -0,0 +1,104 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#ifndef THREADING_H_938743
+#define THREADING_H_938743
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
+
+/**
+ * Windows minimalist Pthread Wrapper, based on :
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+#ifdef WINVER
+#  undef WINVER
+#endif
+#define WINVER       0x0600
+
+#ifdef _WIN32_WINNT
+#  undef _WIN32_WINNT
+#endif
+#define _WIN32_WINNT 0x0600
+
+#ifndef WIN32_LEAN_AND_MEAN
+#  define WIN32_LEAN_AND_MEAN
+#endif
+
+#include <windows.h>
+
+/* mutex */
+#define pthread_mutex_t           CRITICAL_SECTION
+#define pthread_mutex_init(a,b)   InitializeCriticalSection((a))
+#define pthread_mutex_destroy(a)  DeleteCriticalSection((a))
+#define pthread_mutex_lock(a)     EnterCriticalSection((a))
+#define pthread_mutex_unlock(a)   LeaveCriticalSection((a))
+
+/* condition variable */
+#define pthread_cond_t             CONDITION_VARIABLE
+#define pthread_cond_init(a, b)    InitializeConditionVariable((a))
+#define pthread_cond_destroy(a)    /* No delete */
+#define pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
+#define pthread_cond_signal(a)     WakeConditionVariable((a))
+#define pthread_cond_broadcast(a)  WakeAllConditionVariable((a))
+
+/* pthread_create() and pthread_join() */
+typedef struct {
+    HANDLE handle;
+    void* (*start_routine)(void*);
+    void* arg;
+} pthread_t;
+
+int pthread_create(pthread_t* thread, const void* unused,
+                   void* (*start_routine) (void*), void* arg);
+
+#define pthread_join(a, b) _pthread_join(&(a), (b))
+int _pthread_join(pthread_t* thread, void** value_ptr);
+
+/**
+ * add here more wrappers as required
+ */
+
+
+#elif defined(ZSTD_MULTITHREAD)   /* posix assumed ; need a better detection method */
+/* ===   POSIX Systems   === */
+#  include <pthread.h>
+
+#else  /* ZSTD_MULTITHREAD not defined */
+/* No multithreading support */
+
+#define pthread_mutex_t int   /* #define rather than typedef, as sometimes pthread support is implicit, resulting in duplicated symbols */
+#define pthread_mutex_init(a,b)
+#define pthread_mutex_destroy(a)
+#define pthread_mutex_lock(a)
+#define pthread_mutex_unlock(a)
+
+#define pthread_cond_t int
+#define pthread_cond_init(a,b)
+#define pthread_cond_destroy(a)
+#define pthread_cond_wait(a,b)
+#define pthread_cond_signal(a)
+#define pthread_cond_broadcast(a)
+
+/* do not use pthread_t */
+
+#endif /* ZSTD_MULTITHREAD */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* THREADING_H_938743 */

lib/common/xxhash.c

@@ -104,7 +104,9 @@ static void  XXH_free  (void* p)  { free(p); }
 #include <string.h>
 static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
 
-#define XXH_STATIC_LINKING_ONLY
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY
+#endif
 #include "xxhash.h"
 
 

lib/common/xxhash.h

@@ -64,16 +64,12 @@ XXH64       13.8 GB/s            1.9 GB/s
 XXH32        6.8 GB/s            6.0 GB/s
 */
 
-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
-#ifndef XXH_NAMESPACE
-#  define XXH_NAMESPACE ZSTD_  /* Zstandard specific */
-#endif
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
 
 
 /* ****************************
@@ -242,6 +238,11 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH
 /* **************************
 *  Canonical representation
 ****************************/
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+*  These functions allow transformation of hash result into and from its canonical format.
+*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
 typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
 typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
 
@@ -251,14 +252,9 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t
 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
 
-/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
-*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
-*  These functions allow transformation of hash result into and from its canonical format.
-*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
-*/
+#endif /* XXHASH_H_5627135585666179 */
 
 
-#ifdef XXH_STATIC_LINKING_ONLY
 
 /* ================================================================================================
    This section contains definitions which are not guaranteed to remain stable.
@@ -266,6 +262,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
    They shall only be used with static linking.
    Never use these definitions in association with dynamic linking !
 =================================================================================================== */
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345)
+#define XXH_STATIC_H_3543687687345
 
 /* These definitions are only meant to allow allocation of XXH state
    statically, on stack, or in a struct for example.
@@ -299,11 +297,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
 #    include "xxhash.c"   /* include xxhash functions as `static`, for inlining */
 #  endif
 
-#endif /* XXH_STATIC_LINKING_ONLY */
+#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */
 
 
 #if defined (__cplusplus)
 }
 #endif
-
-#endif /* XXHASH_H_5627135585666179 */

lib/common/zstd_common.c

@@ -41,11 +41,7 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
 
 /*! ZSTD_getErrorString() :
 *   provides error code string from enum */
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
-
-/* ---   ZBUFF Error Management  (deprecated)   --- */
-unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
-const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
 
 
 /*=**************************************************************

lib/common/zstd_errors.h

@@ -18,6 +18,20 @@ extern "C" {
 #include <stddef.h>   /* size_t */
 
 
+/* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#  define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+#  define ZSTDERRORLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#endif
+
 /*-****************************************
 *  error codes list
 ******************************************/
@@ -49,8 +63,8 @@ typedef enum {
 /*! ZSTD_getErrorCode() :
     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
     which can be used to compare directly with enum list published into "error_public.h" */
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
 
 
 #if defined (__cplusplus)

lib/common/zstd_internal.h

@@ -49,6 +49,10 @@
 #include "error_private.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
+#endif
+#include "xxhash.h"               /* XXH_reset, update, digest */
 
 
 /*-*************************************
@@ -267,4 +271,13 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
 }
 
 
+/* hidden functions */
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ *        do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
+
+
 #endif   /* ZSTD_CCOMMON_H_MODULE */

lib/compress/fse_compress.c

@@ -201,8 +201,6 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
     return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND;  /* maxSymbolValue==0 ? use default */
 }
 
-static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
-
 static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
                                        const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
                                        unsigned writeIsSafe)
@@ -258,16 +256,16 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
                 bitStream >>= 16;
                 bitCount -= 16;
         }   }
-        {   short count = normalizedCounter[charnum++];
-            const short max = (short)((2*threshold-1)-remaining);
-            remaining -= FSE_abs(count);
-            if (remaining<1) return ERROR(GENERIC);
+        {   int count = normalizedCounter[charnum++];
+            int const max = (2*threshold-1)-remaining;
+            remaining -= count < 0 ? -count : count;
             count++;   /* +1 for extra accuracy */
             if (count>=threshold) count += max;   /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
             bitStream += count << bitCount;
             bitCount  += nbBits;
             bitCount  -= (count<max);
             previous0  = (count==1);
+            if (remaining<1) return ERROR(GENERIC);
             while (remaining<threshold) nbBits--, threshold>>=1;
         }
         if (bitCount>16) {
@@ -508,6 +506,7 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
 
 static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
 {
+    short const NOT_YET_ASSIGNED = -2;
     U32 s;
     U32 distributed = 0;
     U32 ToDistribute;
@@ -533,7 +532,8 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
             total -= count[s];
             continue;
         }
-        norm[s]=-2;
+
+        norm[s]=NOT_YET_ASSIGNED;
     }
     ToDistribute = (1 << tableLog) - distributed;
 
@@ -541,7 +541,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
         /* risk of rounding to zero */
         lowOne = (U32)((total * 3) / (ToDistribute * 2));
         for (s=0; s<=maxSymbolValue; s++) {
-            if ((norm[s] == -2) && (count[s] <= lowOne)) {
+            if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
                 norm[s] = 1;
                 distributed++;
                 total -= count[s];
@@ -561,12 +561,19 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
         return 0;
     }
 
+    if (total == 0) {
+        /* all of the symbols were low enough for the lowOne or lowThreshold */
+        for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
+            if (norm[s] > 0) ToDistribute--, norm[s]++;
+        return 0;
+    }
+
     {   U64 const vStepLog = 62 - tableLog;
         U64 const mid = (1ULL << (vStepLog-1)) - 1;
         U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total;   /* scale on remaining */
         U64 tmpTotal = mid;
         for (s=0; s<=maxSymbolValue; s++) {
-            if (norm[s]==-2) {
+            if (norm[s]==NOT_YET_ASSIGNED) {
                 U64 const end = tmpTotal + (count[s] * rStep);
                 U32 const sStart = (U32)(tmpTotal >> vStepLog);
                 U32 const sEnd = (U32)(end >> vStepLog);

lib/compress/huf_compress.c

@@ -127,7 +127,7 @@ struct HUF_CElt_s {
 };   /* typedef'd to HUF_CElt within "huf.h" */
 
 /*! HUF_writeCTable() :
-    `CTable` : huffman tree to save, using huf representation.
+    `CTable` : Huffman tree to save, using huf representation.
     @return : size of saved CTable */
 size_t HUF_writeCTable (void* dst, size_t maxDstSize,
                         const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
@@ -409,6 +409,25 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
     return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
 }
 
+static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
+{
+    size_t nbBits = 0;
+    int s;
+    for (s = 0; s <= (int)maxSymbolValue; ++s) {
+        nbBits += CTable[s].nbBits * count[s];
+    }
+    return nbBits >> 3;
+}
+
+static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
+  int bad = 0;
+  int s;
+  for (s = 0; s <= (int)maxSymbolValue; ++s) {
+    bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
+  }
+  return !bad;
+}
+
 static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
 {
     BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
@@ -510,25 +529,43 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
 }
 
 
+static size_t HUF_compressCTable_internal(
+                BYTE* const ostart, BYTE* op, BYTE* const oend,
+                const void* src, size_t srcSize,
+                unsigned singleStream, const HUF_CElt* CTable)
+{
+    size_t const cSize = singleStream ?
+                         HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :
+                         HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
+    if (HUF_isError(cSize)) { return cSize; }
+    if (cSize==0) { return 0; }   /* uncompressible */
+    op += cSize;
+    /* check compressibility */
+    if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
+    return op-ostart;
+}
+
+
 /* `workSpace` must a table of at least 1024 unsigned */
 static size_t HUF_compress_internal (
                 void* dst, size_t dstSize,
                 const void* src, size_t srcSize,
                 unsigned maxSymbolValue, unsigned huffLog,
                 unsigned singleStream,
-                void* workSpace, size_t wkspSize)
+                void* workSpace, size_t wkspSize,
+                HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
 {
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstSize;
     BYTE* op = ostart;
 
-    union {
-        U32 count[HUF_SYMBOLVALUE_MAX+1];
-        HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
-    } table;   /* `count` can overlap with `CTable`; saves 1 KB */
+    U32* count;
+    size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
+    HUF_CElt* CTable;
+    size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
 
     /* checks & inits */
-    if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC);
+    if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC);
     if (!srcSize) return 0;  /* Uncompressed (note : 1 means rle, so first byte must be correct) */
     if (!dstSize) return 0;  /* cannot fit within dst budget */
     if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);   /* current block size limit */
@@ -536,38 +573,58 @@ static size_t HUF_compress_internal (
     if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
     if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
 
+    count = (U32*)workSpace;
+    workSpace = (BYTE*)workSpace + countSize;
+    wkspSize -= countSize;
+    CTable = (HUF_CElt*)workSpace;
+    workSpace = (BYTE*)workSpace + CTableSize;
+    wkspSize -= CTableSize;
+
+    /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
+    if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
+        return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+    }
+
     /* Scan input and build symbol stats */
-    {   CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
+    {   CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
         if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }   /* single symbol, rle */
         if (largest <= (srcSize >> 7)+1) return 0;   /* Fast heuristic : not compressible enough */
     }
 
+    /* Check validity of previous table */
+    if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
+        *repeat = HUF_repeat_none;
+    }
+    /* Heuristic : use existing table for small inputs */
+    if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
+        return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+    }
+
     /* Build Huffman Tree */
     huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
-    {   CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) );
+    {   CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
         huffLog = (U32)maxBits;
+        /* Zero the unused symbols so we can check it for validity */
+        memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
     }
 
     /* Write table description header */
-    {   CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) );
-        if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
+    {   CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) );
+        /* Check if using the previous table will be beneficial */
+        if (repeat && *repeat != HUF_repeat_none) {
+            size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
+            size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
+            if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
+                return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+            }
+        }
+        /* Use the new table */
+        if (hSize + 12ul >= srcSize) { return 0; }
         op += hSize;
+        if (repeat) { *repeat = HUF_repeat_none; }
+        if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */
     }
-
-    /* Compress */
-    {   size_t const cSize = (singleStream) ?
-                            HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) :   /* single segment */
-                            HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable);
-        if (HUF_isError(cSize)) return cSize;
-        if (cSize==0) return 0;   /* uncompressible */
-        op += cSize;
-    }
-
-    /* check compressibility */
-    if ((size_t)(op-ostart) >= srcSize-1)
-        return 0;
-
-    return op-ostart;
+    return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
 }
 
 
@@ -576,7 +633,16 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
+}
+
+size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
+                      const void* src, size_t srcSize,
+                      unsigned maxSymbolValue, unsigned huffLog,
+                      void* workSpace, size_t wkspSize,
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
+{
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
 }
 
 size_t HUF_compress1X (void* dst, size_t dstSize,
@@ -592,7 +658,16 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
+}
+
+size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
+                      const void* src, size_t srcSize,
+                      unsigned maxSymbolValue, unsigned huffLog,
+                      void* workSpace, size_t wkspSize,
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
+{
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
 }
 
 size_t HUF_compress2 (void* dst, size_t dstSize,

lib/compress/zstd_compress.c

@@ -13,8 +13,6 @@
 ***************************************/
 #include <string.h>         /* memset */
 #include "mem.h"
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"               /* XXH_reset, update, digest */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
@@ -51,8 +49,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
 /*-*************************************
 *  Context memory management
 ***************************************/
-struct ZSTD_CCtx_s
-{
+struct ZSTD_CCtx_s {
     const BYTE* nextSrc;    /* next block here to continue on current prefix */
     const BYTE* base;       /* All regular indexes relative to this position */
     const BYTE* dictBase;   /* extDict indexes relative to this position */
@@ -61,10 +58,12 @@ struct ZSTD_CCtx_s
     U32   nextToUpdate;     /* index from which to continue dictionary update */
     U32   nextToUpdate3;    /* index from which to continue dictionary update */
     U32   hashLog3;         /* dispatch table : larger == faster, more memory */
-    U32   loadedDictEnd;
+    U32   loadedDictEnd;    /* index of end of dictionary */
+    U32   forceWindow;      /* force back-references to respect limit of 1<<wLog, even for dictionary */
+    U32   forceRawDict;     /* Force loading dictionary in "content-only" mode (no header analysis) */
     ZSTD_compressionStage_e stage;
     U32   rep[ZSTD_REP_NUM];
-    U32   savedRep[ZSTD_REP_NUM];
+    U32   repToConfirm[ZSTD_REP_NUM];
     U32   dictID;
     ZSTD_parameters params;
     void* workSpace;
@@ -80,10 +79,11 @@ struct ZSTD_CCtx_s
     U32* chainTable;
     HUF_CElt* hufTable;
     U32 flagStaticTables;
+    HUF_repeat flagStaticHufTable;
     FSE_CTable offcodeCTable  [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
     FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
     FSE_CTable litlengthCTable  [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
-    unsigned tmpCounters[1024];
+    unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32];
 };
 
 ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -101,7 +101,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
     if (!cctx) return NULL;
     memset(cctx, 0, sizeof(ZSTD_CCtx));
-    memcpy(&(cctx->customMem), &customMem, sizeof(customMem));
+    cctx->customMem = customMem;
     return cctx;
 }
 
@@ -119,6 +119,16 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
     return sizeof(*cctx) + cctx->workSpaceSize;
 }
 
+size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
+{
+    switch(param)
+    {
+    case ZSTD_p_forceWindow : cctx->forceWindow = value>0; cctx->loadedDictEnd = 0; return 0;
+    case ZSTD_p_forceRawDict : cctx->forceRawDict = value>0; return 0;
+    default: return ERROR(parameter_unknown);
+    }
+}
+
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx)   /* hidden interface */
 {
     return &(ctx->seqStore);
@@ -237,14 +247,17 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
 
 /*! ZSTD_resetCCtx_advanced() :
-    note : 'params' must be validated */
+    note : `params` must be validated */
 static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
                                        ZSTD_parameters params, U64 frameContentSize,
                                        ZSTD_compResetPolicy_e const crp)
 {
     if (crp == ZSTDcrp_continue)
-        if (ZSTD_equivalentParams(params, zc->params))
+        if (ZSTD_equivalentParams(params, zc->params)) {
+            zc->flagStaticTables = 0;
+            zc->flagStaticHufTable = HUF_repeat_none;
             return ZSTD_continueCCtx(zc, params, frameContentSize);
+        }
 
     {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
         U32    const divider = (params.cParams.searchLength==3) ? 3 : 4;
@@ -278,6 +291,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         ptr = zc->hashTable3 + h3Size;
         zc->hufTable = (HUF_CElt*)ptr;
         zc->flagStaticTables = 0;
+        zc->flagStaticHufTable = HUF_repeat_none;
         ptr = ((U32*)ptr) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
 
         zc->nextToUpdate = 1;
@@ -318,6 +332,14 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     }
 }
 
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ *        do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
+    int i;
+    for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0;
+}
 
 /*! ZSTD_copyCCtx() :
 *   Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
@@ -327,8 +349,12 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
 {
     if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
+
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
-    ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset);
+    {   ZSTD_parameters params = srcCCtx->params;
+        params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+        ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
+    }
 
     /* copy tables */
     {   size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
@@ -351,12 +377,15 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
 
     /* copy entropy tables */
     dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
+    dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
     if (srcCCtx->flagStaticTables) {
-        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
         memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
         memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
         memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
     }
+    if (srcCCtx->flagStaticHufTable) {
+        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
+    }
 
     return 0;
 }
@@ -470,24 +499,28 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
 
     /* small ? don't even attempt compression (speed opt) */
 #   define LITERAL_NOENTROPY 63
-    {   size_t const minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY;
+    {   size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
         if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }
 
     if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
-    if (zc->flagStaticTables && (lhSize==3)) {
-        hType = set_repeat;
-        singleStream = 1;
-        cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable);
-    } else {
-        cLitSize = singleStream ? HUF_compress1X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters))
-                                : HUF_compress4X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters));
+    {   HUF_repeat repeat = zc->flagStaticHufTable;
+        int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
+        if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
+        cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
+                                : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
+        if (repeat != HUF_repeat_none) { hType = set_repeat; }    /* reused the existing table */
+        else { zc->flagStaticHufTable = HUF_repeat_check; }       /* now have a table to reuse */
     }
 
-    if ((cLitSize==0) | (cLitSize >= srcSize - minGain))
+    if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) {
+        zc->flagStaticHufTable = HUF_repeat_none;
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    if (cLitSize==1)
+    }
+    if (cLitSize==1) {
+        zc->flagStaticHufTable = HUF_repeat_none;
         return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+    }
 
     /* Build header */
     switch(lhSize)
@@ -555,11 +588,11 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
         mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
 }
 
-
-size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
+MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
                               size_t srcSize)
 {
+    const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
     const seqStore_t* seqStorePtr = &(zc->seqStore);
     U32 count[MaxSeq+1];
     S16 norm[MaxSeq+1];
@@ -693,7 +726,18 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
         if (MEM_32bits()) BIT_flushBits(&blockStream);
         BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
         if (MEM_32bits()) BIT_flushBits(&blockStream);
-        BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        if (longOffsets) {
+            U32 const ofBits = ofCodeTable[nbSeq-1];
+            int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+            if (extraBits) {
+                BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
+                BIT_flushBits(&blockStream);
+            }
+            BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
+                        ofBits - extraBits);
+        } else {
+            BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        }
         BIT_flushBits(&blockStream);
 
         {   size_t n;
@@ -715,7 +759,17 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
                 if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
                 BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
                 if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
-                BIT_addBits(&blockStream, sequences[n].offset, ofBits);         /* 31 */
+                if (longOffsets) {
+                    int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+                    if (extraBits) {
+                        BIT_addBits(&blockStream, sequences[n].offset, extraBits);
+                        BIT_flushBits(&blockStream);                            /* (7)*/
+                    }
+                    BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
+                                ofBits - extraBits);                            /* 31 */
+                } else {
+                    BIT_addBits(&blockStream, sequences[n].offset, ofBits);     /* 31 */
+                }
                 BIT_flushBits(&blockStream);                                    /* (7)*/
         }   }
 
@@ -730,16 +784,25 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
 
     /* check compressibility */
 _check_compressibility:
-    { size_t const minGain = ZSTD_minGain(srcSize);
-      size_t const maxCSize = srcSize - minGain;
-      if ((size_t)(op-ostart) >= maxCSize) return 0; }
+    {   size_t const minGain = ZSTD_minGain(srcSize);
+        size_t const maxCSize = srcSize - minGain;
+        if ((size_t)(op-ostart) >= maxCSize) {
+            zc->flagStaticHufTable = HUF_repeat_none;
+            return 0;
+    }   }
 
     /* confirm repcodes */
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->savedRep[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; }
 
     return op - ostart;
 }
 
+#if 0 /* for debug */
+#  define STORESEQ_DEBUG
+#include <stdio.h>   /* fprintf */
+U32 g_startDebug = 0;
+const BYTE* g_start = NULL;
+#endif
 
 /*! ZSTD_storeSeq() :
     Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
@@ -748,13 +811,14 @@ _check_compressibility:
 */
 MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
 {
-#if 0  /* for debug */
-    static const BYTE* g_start = NULL;
-    const U32 pos = (U32)((const BYTE*)literals - g_start);
-    if (g_start==NULL) g_start = (const BYTE*)literals;
-    //if ((pos > 1) && (pos < 50000))
-        printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
-               pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+#ifdef STORESEQ_DEBUG
+    if (g_startDebug) {
+        const U32 pos = (U32)((const BYTE*)literals - g_start);
+        if (g_start==NULL) g_start = (const BYTE*)literals;
+        if ((pos > 1895000) && (pos < 1895300))
+            fprintf(stderr, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
+                   pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+    }
 #endif
     /* copy Literals */
     ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@@ -1004,8 +1068,8 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
     }   }   }
 
     /* save reps for next block */
-    cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved;
-    cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved;
+    cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+    cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1119,7 +1183,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
     }   }   }
 
     /* save reps for next block */
-    ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+    ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1273,8 +1337,8 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
     }   }   }
 
     /* save reps for next block */
-    cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved;
-    cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved;
+    cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+    cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1423,7 +1487,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
     }   }   }
 
     /* save reps for next block */
-    ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+    ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1723,7 +1787,7 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict (
 #define NEXT_IN_CHAIN(d, mask)   chainTable[(d) & mask]
 
 /* Update chains up to ip (excluded)
-   Assumption : always within prefix (ie. not within extDict) */
+   Assumption : always within prefix (i.e. not within extDict) */
 FORCE_INLINE
 U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
 {
@@ -1955,8 +2019,8 @@ _storeSequence:
     }   }
 
     /* Save reps for next block */
-    ctx->savedRep[0] = offset_1 ? offset_1 : savedOffset;
-    ctx->savedRep[1] = offset_2 ? offset_2 : savedOffset;
+    ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
+    ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -2150,7 +2214,7 @@ _storeSequence:
     }   }
 
     /* Save reps for next block */
-    ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+    ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -2283,7 +2347,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         if (remaining < blockSize) blockSize = remaining;
 
         /* preemptive overflow correction */
-        if (cctx->lowLimit > (2U<<30)) {
+        if (cctx->lowLimit > (3U<<29)) {
             U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
             U32 const current = (U32)(ip - cctx->base);
             U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog);
@@ -2337,7 +2401,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     U32   const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536);   /* 0-3 */
     U32   const checksumFlag = params.fParams.checksumFlag>0;
     U32   const windowSize = 1U << params.cParams.windowLog;
-    U32   const singleSegment = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1));
+    U32   const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
     BYTE  const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
     U32   const fcsCode = params.fParams.contentSizeFlag ?
                      (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) :   /* 0-3 */
@@ -2409,12 +2473,14 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
 
     cctx->nextSrc = ip + srcSize;
 
-    {   size_t const cSize = frame ?
+    if (srcSize) {
+        size_t const cSize = frame ?
                              ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
                              ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
         if (ZSTD_isError(cSize)) return cSize;
         return cSize + fhSize;
-    }
+    } else
+        return fhSize;
 }
 
 
@@ -2450,7 +2516,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
     zc->dictBase = zc->base;
     zc->base += ip - zc->nextSrc;
     zc->nextToUpdate = zc->dictLimit;
-    zc->loadedDictEnd = (U32)(iend - zc->base);
+    zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
 
     zc->nextSrc = iend;
     if (srcSize <= HASH_READ_SIZE) return 0;
@@ -2481,7 +2547,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
         return ERROR(GENERIC);   /* strategy doesn't exist; impossible */
     }
 
-    zc->nextToUpdate = zc->loadedDictEnd;
+    zc->nextToUpdate = (U32)(iend - zc->base);
     return 0;
 }
 
@@ -2557,9 +2623,9 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
     }
 
     if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
-    cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
-    cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
-    cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
+    cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] == 0 || cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
+    cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] == 0 || cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
+    cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] == 0 || cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
     dictPtr += 12;
 
     {   U32 offcodeMax = MaxOff;
@@ -2573,6 +2639,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
     }
 
     cctx->flagStaticTables = 1;
+    cctx->flagStaticHufTable = HUF_repeat_valid;
     return dictPtr - (const BYTE*)dict;
 }
 
@@ -2582,8 +2649,9 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
 {
     if ((dict==NULL) || (dictSize<=8)) return 0;
 
-    /* default : dict is pure content */
-    if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
+    /* dict as pure content */
+    if ((MEM_readLE32(dict) != ZSTD_DICT_MAGIC) || (zc->forceRawDict))
+        return ZSTD_loadDictionaryContent(zc, dict, dictSize);
     zc->dictID = zc->params.fParams.noDictIDFlag ? 0 :  MEM_readLE32((const char*)dict+4);
 
     /* known magic number : dict is parsed for entropy stats and content */
@@ -2594,7 +2662,6 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
     }
 }
 
-
 /*! ZSTD_compressBegin_internal() :
 *   @return : 0, or an error code */
 static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
@@ -2626,9 +2693,9 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
 }
 
 
-size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
+size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
 {
-    return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel);
+    return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
 }
 
 
@@ -2733,7 +2800,8 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
 /* =====  Dictionary API  ===== */
 
 struct ZSTD_CDict_s {
-    void* dictContent;
+    void* dictBuffer;
+    const void* dictContent;
     size_t dictContentSize;
     ZSTD_CCtx* refContext;
 };  /* typedef'd tp ZSTD_CDict within "zstd.h" */
@@ -2741,39 +2809,45 @@ struct ZSTD_CDict_s {
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
 {
     if (cdict==NULL) return 0;   /* support sizeof on NULL */
-    return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize;
+    return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict);
 }
 
-ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
+                                      ZSTD_parameters params, ZSTD_customMem customMem)
 {
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
     if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
     {   ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
-        void* const dictContent = ZSTD_malloc(dictSize, customMem);
         ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
 
-        if (!dictContent || !cdict || !cctx) {
-            ZSTD_free(dictContent, customMem);
+        if (!cdict || !cctx) {
             ZSTD_free(cdict, customMem);
-            ZSTD_free(cctx, customMem);
+            ZSTD_freeCCtx(cctx);
             return NULL;
         }
 
-        if (dictSize) {
-            memcpy(dictContent, dict, dictSize);
+        if ((byReference) || (!dictBuffer) || (!dictSize)) {
+            cdict->dictBuffer = NULL;
+            cdict->dictContent = dictBuffer;
+        } else {
+            void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
+            if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; }
+            memcpy(internalBuffer, dictBuffer, dictSize);
+            cdict->dictBuffer = internalBuffer;
+            cdict->dictContent = internalBuffer;
         }
-        {   size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
+
+        {   size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
             if (ZSTD_isError(errorCode)) {
-                ZSTD_free(dictContent, customMem);
+                ZSTD_free(cdict->dictBuffer, customMem);
                 ZSTD_free(cdict, customMem);
-                ZSTD_free(cctx, customMem);
+                ZSTD_freeCCtx(cctx);
                 return NULL;
         }   }
 
-        cdict->dictContent = dictContent;
-        cdict->dictContentSize = dictSize;
         cdict->refContext = cctx;
+        cdict->dictContentSize = dictSize;
         return cdict;
     }
 }
@@ -2783,7 +2857,15 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
     ZSTD_customMem const allocator = { NULL, NULL, NULL };
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
     params.fParams.contentSizeFlag = 1;
-    return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
+    return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator);
+}
+
+ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
+{
+    ZSTD_customMem const allocator = { NULL, NULL, NULL };
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
+    params.fParams.contentSizeFlag = 1;
+    return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator);
 }
 
 size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
@@ -2791,7 +2873,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
     if (cdict==NULL) return 0;   /* support free on NULL */
     {   ZSTD_customMem const cMem = cdict->refContext->customMem;
         ZSTD_freeCCtx(cdict->refContext);
-        ZSTD_free(cdict->dictContent, cMem);
+        ZSTD_free(cdict->dictBuffer, cMem);
         ZSTD_free(cdict, cMem);
         return 0;
     }
@@ -2801,10 +2883,14 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
     return ZSTD_getParamsFromCCtx(cdict->refContext);
 }
 
-size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
 {
     if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
-    else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
+    else {
+        ZSTD_parameters params = cdict->refContext->params;
+        params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+        CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize));
+    }
     return 0;
 }
 
@@ -2898,9 +2984,9 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
 size_t ZSTD_CStreamInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
 size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
 
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
+static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
-    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once */
+    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once => can't reset */
 
     if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
     else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
@@ -2916,6 +3002,14 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     return 0;   /* ready to go */
 }
 
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
+{
+
+    zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+
+    return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
+}
+
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -2937,9 +3031,9 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         if (zcs->outBuff == NULL) return ERROR(memory_allocation);
     }
 
-    if (dict) {
+    if (dict && dictSize >= 8) {
         ZSTD_freeCDict(zcs->cdictLocal);
-        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
+        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
         zcs->cdict = zcs->cdictLocal;
     } else zcs->cdict = NULL;
@@ -2947,7 +3041,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
     zcs->checksum = params.fParams.checksumFlag > 0;
     zcs->params = params;
 
-    return ZSTD_resetCStream(zcs, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
 }
 
 /* note : cdict must outlive compression session */
@@ -2956,6 +3050,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
     ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
     size_t const initError =  ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
     zcs->cdict = cdict;
+    zcs->cctx->dictID = params.fParams.noDictIDFlag ? 0 : cdict->refContext->dictID;
     return initError;
 }
 
@@ -2967,7 +3062,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
 
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
 {
-    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+    if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
     return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
 }
 
@@ -2979,7 +3075,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
 {
     if (zcs==NULL) return 0;   /* support sizeof on NULL */
-    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
+    return sizeof(*zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
 }
 
 /*======   Compression   ======*/

lib/compress/zstd_opt.h

@@ -38,7 +38,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t src
 
     ssPtr->cachedLiterals = NULL;
     ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
-    ssPtr->staticPrices = 0; 
+    ssPtr->staticPrices = 0;
 
     if (ssPtr->litLengthSum == 0) {
         if (srcSize <= 1024) ssPtr->staticPrices = 1;
@@ -56,7 +56,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t src
 
         for (u=0; u<=MaxLit; u++) {
             ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV);
-            ssPtr->litSum += ssPtr->litFreq[u]; 
+            ssPtr->litSum += ssPtr->litFreq[u];
         }
         for (u=0; u<=MaxLL; u++)
             ssPtr->litLengthFreq[u] = 1;
@@ -203,7 +203,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
 
 
 /* Update hashTable3 up to ip (excluded)
-   Assumption : always within prefix (ie. not within extDict) */
+   Assumption : always within prefix (i.e. not within extDict) */
 FORCE_INLINE
 U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
 {
@@ -634,7 +634,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
     }    }   /* for (cur=0; cur < last_pos; ) */
 
     /* Save reps for next block */
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -825,7 +825,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
 
             match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
 
-            if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
+            if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
                 best_mlen = matches[match_num-1].len;
                 best_off = matches[match_num-1].off;
                 last_pos = cur + 1;
@@ -835,7 +835,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
             /* set prices using matches at position = cur */
             for (u = 0; u < match_num; u++) {
                 mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
-                best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur;
+                best_mlen = matches[u].len;
 
                 while (mlen <= best_mlen) {
                     if (opt[cur].mlen == 1) {
@@ -907,7 +907,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
     }    }   /* for (cur=0; cur < last_pos; ) */
 
     /* Save reps for next block */
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
 
     /* Last Literals */
     {   size_t lastLLSize = iend - anchor;

lib/compress/zstdmt_compress.c

@@ -0,0 +1,739 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* ======   Tuning parameters   ====== */
+#define ZSTDMT_NBTHREADS_MAX 128
+
+
+/* ======   Compiler specifics   ====== */
+#if defined(_MSC_VER)
+#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+/* ======   Dependencies   ====== */
+#include <stdlib.h>   /* malloc */
+#include <string.h>   /* memcpy */
+#include "pool.h"     /* threadpool */
+#include "threading.h"  /* mutex */
+#include "zstd_internal.h"   /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
+#include "zstdmt_compress.h"
+
+
+/* ======   Debug   ====== */
+#if 0
+
+#  include <stdio.h>
+#  include <unistd.h>
+#  include <sys/times.h>
+   static unsigned g_debugLevel = 3;
+#  define DEBUGLOGRAW(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __VA_ARGS__); }
+#  define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); }
+
+#  define DEBUG_PRINTHEX(l,p,n) { \
+    unsigned debug_u;                   \
+    for (debug_u=0; debug_u<(n); debug_u++)           \
+        DEBUGLOGRAW(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \
+    DEBUGLOGRAW(l, " \n");       \
+}
+
+static unsigned long long GetCurrentClockTimeMicroseconds()
+{
+   static clock_t _ticksPerSecond = 0;
+   if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK);
+
+   struct tms junk; clock_t newTicks = (clock_t) times(&junk);
+   return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond);
+}
+
+#define MUTEX_WAIT_TIME_DLEVEL 5
+#define PTHREAD_MUTEX_LOCK(mutex) \
+if (g_debugLevel>=MUTEX_WAIT_TIME_DLEVEL) { \
+   unsigned long long beforeTime = GetCurrentClockTimeMicroseconds(); \
+   pthread_mutex_lock(mutex); \
+   unsigned long long afterTime = GetCurrentClockTimeMicroseconds(); \
+   unsigned long long elapsedTime = (afterTime-beforeTime); \
+   if (elapsedTime > 1000) {  /* or whatever threshold you like; I'm using 1 millisecond here */ \
+      DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
+               elapsedTime, #mutex); \
+  } \
+} else pthread_mutex_lock(mutex);
+
+#else
+
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#  define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
+#  define DEBUG_PRINTHEX(l,p,n) {}
+
+#endif
+
+
+/* =====   Buffer Pool   ===== */
+
+typedef struct buffer_s {
+    void* start;
+    size_t size;
+} buffer_t;
+
+static const buffer_t g_nullBuffer = { NULL, 0 };
+
+typedef struct ZSTDMT_bufferPool_s {
+    unsigned totalBuffers;
+    unsigned nbBuffers;
+    buffer_t bTable[1];   /* variable size */
+} ZSTDMT_bufferPool;
+
+static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads)
+{
+    unsigned const maxNbBuffers = 2*nbThreads + 2;
+    ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)calloc(1, sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t));
+    if (bufPool==NULL) return NULL;
+    bufPool->totalBuffers = maxNbBuffers;
+    bufPool->nbBuffers = 0;
+    return bufPool;
+}
+
+static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
+{
+    unsigned u;
+    if (!bufPool) return;   /* compatibility with free on NULL */
+    for (u=0; u<bufPool->totalBuffers; u++)
+        free(bufPool->bTable[u].start);
+    free(bufPool);
+}
+
+/* assumption : invocation from main thread only ! */
+static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
+{
+    if (pool->nbBuffers) {   /* try to use an existing buffer */
+        buffer_t const buf = pool->bTable[--(pool->nbBuffers)];
+        size_t const availBufferSize = buf.size;
+        if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize))   /* large enough, but not too much */
+            return buf;
+        free(buf.start);   /* size conditions not respected : scratch this buffer and create a new one */
+    }
+    /* create new buffer */
+    {   buffer_t buffer;
+        void* const start = malloc(bSize);
+        if (start==NULL) bSize = 0;
+        buffer.start = start;   /* note : start can be NULL if malloc fails ! */
+        buffer.size = bSize;
+        return buffer;
+    }
+}
+
+/* store buffer for later re-use, up to pool capacity */
+static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
+{
+    if (buf.start == NULL) return;   /* release on NULL */
+    if (pool->nbBuffers < pool->totalBuffers) {
+        pool->bTable[pool->nbBuffers++] = buf;   /* store for later re-use */
+        return;
+    }
+    /* Reached bufferPool capacity (should not happen) */
+    free(buf.start);
+}
+
+
+/* =====   CCtx Pool   ===== */
+
+typedef struct {
+    unsigned totalCCtx;
+    unsigned availCCtx;
+    ZSTD_CCtx* cctx[1];   /* variable size */
+} ZSTDMT_CCtxPool;
+
+/* assumption : CCtxPool invocation only from main thread */
+
+/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
+static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
+{
+    unsigned u;
+    for (u=0; u<pool->totalCCtx; u++)
+        ZSTD_freeCCtx(pool->cctx[u]);  /* note : compatible with free on NULL */
+    free(pool);
+}
+
+/* ZSTDMT_createCCtxPool() :
+ * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */
+static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads)
+{
+    ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) calloc(1, sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*));
+    if (!cctxPool) return NULL;
+    cctxPool->totalCCtx = nbThreads;
+    cctxPool->availCCtx = 1;   /* at least one cctx for single-thread mode */
+    cctxPool->cctx[0] = ZSTD_createCCtx();
+    if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
+    DEBUGLOG(1, "cctxPool created, with %u threads", nbThreads);
+    return cctxPool;
+}
+
+static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
+{
+    if (pool->availCCtx) {
+        pool->availCCtx--;
+        return pool->cctx[pool->availCCtx];
+    }
+    return ZSTD_createCCtx();   /* note : can be NULL, when creation fails ! */
+}
+
+static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
+{
+    if (cctx==NULL) return;   /* compatibility with release on NULL */
+    if (pool->availCCtx < pool->totalCCtx)
+        pool->cctx[pool->availCCtx++] = cctx;
+    else
+        /* pool overflow : should not happen, since totalCCtx==nbThreads */
+        ZSTD_freeCCtx(cctx);
+}
+
+
+/* =====   Thread worker   ===== */
+
+typedef struct {
+    buffer_t buffer;
+    size_t filled;
+} inBuff_t;
+
+typedef struct {
+    ZSTD_CCtx* cctx;
+    buffer_t src;
+    const void* srcStart;
+    size_t   srcSize;
+    size_t   dictSize;
+    buffer_t dstBuff;
+    size_t   cSize;
+    size_t   dstFlushed;
+    unsigned firstChunk;
+    unsigned lastChunk;
+    unsigned jobCompleted;
+    unsigned jobScanned;
+    pthread_mutex_t* jobCompleted_mutex;
+    pthread_cond_t* jobCompleted_cond;
+    ZSTD_parameters params;
+    ZSTD_CDict* cdict;
+    unsigned long long fullFrameSize;
+} ZSTDMT_jobDescription;
+
+/* ZSTDMT_compressChunk() : POOL_function type */
+void ZSTDMT_compressChunk(void* jobDescription)
+{
+    ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
+    const void* const src = (const char*)job->srcStart + job->dictSize;
+    buffer_t const dstBuff = job->dstBuff;
+    DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
+    if (job->cdict) {  /* should only happen for first segment */
+        size_t const initError = ZSTD_compressBegin_usingCDict(job->cctx, job->cdict, job->fullFrameSize);
+        if (job->cdict) DEBUGLOG(3, "using CDict ");
+        if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
+    } else {  /* srcStart points at reloaded section */
+        size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1);  /* Force loading dictionary in "content-only" mode (no header analysis) */
+        size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, 0);
+        if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; }
+        ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
+    }
+    if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
+        size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
+        if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
+        ZSTD_invalidateRepCodes(job->cctx);
+    }
+
+    DEBUGLOG(4, "Compressing : ");
+    DEBUG_PRINTHEX(4, job->srcStart, 12);
+    job->cSize = (job->lastChunk) ?
+                 ZSTD_compressEnd     (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
+                 ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
+    DEBUGLOG(3, "compressed %u bytes into %u bytes   (first:%u) (last:%u)", (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
+
+_endJob:
+    PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
+    job->jobCompleted = 1;
+    job->jobScanned = 0;
+    pthread_cond_signal(job->jobCompleted_cond);
+    pthread_mutex_unlock(job->jobCompleted_mutex);
+}
+
+
+/* ------------------------------------------ */
+/* =====   Multi-threaded compression   ===== */
+/* ------------------------------------------ */
+
+struct ZSTDMT_CCtx_s {
+    POOL_ctx* factory;
+    ZSTDMT_bufferPool* buffPool;
+    ZSTDMT_CCtxPool* cctxPool;
+    pthread_mutex_t jobCompleted_mutex;
+    pthread_cond_t jobCompleted_cond;
+    size_t targetSectionSize;
+    size_t marginSize;
+    size_t inBuffSize;
+    size_t dictSize;
+    size_t targetDictSize;
+    inBuff_t inBuff;
+    ZSTD_parameters params;
+    XXH64_state_t xxhState;
+    unsigned nbThreads;
+    unsigned jobIDMask;
+    unsigned doneJobID;
+    unsigned nextJobID;
+    unsigned frameEnded;
+    unsigned allJobsCompleted;
+    unsigned overlapRLog;
+    unsigned long long frameContentSize;
+    size_t sectionSize;
+    ZSTD_CDict* cdict;
+    ZSTD_CStream* cstream;
+    ZSTDMT_jobDescription jobs[1];   /* variable size (must lies at the end) */
+};
+
+ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
+{
+    ZSTDMT_CCtx* cctx;
+    U32 const minNbJobs = nbThreads + 2;
+    U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
+    U32 const nbJobs = 1 << nbJobsLog2;
+    DEBUGLOG(5, "nbThreads : %u  ; minNbJobs : %u ;  nbJobsLog2 : %u ;  nbJobs : %u  \n",
+            nbThreads, minNbJobs, nbJobsLog2, nbJobs);
+    if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
+    cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx) + nbJobs*sizeof(ZSTDMT_jobDescription));
+    if (!cctx) return NULL;
+    cctx->nbThreads = nbThreads;
+    cctx->jobIDMask = nbJobs - 1;
+    cctx->allJobsCompleted = 1;
+    cctx->sectionSize = 0;
+    cctx->overlapRLog = 3;
+    cctx->factory = POOL_create(nbThreads, 1);
+    cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
+    cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
+    if (!cctx->factory | !cctx->buffPool | !cctx->cctxPool) {  /* one object was not created */
+        ZSTDMT_freeCCtx(cctx);
+        return NULL;
+    }
+    if (nbThreads==1) {
+        cctx->cstream = ZSTD_createCStream();
+        if (!cctx->cstream) {
+            ZSTDMT_freeCCtx(cctx); return NULL;
+    }   }
+    pthread_mutex_init(&cctx->jobCompleted_mutex, NULL);   /* Todo : check init function return */
+    pthread_cond_init(&cctx->jobCompleted_cond, NULL);
+    DEBUGLOG(4, "mt_cctx created, for %u threads \n", nbThreads);
+    return cctx;
+}
+
+/* ZSTDMT_releaseAllJobResources() :
+ * Ensure all workers are killed first. */
+static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
+{
+    unsigned jobID;
+    for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
+        ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].dstBuff);
+        mtctx->jobs[jobID].dstBuff = g_nullBuffer;
+        ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].src);
+        mtctx->jobs[jobID].src = g_nullBuffer;
+        ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[jobID].cctx);
+        mtctx->jobs[jobID].cctx = NULL;
+    }
+    memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
+    ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->inBuff.buffer);
+    mtctx->inBuff.buffer = g_nullBuffer;
+    mtctx->allJobsCompleted = 1;
+}
+
+size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
+{
+    if (mtctx==NULL) return 0;   /* compatible with free on NULL */
+    POOL_free(mtctx->factory);
+    if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
+    ZSTDMT_freeBufferPool(mtctx->buffPool);  /* release job resources into pools first */
+    ZSTDMT_freeCCtxPool(mtctx->cctxPool);
+    ZSTD_freeCDict(mtctx->cdict);
+    ZSTD_freeCStream(mtctx->cstream);
+    pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
+    pthread_cond_destroy(&mtctx->jobCompleted_cond);
+    free(mtctx);
+    return 0;
+}
+
+size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
+{
+    switch(parameter)
+    {
+    case ZSTDMT_p_sectionSize :
+        mtctx->sectionSize = value;
+        return 0;
+    case ZSTDMT_p_overlapSectionLog :
+    DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
+        mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
+        return 0;
+    default :
+        return ERROR(compressionParameter_unsupported);
+    }
+}
+
+
+/* ------------------------------------------ */
+/* =====   Multi-threaded compression   ===== */
+/* ------------------------------------------ */
+
+size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
+                           void* dst, size_t dstCapacity,
+                     const void* src, size_t srcSize,
+                           int compressionLevel)
+{
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
+    size_t const chunkTargetSize = (size_t)1 << (params.cParams.windowLog + 2);
+    unsigned const nbChunksMax = (unsigned)(srcSize / chunkTargetSize) + (srcSize < chunkTargetSize) /* min 1 */;
+    unsigned nbChunks = MIN(nbChunksMax, mtctx->nbThreads);
+    size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
+    size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0xFFFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize;   /* avoid too small last block */
+    size_t remainingSrcSize = srcSize;
+    const char* const srcStart = (const char*)src;
+    size_t frameStartPos = 0;
+
+    DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes  ", params.cParams.windowLog, (U32)chunkTargetSize);
+    DEBUGLOG(2, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
+    params.fParams.contentSizeFlag = 1;
+
+    if (nbChunks==1) {   /* fallback to single-thread mode */
+        ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
+        return ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel);
+    }
+
+    {   unsigned u;
+        for (u=0; u<nbChunks; u++) {
+            size_t const chunkSize = MIN(remainingSrcSize, avgChunkSize);
+            size_t const dstBufferCapacity = u ? ZSTD_compressBound(chunkSize) : dstCapacity;
+            buffer_t const dstAsBuffer = { dst, dstCapacity };
+            buffer_t const dstBuffer = u ? ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity) : dstAsBuffer;
+            ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(mtctx->cctxPool);
+
+            if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+                mtctx->jobs[u].cSize = ERROR(memory_allocation);   /* job result */
+                mtctx->jobs[u].jobCompleted = 1;
+                nbChunks = u+1;
+                break;   /* let's wait for previous jobs to complete, but don't start new ones */
+            }
+
+            mtctx->jobs[u].srcStart = srcStart + frameStartPos;
+            mtctx->jobs[u].srcSize = chunkSize;
+            mtctx->jobs[u].fullFrameSize = srcSize;
+            mtctx->jobs[u].params = params;
+            mtctx->jobs[u].dstBuff = dstBuffer;
+            mtctx->jobs[u].cctx = cctx;
+            mtctx->jobs[u].firstChunk = (u==0);
+            mtctx->jobs[u].lastChunk = (u==nbChunks-1);
+            mtctx->jobs[u].jobCompleted = 0;
+            mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex;
+            mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond;
+
+            DEBUGLOG(3, "posting job %u   (%u bytes)", u, (U32)chunkSize);
+            DEBUG_PRINTHEX(3, mtctx->jobs[u].srcStart, 12);
+            POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);
+
+            frameStartPos += chunkSize;
+            remainingSrcSize -= chunkSize;
+    }   }
+    /* note : since nbChunks <= nbThreads, all jobs should be running immediately in parallel */
+
+    {   unsigned chunkID;
+        size_t error = 0, dstPos = 0;
+        for (chunkID=0; chunkID<nbChunks; chunkID++) {
+            DEBUGLOG(3, "waiting for chunk %u ", chunkID);
+            PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
+            while (mtctx->jobs[chunkID].jobCompleted==0) {
+                DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", chunkID);
+                pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
+            }
+            pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
+            DEBUGLOG(3, "ready to write chunk %u ", chunkID);
+
+            ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx);
+            mtctx->jobs[chunkID].cctx = NULL;
+            mtctx->jobs[chunkID].srcStart = NULL;
+            {   size_t const cSize = mtctx->jobs[chunkID].cSize;
+                if (ZSTD_isError(cSize)) error = cSize;
+                if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall);
+                if (chunkID) {   /* note : chunk 0 is already written directly into dst */
+                    if (!error) memcpy((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize);
+                    ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff);
+                    mtctx->jobs[chunkID].dstBuff = g_nullBuffer;
+                }
+                dstPos += cSize ;
+            }
+        }
+        if (!error) DEBUGLOG(3, "compressed size : %u  ", (U32)dstPos);
+        return error ? error : dstPos;
+    }
+
+}
+
+
+/* ====================================== */
+/* =======      Streaming API     ======= */
+/* ====================================== */
+
+static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) {
+    while (zcs->doneJobID < zcs->nextJobID) {
+        unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
+        PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+        while (zcs->jobs[jobID].jobCompleted==0) {
+            DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID);   /* we want to block when waiting for data to flush */
+            pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
+        }
+        pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+        zcs->doneJobID++;
+    }
+}
+
+
+static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
+                                    const void* dict, size_t dictSize, unsigned updateDict,
+                                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+    ZSTD_customMem const cmem = { NULL, NULL, NULL };
+    DEBUGLOG(3, "Started new compression, with windowLog : %u", params.cParams.windowLog);
+    if (zcs->nbThreads==1) return ZSTD_initCStream_advanced(zcs->cstream, dict, dictSize, params, pledgedSrcSize);
+    if (zcs->allJobsCompleted == 0) {   /* previous job not correctly finished */
+        ZSTDMT_waitForAllJobsCompleted(zcs);
+        ZSTDMT_releaseAllJobResources(zcs);
+        zcs->allJobsCompleted = 1;
+    }
+    zcs->params = params;
+    if (updateDict) {
+        ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
+        if (dict && dictSize) {
+            zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params, cmem);
+            if (zcs->cdict == NULL) return ERROR(memory_allocation);
+    }   }
+    zcs->frameContentSize = pledgedSrcSize;
+    zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog);
+    DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog);
+    DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
+    zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
+    zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
+    zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
+    DEBUGLOG(3, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
+    zcs->marginSize = zcs->targetSectionSize >> 2;
+    zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
+    zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+    if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
+    zcs->inBuff.filled = 0;
+    zcs->dictSize = 0;
+    zcs->doneJobID = 0;
+    zcs->nextJobID = 0;
+    zcs->frameEnded = 0;
+    zcs->allJobsCompleted = 0;
+    if (params.fParams.checksumFlag) XXH64_reset(&zcs->xxhState, 0);
+    return 0;
+}
+
+size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* zcs,
+                                const void* dict, size_t dictSize,
+                                ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+    return ZSTDMT_initCStream_internal(zcs, dict, dictSize, 1, params, pledgedSrcSize);
+}
+
+/* ZSTDMT_resetCStream() :
+ * pledgedSrcSize is optional and can be zero == unknown */
+size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
+{
+    if (zcs->nbThreads==1) return ZSTD_resetCStream(zcs->cstream, pledgedSrcSize);
+    return ZSTDMT_initCStream_internal(zcs, NULL, 0, 0, zcs->params, pledgedSrcSize);
+}
+
+size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
+    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0);
+    return ZSTDMT_initCStream_internal(zcs, NULL, 0, 1, params, 0);
+}
+
+
+static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsigned endFrame)
+{
+    size_t const dstBufferCapacity = ZSTD_compressBound(srcSize);
+    buffer_t const dstBuffer = ZSTDMT_getBuffer(zcs->buffPool, dstBufferCapacity);
+    ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(zcs->cctxPool);
+    unsigned const jobID = zcs->nextJobID & zcs->jobIDMask;
+
+    if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+        zcs->jobs[jobID].jobCompleted = 1;
+        zcs->nextJobID++;
+        ZSTDMT_waitForAllJobsCompleted(zcs);
+        ZSTDMT_releaseAllJobResources(zcs);
+        return ERROR(memory_allocation);
+    }
+
+    DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
+    zcs->jobs[jobID].src = zcs->inBuff.buffer;
+    zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
+    zcs->jobs[jobID].srcSize = srcSize;
+    zcs->jobs[jobID].dictSize = zcs->dictSize;   /* note : zcs->inBuff.filled is presumed >= srcSize + dictSize */
+    zcs->jobs[jobID].params = zcs->params;
+    if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0;  /* do not calculate checksum within sections, just keep it in header for first section */
+    zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL;
+    zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize;
+    zcs->jobs[jobID].dstBuff = dstBuffer;
+    zcs->jobs[jobID].cctx = cctx;
+    zcs->jobs[jobID].firstChunk = (zcs->nextJobID==0);
+    zcs->jobs[jobID].lastChunk = endFrame;
+    zcs->jobs[jobID].jobCompleted = 0;
+    zcs->jobs[jobID].dstFlushed = 0;
+    zcs->jobs[jobID].jobCompleted_mutex = &zcs->jobCompleted_mutex;
+    zcs->jobs[jobID].jobCompleted_cond = &zcs->jobCompleted_cond;
+
+    /* get a new buffer for next input */
+    if (!endFrame) {
+        size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
+        zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+        if (zcs->inBuff.buffer.start == NULL) {   /* not enough memory to allocate next input buffer */
+            zcs->jobs[jobID].jobCompleted = 1;
+            zcs->nextJobID++;
+            ZSTDMT_waitForAllJobsCompleted(zcs);
+            ZSTDMT_releaseAllJobResources(zcs);
+            return ERROR(memory_allocation);
+        }
+        DEBUGLOG(5, "inBuff filled to %u", (U32)zcs->inBuff.filled);
+        zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize;
+        DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src", (U32)zcs->inBuff.filled, (U32)newDictSize, (U32)(zcs->inBuff.filled - newDictSize));
+        memmove(zcs->inBuff.buffer.start, (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, zcs->inBuff.filled);
+        DEBUGLOG(5, "new inBuff pre-filled");
+        zcs->dictSize = newDictSize;
+    } else {
+        zcs->inBuff.buffer = g_nullBuffer;
+        zcs->inBuff.filled = 0;
+        zcs->dictSize = 0;
+        zcs->frameEnded = 1;
+        if (zcs->nextJobID == 0)
+            zcs->params.fParams.checksumFlag = 0;   /* single chunk : checksum is calculated directly within worker thread */
+    }
+
+    DEBUGLOG(3, "posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)", zcs->nextJobID, (U32)zcs->jobs[jobID].srcSize, zcs->jobs[jobID].lastChunk, zcs->doneJobID, zcs->doneJobID & zcs->jobIDMask);
+    POOL_add(zcs->factory, ZSTDMT_compressChunk, &zcs->jobs[jobID]);   /* this call is blocking when thread worker pool is exhausted */
+    zcs->nextJobID++;
+    return 0;
+}
+
+
+/* ZSTDMT_flushNextJob() :
+ * output : will be updated with amount of data flushed .
+ * blockToFlush : if >0, the function will block and wait if there is no data available to flush .
+ * @return : amount of data remaining within internal buffer, 1 if unknown but > 0, 0 if no more, or an error code */
+static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush)
+{
+    unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
+    if (zcs->doneJobID == zcs->nextJobID) return 0;   /* all flushed ! */
+    PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+    while (zcs->jobs[wJobID].jobCompleted==0) {
+        DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID);
+        if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; }  /* nothing ready to be flushed => skip */
+        pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);  /* block when nothing available to flush */
+    }
+    pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+    /* compression job completed : output can be flushed */
+    {   ZSTDMT_jobDescription job = zcs->jobs[wJobID];
+        if (!job.jobScanned) {
+            if (ZSTD_isError(job.cSize)) {
+                DEBUGLOG(5, "compression error detected ");
+                ZSTDMT_waitForAllJobsCompleted(zcs);
+                ZSTDMT_releaseAllJobResources(zcs);
+                return job.cSize;
+            }
+            ZSTDMT_releaseCCtx(zcs->cctxPool, job.cctx);
+            zcs->jobs[wJobID].cctx = NULL;
+            DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag);
+            if (zcs->params.fParams.checksumFlag) {
+                XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
+                if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) {  /* write checksum at end of last section */
+                    U32 const checksum = (U32)XXH64_digest(&zcs->xxhState);
+                    DEBUGLOG(4, "writing checksum : %08X \n", checksum);
+                    MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum);
+                    job.cSize += 4;
+                    zcs->jobs[wJobID].cSize += 4;
+            }   }
+            ZSTDMT_releaseBuffer(zcs->buffPool, job.src);
+            zcs->jobs[wJobID].srcStart = NULL;
+            zcs->jobs[wJobID].src = g_nullBuffer;
+            zcs->jobs[wJobID].jobScanned = 1;
+        }
+        {   size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
+            DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
+            memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
+            output->pos += toWrite;
+            job.dstFlushed += toWrite;
+        }
+        if (job.dstFlushed == job.cSize) {   /* output buffer fully flushed => move to next one */
+            ZSTDMT_releaseBuffer(zcs->buffPool, job.dstBuff);
+            zcs->jobs[wJobID].dstBuff = g_nullBuffer;
+            zcs->jobs[wJobID].jobCompleted = 0;
+            zcs->doneJobID++;
+        } else {
+            zcs->jobs[wJobID].dstFlushed = job.dstFlushed;
+        }
+        /* return value : how many bytes left in buffer ; fake it to 1 if unknown but >0 */
+        if (job.cSize > job.dstFlushed) return (job.cSize - job.dstFlushed);
+        if (zcs->doneJobID < zcs->nextJobID) return 1;   /* still some buffer to flush */
+        zcs->allJobsCompleted = zcs->frameEnded;   /* frame completed and entirely flushed */
+        return 0;   /* everything flushed */
+}   }
+
+
+size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+    size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize;
+    if (zcs->frameEnded) return ERROR(stage_wrong);   /* current frame being ended. Only flush is allowed. Restart with init */
+    if (zcs->nbThreads==1) return ZSTD_compressStream(zcs->cstream, output, input);
+
+    /* fill input buffer */
+    {   size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled);
+        memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, input->src, toLoad);
+        input->pos += toLoad;
+        zcs->inBuff.filled += toLoad;
+    }
+
+    if ( (zcs->inBuff.filled >= newJobThreshold)  /* filled enough : let's compress */
+        && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {   /* avoid overwriting job round buffer */
+        CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0) );
+    }
+
+    /* check for data to flush */
+    CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize)) ); /* block if it wasn't possible to create new job due to saturation */
+
+    /* recommended next input size : fill current input buffer */
+    return zcs->inBuffSize - zcs->inBuff.filled;   /* note : could be zero when input buffer is fully filled and no more availability to create new job */
+}
+
+
+static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned endFrame)
+{
+    size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
+
+    if (srcSize) DEBUGLOG(4, "flushing : %u bytes left to compress", (U32)srcSize);
+    if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
+       && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
+        CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) );
+    }
+
+    /* check if there is any data available to flush */
+    DEBUGLOG(5, "zcs->doneJobID : %u  ; zcs->nextJobID : %u ", zcs->doneJobID, zcs->nextJobID);
+    return ZSTDMT_flushNextJob(zcs, output, 1);
+}
+
+
+size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+{
+    if (zcs->nbThreads==1) return ZSTD_flushStream(zcs->cstream, output);
+    return ZSTDMT_flushStream_internal(zcs, output, 0);
+}
+
+size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+{
+    if (zcs->nbThreads==1) return ZSTD_endStream(zcs->cstream, output);
+    return ZSTDMT_flushStream_internal(zcs, output, 1);
+}

lib/compress/zstdmt_compress.h

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+ #ifndef ZSTDMT_COMPRESS_H
+ #define ZSTDMT_COMPRESS_H
+
+ #if defined (__cplusplus)
+ extern "C" {
+ #endif
+
+
+/* Note : All prototypes defined in this file shall be considered experimental.
+ *        There is no guarantee of API continuity (yet) on any of these prototypes */
+
+/* ===   Dependencies   === */
+#include <stddef.h>   /* size_t */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_parameters */
+#include "zstd.h"     /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
+
+
+/* ===   Simple one-pass functions   === */
+
+typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
+ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
+ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
+
+ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
+                           void* dst, size_t dstCapacity,
+                     const void* src, size_t srcSize,
+                           int compressionLevel);
+
+
+/* ===   Streaming functions   === */
+
+ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
+ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);    /**< pledgedSrcSize is optional and can be zero == unknown */
+
+ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);   /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);     /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+
+
+/* ===   Advanced functions and parameters  === */
+
+#ifndef ZSTDMT_SECTION_SIZE_MIN
+#  define ZSTDMT_SECTION_SIZE_MIN (1U << 20)   /* 1 MB - Minimum size of each compression job */
+#endif
+
+ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize,  /**< dict can be released after init, a local copy is preserved within zcs */
+                                          ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+
+/* ZSDTMT_parameter :
+ * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
+typedef enum {
+    ZSTDMT_p_sectionSize,        /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
+    ZSTDMT_p_overlapSectionLog   /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
+} ZSDTMT_parameter;
+
+/* ZSTDMT_setMTCtxParameter() :
+ * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
+ * The function must be called typically after ZSTD_createCCtx().
+ * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif   /* ZSTDMT_COMPRESS_H */

lib/decompress/huf_decompress.c

@@ -35,16 +35,19 @@
 /* **************************************************************
 *  Compiler specifics
 ****************************************************************/
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-/* inline is defined */
-#elif defined(_MSC_VER) || defined(__GNUC__)
-#  define inline __inline
-#else
-#  define inline /* disable inline */
-#endif
-
 #ifdef _MSC_VER    /* Visual Studio */
+#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#else
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -102,16 +105,16 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
 
     /* Table header */
     {   DTableDesc dtd = HUF_getDTableDesc(DTable);
-        if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge);   /* DTable too small, huffman tree cannot fit in */
+        if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge);   /* DTable too small, Huffman tree cannot fit in */
         dtd.tableType = 0;
         dtd.tableLog = (BYTE)tableLog;
         memcpy(DTable, &dtd, sizeof(dtd));
     }
 
-    /* Prepare ranks */
+    /* Calculate starting value for each rank */
     {   U32 n, nextRankStart = 0;
         for (n=1; n<tableLog+1; n++) {
-            U32 current = nextRankStart;
+            U32 const current = nextRankStart;
             nextRankStart += (rankVal[n] << (n-1));
             rankVal[n] = current;
     }   }
@@ -121,11 +124,11 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
         for (n=0; n<nbSymbols; n++) {
             U32 const w = huffWeight[n];
             U32 const length = (1 << w) >> 1;
-            U32 i;
+            U32 u;
             HUF_DEltX2 D;
             D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
-            for (i = rankVal[w]; i < rankVal[w] + length; i++)
-                dt[i] = D;
+            for (u = rankVal[w]; u < rankVal[w] + length; u++)
+                dt[u] = D;
             rankVal[w] += length;
     }   }
 
@@ -152,7 +155,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
     if (MEM_64bits()) \
         HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
 
-static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
+FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
 {
     BYTE* const pStart = p;
 
@@ -459,7 +462,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
     void* dtPtr = DTable+1;   /* force compiler to avoid strict-aliasing */
     HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
 
-    HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable));   /* if compilation fails here, assertion is false */
+    HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable));   /* if compiler fails here, assertion is wrong */
     if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
     /* memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
 
@@ -559,7 +562,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
     if (MEM_64bits()) \
         ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
 
-static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
+FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
 {
     BYTE* const pStart = p;
 

lib/decompress/zstd_decompress.c

@@ -43,8 +43,6 @@
 *********************************************************/
 #include <string.h>      /* memcpy, memmove, memset */
 #include "mem.h"         /* low level memory routines */
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"      /* XXH64_* */
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
@@ -87,22 +85,26 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
                ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
                ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
 
+typedef struct {
+    FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
+    FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+    FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+    HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
+    U32 rep[ZSTD_REP_NUM];
+} ZSTD_entropyTables_t;
+
 struct ZSTD_DCtx_s
 {
     const FSE_DTable* LLTptr;
     const FSE_DTable* MLTptr;
     const FSE_DTable* OFTptr;
     const HUF_DTable* HUFptr;
-    FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
-    FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
-    FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
-    HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
-    const void* previousDstEnd;
-    const void* base;
-    const void* vBase;
-    const void* dictEnd;
+    ZSTD_entropyTables_t entropy;
+    const void* previousDstEnd;   /* detect continuity */
+    const void* base;             /* start of current segment */
+    const void* vBase;            /* virtual start of previous segment if it was just before current one */
+    const void* dictEnd;          /* end of previous segment */
     size_t expected;
-    U32 rep[ZSTD_REP_NUM];
     ZSTD_frameParams fParams;
     blockType_e bType;   /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
     ZSTD_dStage stage;
@@ -131,15 +133,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     dctx->base = NULL;
     dctx->vBase = NULL;
     dctx->dictEnd = NULL;
-    dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
+    dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
     dctx->litEntropy = dctx->fseEntropy = 0;
     dctx->dictID = 0;
-    MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue));
-    memcpy(dctx->rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
-    dctx->LLTptr = dctx->LLTable;
-    dctx->MLTptr = dctx->MLTable;
-    dctx->OFTptr = dctx->OFTable;
-    dctx->HUFptr = dctx->hufTable;
+    MEM_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
+    memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
+    dctx->LLTptr = dctx->entropy.LLTable;
+    dctx->MLTptr = dctx->entropy.MLTable;
+    dctx->OFTptr = dctx->entropy.OFTable;
+    dctx->HUFptr = dctx->entropy.hufTable;
     return 0;
 }
 
@@ -175,6 +177,8 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
     memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize);  /* no need to copy workspace */
 }
 
+#if 0
+/* deprecated */
 static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 {
     ZSTD_decompressBegin(dstDCtx);  /* init */
@@ -186,15 +190,18 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
         dstDCtx->dictID = srcDCtx->dictID;
         dstDCtx->litEntropy = srcDCtx->litEntropy;
         dstDCtx->fseEntropy = srcDCtx->fseEntropy;
-        dstDCtx->LLTptr = srcDCtx->LLTable;
-        dstDCtx->MLTptr = srcDCtx->MLTable;
-        dstDCtx->OFTptr = srcDCtx->OFTable;
-        dstDCtx->HUFptr = srcDCtx->hufTable;
-        dstDCtx->rep[0] = srcDCtx->rep[0];
-        dstDCtx->rep[1] = srcDCtx->rep[1];
-        dstDCtx->rep[2] = srcDCtx->rep[2];
+        dstDCtx->LLTptr = srcDCtx->entropy.LLTable;
+        dstDCtx->MLTptr = srcDCtx->entropy.MLTable;
+        dstDCtx->OFTptr = srcDCtx->entropy.OFTable;
+        dstDCtx->HUFptr = srcDCtx->entropy.hufTable;
+        dstDCtx->entropy.rep[0] = srcDCtx->entropy.rep[0];
+        dstDCtx->entropy.rep[1] = srcDCtx->entropy.rep[1];
+        dstDCtx->entropy.rep[2] = srcDCtx->entropy.rep[2];
     }
 }
+#endif
+
+static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict);
 
 
 /*-*************************************************************
@@ -306,6 +313,86 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
     return 0;
 }
 
+/** ZSTD_getFrameContentSize() :
+*   compatible with legacy mode
+*   @return : decompressed size of the single frame pointed to be `src` if known, otherwise
+*             - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+*             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
+{
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (ZSTD_isLegacy(src, srcSize)) {
+        unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
+        return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
+    }
+#endif
+    {
+        ZSTD_frameParams fParams;
+        if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR;
+        if (fParams.windowSize == 0) {
+            /* Either skippable or empty frame, size == 0 either way */
+            return 0;
+        } else if (fParams.frameContentSize != 0) {
+            return fParams.frameContentSize;
+        } else {
+            return ZSTD_CONTENTSIZE_UNKNOWN;
+        }
+    }
+}
+
+/** ZSTD_findDecompressedSize() :
+ *  compatible with legacy mode
+ *  `srcSize` must be the exact length of some number of ZSTD compressed and/or
+ *      skippable frames
+ *  @return : decompressed size of the frames contained */
+unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
+{
+    {
+        unsigned long long totalDstSize = 0;
+        while (srcSize >= ZSTD_frameHeaderSize_prefix) {
+            const U32 magicNumber = MEM_readLE32(src);
+
+            if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+                size_t skippableSize;
+                if (srcSize < ZSTD_skippableHeaderSize)
+                    return ERROR(srcSize_wrong);
+                skippableSize = MEM_readLE32((const BYTE *)src + 4) +
+                                ZSTD_skippableHeaderSize;
+                if (srcSize < skippableSize) {
+                    return ZSTD_CONTENTSIZE_ERROR;
+                }
+
+                src = (const BYTE *)src + skippableSize;
+                srcSize -= skippableSize;
+                continue;
+            }
+
+            {
+                unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+                if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
+
+                /* check for overflow */
+                if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
+                totalDstSize += ret;
+            }
+            {
+                size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
+                if (ZSTD_isError(frameSrcSize)) {
+                    return ZSTD_CONTENTSIZE_ERROR;
+                }
+
+                src = (const BYTE *)src + frameSrcSize;
+                srcSize -= frameSrcSize;
+            }
+        }
+
+        if (srcSize) {
+            return ZSTD_CONTENTSIZE_ERROR;
+        }
+
+        return totalDstSize;
+    }
+}
 
 /** ZSTD_getDecompressedSize() :
 *   compatible with legacy mode
@@ -316,14 +403,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
                    - frame header not complete (`srcSize` too small) */
 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
 {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize);
-#endif
-    {   ZSTD_frameParams fparams;
-        size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize);
-        if (frResult!=0) return 0;
-        return fparams.frameContentSize;
-    }
+    unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+    return ret >= ZSTD_CONTENTSIZE_ERROR ? 0 : ret;
 }
 
 
@@ -432,14 +513,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                                         HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
                                         HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
                                     ( singleStream ?
-                                        HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
-                                        HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
+                                        HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
+                                        HUF_decompress4X_hufOnly (dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
                     return ERROR(corruption_detected);
 
                 dctx->litPtr = dctx->litBuffer;
                 dctx->litSize = litSize;
                 dctx->litEntropy = 1;
-                if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
+                if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
                 memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
                 return litCSize + lhSize;
             }
@@ -756,19 +837,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
         ip++;
 
         /* Build DTables */
-        {   size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, &dctx->LLTptr,
+        {   size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
                                                       LLtype, MaxLL, LLFSELog,
                                                       ip, iend-ip, LL_defaultDTable, dctx->fseEntropy);
             if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
             ip += llhSize;
         }
-        {   size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, &dctx->OFTptr,
+        {   size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
                                                       OFtype, MaxOff, OffFSELog,
                                                       ip, iend-ip, OF_defaultDTable, dctx->fseEntropy);
             if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
             ip += ofhSize;
         }
-        {   size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, &dctx->MLTptr,
+        {   size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
                                                       MLtype, MaxML, MLFSELog,
                                                       ip, iend-ip, ML_defaultDTable, dctx->fseEntropy);
             if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
@@ -795,7 +876,7 @@ typedef struct {
     size_t prevOffset[ZSTD_REP_NUM];
     const BYTE* base;
     size_t pos;
-    iPtrDiff gotoDict;
+    uPtrDiff gotoDict;
 } seqState_t;
 
 
@@ -925,9 +1006,9 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
 
 FORCE_INLINE
 size_t ZSTD_execSequence(BYTE* op,
-                                BYTE* const oend, seq_t sequence,
-                                const BYTE** litPtr, const BYTE* const litLimit,
-                                const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
+                         BYTE* const oend, seq_t sequence,
+                         const BYTE** litPtr, const BYTE* const litLimit,
+                         const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
 {
     BYTE* const oLitEnd = op + sequence.litLength;
     size_t const sequenceLength = sequence.litLength + sequence.matchLength;
@@ -952,7 +1033,7 @@ size_t ZSTD_execSequence(BYTE* op,
     if (sequence.offset > (size_t)(oLitEnd - base)) {
         /* offset beyond prefix */
         if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
-        match += (dictEnd-base);
+        match = dictEnd + (match - base);
         if (match + sequence.matchLength <= dictEnd) {
             memmove(oLitEnd, match, sequence.matchLength);
             return sequenceLength;
@@ -975,7 +1056,7 @@ size_t ZSTD_execSequence(BYTE* op,
     if (sequence.offset < 8) {
         /* close range match, overlap */
         static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
-        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* substracted */
+        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* subtracted */
         int const sub2 = dec64table[sequence.offset];
         op[0] = match[0];
         op[1] = match[1];
@@ -1030,7 +1111,7 @@ static size_t ZSTD_decompressSequences(
     if (nbSeq) {
         seqState_t seqState;
         dctx->fseEntropy = 1;
-        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
         CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
         FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
         FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1047,7 +1128,7 @@ static size_t ZSTD_decompressSequences(
         /* check if reached exact end */
         if (nbSeq) return ERROR(corruption_detected);
         /* save reps for next block */
-        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
     }
 
     /* last literal segment */
@@ -1061,7 +1142,7 @@ static size_t ZSTD_decompressSequences(
 }
 
 
-static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
+FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int const longOffsets)
 {
     seq_t seq;
 
@@ -1096,8 +1177,15 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
         if (!ofCode)
             offset = 0;
         else {
-            offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
-            if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+            if (longOffsets) {
+                int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
+                offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
+                if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream);
+                if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+            } else {
+                offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
+                if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+            }
         }
 
         if (ofCode <= 1) {
@@ -1141,6 +1229,14 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
     return seq;
 }
 
+static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const windowSize) {
+    if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
+        return ZSTD_decodeSequenceLong_generic(seqState, 1);
+    } else {
+        return ZSTD_decodeSequenceLong_generic(seqState, 0);
+    }
+}
+
 FORCE_INLINE
 size_t ZSTD_execSequenceLong(BYTE* op,
                                 BYTE* const oend, seq_t sequence,
@@ -1196,7 +1292,7 @@ size_t ZSTD_execSequenceLong(BYTE* op,
     if (sequence.offset < 8) {
         /* close range match, overlap */
         static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
-        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* substracted */
+        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* subtracted */
         int const sub2 = dec64table[sequence.offset];
         op[0] = match[0];
         op[1] = match[1];
@@ -1238,6 +1334,7 @@ static size_t ZSTD_decompressSequencesLong(
     const BYTE* const base = (const BYTE*) (dctx->base);
     const BYTE* const vBase = (const BYTE*) (dctx->vBase);
     const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+    unsigned const windowSize = dctx->fParams.windowSize;
     int nbSeq;
 
     /* Build Decoding Tables */
@@ -1256,10 +1353,10 @@ static size_t ZSTD_decompressSequencesLong(
         seqState_t seqState;
         int seqNb;
         dctx->fseEntropy = 1;
-        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
         seqState.base = base;
         seqState.pos = (size_t)(op-base);
-        seqState.gotoDict = (iPtrDiff)(dictEnd - base);
+        seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
         CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
         FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
         FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1267,13 +1364,13 @@ static size_t ZSTD_decompressSequencesLong(
 
         /* prepare in advance */
         for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb<seqAdvance; seqNb++) {
-            sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState);
+            sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
         }
         if (seqNb<seqAdvance) return ERROR(corruption_detected);
 
         /* decode and decompress */
         for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
-            seq_t const sequence = ZSTD_decodeSequenceLong(&seqState);
+            seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
             size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
             if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
             ZSTD_PREFETCH(sequence.match);
@@ -1291,7 +1388,7 @@ static size_t ZSTD_decompressSequencesLong(
         }
 
         /* save reps for next block */
-        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
     }
 
     /* last literal segment */
@@ -1313,13 +1410,18 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
     if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);
 
-    /* Decode literals sub-block */
+    /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
         if (ZSTD_isError(litCSize)) return litCSize;
         ip += litCSize;
         srcSize -= litCSize;
     }
-    if (dctx->fParams.windowSize > (1<<23)) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
+    if (sizeof(size_t) > 4)  /* do not enable prefetching on 32-bits x86, as it's performance detrimental */
+                             /* likely because of register pressure */
+                             /* if that's the correct cause, then 32-bits ARM should be affected differently */
+                             /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */
+        if (dctx->fParams.windowSize > (1<<23))
+            return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
     return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
 }
 
@@ -1363,27 +1465,81 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
     return length;
 }
 
+/** ZSTD_findFrameCompressedSize() :
+ *  compatible with legacy mode
+ *  `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
+ *  `srcSize` must be at least as large as the frame contained
+ *  @return : the compressed size of the frame starting at `src` */
+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
+#endif
+    if (srcSize >= ZSTD_skippableHeaderSize &&
+            (MEM_readLE32(src) & 0xFFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+        return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + 4);
+    } else {
+        const BYTE* ip = (const BYTE*)src;
+        const BYTE* const ipstart = ip;
+        size_t remainingSize = srcSize;
+        ZSTD_frameParams fParams;
+
+        size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
+        if (ZSTD_isError(headerSize)) return headerSize;
+
+        /* Frame Header */
+        {   size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
+            if (ZSTD_isError(ret)) return ret;
+            if (ret > 0) return ERROR(srcSize_wrong);
+        }
+
+        ip += headerSize;
+        remainingSize -= headerSize;
+
+        /* Loop on each block */
+        while (1) {
+            blockProperties_t blockProperties;
+            size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+            if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+            if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+            ip += ZSTD_blockHeaderSize + cBlockSize;
+            remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
+
+            if (blockProperties.lastBlock) break;
+        }
+
+        if (fParams.checksumFlag) {   /* Frame content checksum */
+            if (remainingSize < 4) return ERROR(srcSize_wrong);
+            ip += 4;
+            remainingSize -= 4;
+        }
+
+        return ip - ipstart;
+    }
+}
 
 /*! ZSTD_decompressFrame() :
-*   `dctx` must be properly initialized */
+*   @dctx must be properly initialized */
 static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
                                  void* dst, size_t dstCapacity,
-                                 const void* src, size_t srcSize)
+                                 const void** srcPtr, size_t *srcSizePtr)
 {
-    const BYTE* ip = (const BYTE*)src;
+    const BYTE* ip = (const BYTE*)(*srcPtr);
     BYTE* const ostart = (BYTE* const)dst;
     BYTE* const oend = ostart + dstCapacity;
     BYTE* op = ostart;
-    size_t remainingSize = srcSize;
+    size_t remainingSize = *srcSizePtr;
 
     /* check */
-    if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
 
     /* Frame Header */
-    {   size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
+    {   size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
         if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
-        if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
-        CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize));
+        if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+        CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize));
         ip += frameHeaderSize; remainingSize -= frameHeaderSize;
     }
 
@@ -1428,25 +1584,109 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         if (remainingSize<4) return ERROR(checksum_wrong);
         checkRead = MEM_readLE32(ip);
         if (checkRead != checkCalc) return ERROR(checksum_wrong);
+        ip += 4;
         remainingSize -= 4;
     }
 
-    if (remainingSize) return ERROR(srcSize_wrong);
+    /* Allow caller to get size read */
+    *srcPtr = ip;
+    *srcSizePtr = remainingSize;
     return op-ostart;
 }
 
+static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict);
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
+
+static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
+                                        void* dst, size_t dstCapacity,
+                                  const void* src, size_t srcSize,
+                                  const void *dict, size_t dictSize,
+                                  const ZSTD_DDict* ddict)
+{
+    void* const dststart = dst;
+
+    if (ddict) {
+        if (dict) {
+            /* programmer error, these two cases should be mutually exclusive */
+            return ERROR(GENERIC);
+        }
+
+        dict = ZSTD_DDictDictContent(ddict);
+        dictSize = ZSTD_DDictDictSize(ddict);
+    }
+
+    while (srcSize >= ZSTD_frameHeaderSize_prefix) {
+        U32 magicNumber;
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+        if (ZSTD_isLegacy(src, srcSize)) {
+            size_t decodedSize;
+            size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
+            if (ZSTD_isError(frameSize)) return frameSize;
+
+            decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
+
+            dst = (BYTE*)dst + decodedSize;
+            dstCapacity -= decodedSize;
+
+            src = (const BYTE*)src + frameSize;
+            srcSize -= frameSize;
+
+            continue;
+        }
+#endif
+
+        magicNumber = MEM_readLE32(src);
+        if (magicNumber != ZSTD_MAGICNUMBER) {
+            if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+                size_t skippableSize;
+                if (srcSize < ZSTD_skippableHeaderSize)
+                    return ERROR(srcSize_wrong);
+                skippableSize = MEM_readLE32((const BYTE *)src + 4) +
+                                ZSTD_skippableHeaderSize;
+                if (srcSize < skippableSize) {
+                    return ERROR(srcSize_wrong);
+                }
+
+                src = (const BYTE *)src + skippableSize;
+                srcSize -= skippableSize;
+                continue;
+            } else {
+                return ERROR(prefix_unknown);
+            }
+        }
+
+        if (ddict) {
+            /* we were called from ZSTD_decompress_usingDDict */
+            ZSTD_refDDict(dctx, ddict);
+        } else {
+            /* this will initialize correctly with no dict if dict == NULL, so
+             * use this in all cases but ddict */
+            CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
+        }
+        ZSTD_checkContinuity(dctx, dst);
+
+        {   const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
+                                                    &src, &srcSize);
+            if (ZSTD_isError(res)) return res;
+            /* don't need to bounds check this, ZSTD_decompressFrame will have
+             * already */
+            dst = (BYTE*)dst + res;
+            dstCapacity -= res;
+        }
+    }
+
+    if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */
+
+    return (BYTE*)dst - (BYTE*)dststart;
+}
 
 size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                  void* dst, size_t dstCapacity,
                            const void* src, size_t srcSize,
                            const void* dict, size_t dictSize)
 {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
-#endif
-    ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
-    ZSTD_checkContinuity(dctx, dst);
-    return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
+    return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
 }
 
 
@@ -1633,22 +1873,29 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
     return 0;
 }
 
-static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
+/* ZSTD_loadEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary
+ * @return : size of entropy tables read */
+static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const dict, size_t const dictSize)
 {
     const BYTE* dictPtr = (const BYTE*)dict;
     const BYTE* const dictEnd = dictPtr + dictSize;
 
-    {   size_t const hSize = HUF_readDTableX4(dctx->hufTable, dict, dictSize);
+    if (dictSize <= 8) return ERROR(dictionary_corrupted);
+    dictPtr += 8;   /* skip header = magic + dictID */
+
+
+    {   size_t const hSize = HUF_readDTableX4(entropy->hufTable, dictPtr, dictEnd-dictPtr);
         if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
         dictPtr += hSize;
     }
 
     {   short offcodeNCount[MaxOff+1];
-        U32 offcodeMaxValue=MaxOff, offcodeLog;
+        U32 offcodeMaxValue = MaxOff, offcodeLog;
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
 
@@ -1657,7 +1904,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
         dictPtr += matchlengthHeaderSize;
     }
 
@@ -1666,17 +1913,19 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
         dictPtr += litlengthHeaderSize;
     }
 
     if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
-    dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
-    dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
-    dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
-    dictPtr += 12;
+    {   int i;
+        size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
+        for (i=0; i<3; i++) {
+            U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
+            if (rep==0 || rep >= dictContentSize) return ERROR(dictionary_corrupted);
+            entropy->rep[i] = rep;
+    }   }
 
-    dctx->litEntropy = dctx->fseEntropy = 1;
     return dictPtr - (const BYTE*)dict;
 }
 
@@ -1690,13 +1939,12 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
     dctx->dictID = MEM_readLE32((const char*)dict + 4);
 
     /* load entropy tables */
-    dict = (const char*)dict + 8;
-    dictSize -= 8;
-    {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+    {   size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
         if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
         dict = (const char*)dict + eSize;
         dictSize -= eSize;
     }
+    dctx->litEntropy = dctx->fseEntropy = 1;
 
     /* reference dictionary content */
     return ZSTD_refDictContent(dctx, dict, dictSize);
@@ -1713,60 +1961,127 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
 /* ======   ZSTD_DDict   ====== */
 
 struct ZSTD_DDict_s {
-    void* dict;
+    void* dictBuffer;
+    const void* dictContent;
     size_t dictSize;
-    ZSTD_DCtx* refContext;
+    ZSTD_entropyTables_t entropy;
+    U32 dictID;
+    U32 entropyPresent;
+    ZSTD_customMem cMem;
 };  /* typedef'd to ZSTD_DDict within "zstd.h" */
 
-ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
+static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict)
+{
+    return ddict->dictContent;
+}
+
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
+{
+    return ddict->dictSize;
+}
+
+static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
+{
+    ZSTD_decompressBegin(dstDCtx);  /* init */
+    if (ddict) {   /* support refDDict on NULL */
+        dstDCtx->dictID = ddict->dictID;
+        dstDCtx->base = ddict->dictContent;
+        dstDCtx->vBase = ddict->dictContent;
+        dstDCtx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
+        dstDCtx->previousDstEnd = dstDCtx->dictEnd;
+        if (ddict->entropyPresent) {
+            dstDCtx->litEntropy = 1;
+            dstDCtx->fseEntropy = 1;
+            dstDCtx->LLTptr = ddict->entropy.LLTable;
+            dstDCtx->MLTptr = ddict->entropy.MLTable;
+            dstDCtx->OFTptr = ddict->entropy.OFTable;
+            dstDCtx->HUFptr = ddict->entropy.hufTable;
+            dstDCtx->entropy.rep[0] = ddict->entropy.rep[0];
+            dstDCtx->entropy.rep[1] = ddict->entropy.rep[1];
+            dstDCtx->entropy.rep[2] = ddict->entropy.rep[2];
+        } else {
+            dstDCtx->litEntropy = 0;
+            dstDCtx->fseEntropy = 0;
+        }
+    }
+}
+
+static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
+{
+    ddict->dictID = 0;
+    ddict->entropyPresent = 0;
+    if (ddict->dictSize < 8) return 0;
+    {   U32 const magic = MEM_readLE32(ddict->dictContent);
+        if (magic != ZSTD_DICT_MAGIC) return 0;   /* pure content mode */
+    }
+    ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
+
+    /* load entropy tables */
+    CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
+    ddict->entropyPresent = 1;
+    return 0;
+}
+
+
+ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
 {
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
     if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
     {   ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
-        void* const dictContent = ZSTD_malloc(dictSize, customMem);
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
+        if (!ddict) return NULL;
+        ddict->cMem = customMem;
 
-        if (!dictContent || !ddict || !dctx) {
-            ZSTD_free(dictContent, customMem);
-            ZSTD_free(ddict, customMem);
-            ZSTD_free(dctx, customMem);
-            return NULL;
+        if ((byReference) || (!dict) || (!dictSize)) {
+            ddict->dictBuffer = NULL;
+            ddict->dictContent = dict;
+        } else {
+            void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
+            if (!internalBuffer) { ZSTD_freeDDict(ddict); return NULL; }
+            memcpy(internalBuffer, dict, dictSize);
+            ddict->dictBuffer = internalBuffer;
+            ddict->dictContent = internalBuffer;
         }
-
-        if (dictSize) {
-            memcpy(dictContent, dict, dictSize);
-        }
-        {   size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
+        ddict->dictSize = dictSize;
+        ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
+        /* parse dictionary content */
+        {   size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
             if (ZSTD_isError(errorCode)) {
-                ZSTD_free(dictContent, customMem);
-                ZSTD_free(ddict, customMem);
-                ZSTD_free(dctx, customMem);
+                ZSTD_freeDDict(ddict);
                 return NULL;
         }   }
 
-        ddict->dict = dictContent;
-        ddict->dictSize = dictSize;
-        ddict->refContext = dctx;
         return ddict;
     }
 }
 
 /*! ZSTD_createDDict() :
-*   Create a digested dictionary, ready to start decompression without startup delay.
-*   `dict` can be released after `ZSTD_DDict` creation */
+*   Create a digested dictionary, to start decompression without startup delay.
+*   `dict` content is copied inside DDict.
+*   Consequently, `dict` can be released after `ZSTD_DDict` creation */
 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
 {
     ZSTD_customMem const allocator = { NULL, NULL, NULL };
-    return ZSTD_createDDict_advanced(dict, dictSize, allocator);
+    return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
 }
 
+
+/*! ZSTD_createDDict_byReference() :
+ *  Create a digested dictionary, to start decompression without startup delay.
+ *  Dictionary content is simply referenced, it will be accessed during decompression.
+ *  Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
+ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
+{
+    ZSTD_customMem const allocator = { NULL, NULL, NULL };
+    return ZSTD_createDDict_advanced(dictBuffer, dictSize, 1, allocator);
+}
+
+
 size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;   /* support free on NULL */
-    {   ZSTD_customMem const cMem = ddict->refContext->customMem;
-        ZSTD_freeDCtx(ddict->refContext);
-        ZSTD_free(ddict->dict, cMem);
+    {   ZSTD_customMem const cMem = ddict->cMem;
+        ZSTD_free(ddict->dictBuffer, cMem);
         ZSTD_free(ddict, cMem);
         return 0;
     }
@@ -1775,7 +2090,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;   /* support sizeof on NULL */
-    return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
+    return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
 }
 
 /*! ZSTD_getDictID_fromDict() :
@@ -1796,7 +2111,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;
-    return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize);
+    return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
 }
 
 /*! ZSTD_getDictID_fromFrame() :
@@ -1826,12 +2141,10 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                             const void* src, size_t srcSize,
                             const ZSTD_DDict* ddict)
 {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
-#endif
-    ZSTD_refDCtx(dctx, ddict->refContext);
-    ZSTD_checkContinuity(dctx, dst);
-    return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
+    /* pass content and size in case legacy frames are encountered */
+    return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
+                                     NULL, 0,
+                                     ddict);
 }
 
 
@@ -1919,7 +2232,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
     zds->stage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
     ZSTD_freeDDict(zds->ddictLocal);
-    if (dict) {
+    if (dict && dictSize >= 8) {
         zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
         if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
     } else zds->ddictLocal = NULL;
@@ -1956,7 +2269,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
     switch(paramType)
     {
         default : return ERROR(parameter_unknown);
-        case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
+        case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
     }
     return 0;
 }
@@ -2007,7 +2320,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
                 {   U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
                     if (legacyVersion) {
-                        const void* const dict = zds->ddict ? zds->ddict->dict : NULL;
+                        const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
                         size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
                         CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
                                                        dict, dictSize));
@@ -2031,10 +2344,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     break;
             }   }
 
+            /* check for single-pass mode opportunity */
+            if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
+                && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
+                size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
+                if (cSize <= (size_t)(iend-istart)) {
+                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict);
+                    if (ZSTD_isError(decompressedSize)) return decompressedSize;
+                    ip = istart + cSize;
+                    op += decompressedSize;
+                    zds->dctx->expected = 0;
+                    zds->stage = zdss_init;
+                    someMoreWork = 0;
+                    break;
+            }   }
+
             /* Consume header */
-            {   const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL;
-                ZSTD_refDCtx(zds->dctx, refContext);
-            }
+            ZSTD_refDDict(zds->dctx, zds->ddict);
             {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);  /* == ZSTD_frameHeaderSize_prefix */
                 CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
                 {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
@@ -2046,7 +2372,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
             /* Adapt buffer sizes to frame header instructions */
             {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
-                size_t const neededOutSize = zds->fParams.windowSize + blockSize;
+                size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
                 zds->blockSize = blockSize;
                 if (zds->inBuffSize < blockSize) {
                     ZSTD_free(zds->inBuff, zds->customMem);

lib/deprecated/zbuff.h

@@ -42,7 +42,9 @@ extern "C" {
 #ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
 #  define ZBUFF_DEPRECATED(message) ZSTDLIB_API  /* disable deprecation warnings */
 #else
-#  if (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
+#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#    define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
+#  elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
 #    define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
 #  elif defined(__GNUC__) && (__GNUC__ >= 3)
 #    define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))

lib/deprecated/zbuff_common.c

@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*-*************************************
+*  Dependencies
+***************************************/
+#include "error_private.h"
+#include "zbuff.h"
+
+/*-****************************************
+*  ZBUFF Error Management  (deprecated)
+******************************************/
+
+/*! ZBUFF_isError() :
+*   tells if a return value is an error code */
+unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
+/*! ZBUFF_getErrorName() :
+*   provides error code string from function result (useful for debugging) */
+const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+

lib/dictBuilder/cover.c

@@ -0,0 +1,1029 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/* *****************************************************************************
+ * Constructs a dictionary using a heuristic based on the following paper:
+ *
+ * Liao, Petri, Moffat, Wirth
+ * Effective Construction of Relative Lempel-Ziv Dictionaries
+ * Published in WWW 2016.
+ *
+ * Adapted from code originally written by @ot (Giuseppe Ottaviano).
+ ******************************************************************************/
+
+/*-*************************************
+*  Dependencies
+***************************************/
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h> /* memset */
+#include <time.h>   /* clock */
+
+#include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#define ZDICT_STATIC_LINKING_ONLY
+#endif
+#include "zdict.h"
+
+/*-*************************************
+*  Constants
+***************************************/
+#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
+
+/*-*************************************
+*  Console display
+***************************************/
+static int g_displayLevel = 2;
+#define DISPLAY(...)                                                           \
+  {                                                                            \
+    fprintf(stderr, __VA_ARGS__);                                              \
+    fflush(stderr);                                                            \
+  }
+#define LOCALDISPLAYLEVEL(displayLevel, l, ...)                                \
+  if (displayLevel >= l) {                                                     \
+    DISPLAY(__VA_ARGS__);                                                      \
+  } /* 0 : no display;   1: errors;   2: default;  3: details;  4: debug */
+#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
+
+#define LOCALDISPLAYUPDATE(displayLevel, l, ...)                               \
+  if (displayLevel >= l) {                                                     \
+    if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) {             \
+      g_time = clock();                                                        \
+      DISPLAY(__VA_ARGS__);                                                    \
+      if (displayLevel >= 4)                                                   \
+        fflush(stdout);                                                        \
+    }                                                                          \
+  }
+#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+/*-*************************************
+* Hash table
+***************************************
+* A small specialized hash map for storing activeDmers.
+* The map does not resize, so if it becomes full it will loop forever.
+* Thus, the map must be large enough to store every value.
+* The map implements linear probing and keeps its load less than 0.5.
+*/
+
+#define MAP_EMPTY_VALUE ((U32)-1)
+typedef struct COVER_map_pair_t_s {
+  U32 key;
+  U32 value;
+} COVER_map_pair_t;
+
+typedef struct COVER_map_s {
+  COVER_map_pair_t *data;
+  U32 sizeLog;
+  U32 size;
+  U32 sizeMask;
+} COVER_map_t;
+
+/**
+ * Clear the map.
+ */
+static void COVER_map_clear(COVER_map_t *map) {
+  memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t));
+}
+
+/**
+ * Initializes a map of the given size.
+ * Returns 1 on success and 0 on failure.
+ * The map must be destroyed with COVER_map_destroy().
+ * The map is only guaranteed to be large enough to hold size elements.
+ */
+static int COVER_map_init(COVER_map_t *map, U32 size) {
+  map->sizeLog = ZSTD_highbit32(size) + 2;
+  map->size = (U32)1 << map->sizeLog;
+  map->sizeMask = map->size - 1;
+  map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t));
+  if (!map->data) {
+    map->sizeLog = 0;
+    map->size = 0;
+    return 0;
+  }
+  COVER_map_clear(map);
+  return 1;
+}
+
+/**
+ * Internal hash function
+ */
+static const U32 prime4bytes = 2654435761U;
+static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
+  return (key * prime4bytes) >> (32 - map->sizeLog);
+}
+
+/**
+ * Helper function that returns the index that a key should be placed into.
+ */
+static U32 COVER_map_index(COVER_map_t *map, U32 key) {
+  const U32 hash = COVER_map_hash(map, key);
+  U32 i;
+  for (i = hash;; i = (i + 1) & map->sizeMask) {
+    COVER_map_pair_t *pos = &map->data[i];
+    if (pos->value == MAP_EMPTY_VALUE) {
+      return i;
+    }
+    if (pos->key == key) {
+      return i;
+    }
+  }
+}
+
+/**
+ * Returns the pointer to the value for key.
+ * If key is not in the map, it is inserted and the value is set to 0.
+ * The map must not be full.
+ */
+static U32 *COVER_map_at(COVER_map_t *map, U32 key) {
+  COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)];
+  if (pos->value == MAP_EMPTY_VALUE) {
+    pos->key = key;
+    pos->value = 0;
+  }
+  return &pos->value;
+}
+
+/**
+ * Deletes key from the map if present.
+ */
+static void COVER_map_remove(COVER_map_t *map, U32 key) {
+  U32 i = COVER_map_index(map, key);
+  COVER_map_pair_t *del = &map->data[i];
+  U32 shift = 1;
+  if (del->value == MAP_EMPTY_VALUE) {
+    return;
+  }
+  for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) {
+    COVER_map_pair_t *const pos = &map->data[i];
+    /* If the position is empty we are done */
+    if (pos->value == MAP_EMPTY_VALUE) {
+      del->value = MAP_EMPTY_VALUE;
+      return;
+    }
+    /* If pos can be moved to del do so */
+    if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) {
+      del->key = pos->key;
+      del->value = pos->value;
+      del = pos;
+      shift = 1;
+    } else {
+      ++shift;
+    }
+  }
+}
+
+/**
+ * Destroyes a map that is inited with COVER_map_init().
+ */
+static void COVER_map_destroy(COVER_map_t *map) {
+  if (map->data) {
+    free(map->data);
+  }
+  map->data = NULL;
+  map->size = 0;
+}
+
+/*-*************************************
+* Context
+***************************************/
+
+typedef struct {
+  const BYTE *samples;
+  size_t *offsets;
+  const size_t *samplesSizes;
+  size_t nbSamples;
+  U32 *suffix;
+  size_t suffixSize;
+  U32 *freqs;
+  U32 *dmerAt;
+  unsigned d;
+} COVER_ctx_t;
+
+/* We need a global context for qsort... */
+static COVER_ctx_t *g_ctx = NULL;
+
+/*-*************************************
+*  Helper functions
+***************************************/
+
+/**
+ * Returns the sum of the sample sizes.
+ */
+static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
+  size_t sum = 0;
+  size_t i;
+  for (i = 0; i < nbSamples; ++i) {
+    sum += samplesSizes[i];
+  }
+  return sum;
+}
+
+/**
+ * Returns -1 if the dmer at lp is less than the dmer at rp.
+ * Return 0 if the dmers at lp and rp are equal.
+ * Returns 1 if the dmer at lp is greater than the dmer at rp.
+ */
+static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) {
+  const U32 lhs = *(const U32 *)lp;
+  const U32 rhs = *(const U32 *)rp;
+  return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d);
+}
+
+/**
+ * Same as COVER_cmp() except ties are broken by pointer value
+ * NOTE: g_ctx must be set to call this function.  A global is required because
+ * qsort doesn't take an opaque pointer.
+ */
+static int COVER_strict_cmp(const void *lp, const void *rp) {
+  int result = COVER_cmp(g_ctx, lp, rp);
+  if (result == 0) {
+    result = lp < rp ? -1 : 1;
+  }
+  return result;
+}
+
+/**
+ * Returns the first pointer in [first, last) whose element does not compare
+ * less than value.  If no such element exists it returns last.
+ */
+static const size_t *COVER_lower_bound(const size_t *first, const size_t *last,
+                                       size_t value) {
+  size_t count = last - first;
+  while (count != 0) {
+    size_t step = count / 2;
+    const size_t *ptr = first;
+    ptr += step;
+    if (*ptr < value) {
+      first = ++ptr;
+      count -= step + 1;
+    } else {
+      count = step;
+    }
+  }
+  return first;
+}
+
+/**
+ * Generic groupBy function.
+ * Groups an array sorted by cmp into groups with equivalent values.
+ * Calls grp for each group.
+ */
+static void
+COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx,
+              int (*cmp)(COVER_ctx_t *, const void *, const void *),
+              void (*grp)(COVER_ctx_t *, const void *, const void *)) {
+  const BYTE *ptr = (const BYTE *)data;
+  size_t num = 0;
+  while (num < count) {
+    const BYTE *grpEnd = ptr + size;
+    ++num;
+    while (num < count && cmp(ctx, ptr, grpEnd) == 0) {
+      grpEnd += size;
+      ++num;
+    }
+    grp(ctx, ptr, grpEnd);
+    ptr = grpEnd;
+  }
+}
+
+/*-*************************************
+*  Cover functions
+***************************************/
+
+/**
+ * Called on each group of positions with the same dmer.
+ * Counts the frequency of each dmer and saves it in the suffix array.
+ * Fills `ctx->dmerAt`.
+ */
+static void COVER_group(COVER_ctx_t *ctx, const void *group,
+                        const void *groupEnd) {
+  /* The group consists of all the positions with the same first d bytes. */
+  const U32 *grpPtr = (const U32 *)group;
+  const U32 *grpEnd = (const U32 *)groupEnd;
+  /* The dmerId is how we will reference this dmer.
+   * This allows us to map the whole dmer space to a much smaller space, the
+   * size of the suffix array.
+   */
+  const U32 dmerId = (U32)(grpPtr - ctx->suffix);
+  /* Count the number of samples this dmer shows up in */
+  U32 freq = 0;
+  /* Details */
+  const size_t *curOffsetPtr = ctx->offsets;
+  const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples;
+  /* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a
+   * different sample than the last.
+   */
+  size_t curSampleEnd = ctx->offsets[0];
+  for (; grpPtr != grpEnd; ++grpPtr) {
+    /* Save the dmerId for this position so we can get back to it. */
+    ctx->dmerAt[*grpPtr] = dmerId;
+    /* Dictionaries only help for the first reference to the dmer.
+     * After that zstd can reference the match from the previous reference.
+     * So only count each dmer once for each sample it is in.
+     */
+    if (*grpPtr < curSampleEnd) {
+      continue;
+    }
+    freq += 1;
+    /* Binary search to find the end of the sample *grpPtr is in.
+     * In the common case that grpPtr + 1 == grpEnd we can skip the binary
+     * search because the loop is over.
+     */
+    if (grpPtr + 1 != grpEnd) {
+      const size_t *sampleEndPtr =
+          COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr);
+      curSampleEnd = *sampleEndPtr;
+      curOffsetPtr = sampleEndPtr + 1;
+    }
+  }
+  /* At this point we are never going to look at this segment of the suffix
+   * array again.  We take advantage of this fact to save memory.
+   * We store the frequency of the dmer in the first position of the group,
+   * which is dmerId.
+   */
+  ctx->suffix[dmerId] = freq;
+}
+
+/**
+ * A segment is a range in the source as well as the score of the segment.
+ */
+typedef struct {
+  U32 begin;
+  U32 end;
+  double score;
+} COVER_segment_t;
+
+/**
+ * Selects the best segment in an epoch.
+ * Segments of are scored according to the function:
+ *
+ * Let F(d) be the frequency of dmer d.
+ * Let S_i be the dmer at position i of segment S which has length k.
+ *
+ *     Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
+ *
+ * Once the dmer d is in the dictionay we set F(d) = 0.
+ */
+static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
+                                           COVER_map_t *activeDmers, U32 begin,
+                                           U32 end, COVER_params_t parameters) {
+  /* Constants */
+  const U32 k = parameters.k;
+  const U32 d = parameters.d;
+  const U32 dmersInK = k - d + 1;
+  /* Try each segment (activeSegment) and save the best (bestSegment) */
+  COVER_segment_t bestSegment = {0, 0, 0};
+  COVER_segment_t activeSegment;
+  /* Reset the activeDmers in the segment */
+  COVER_map_clear(activeDmers);
+  /* The activeSegment starts at the beginning of the epoch. */
+  activeSegment.begin = begin;
+  activeSegment.end = begin;
+  activeSegment.score = 0;
+  /* Slide the activeSegment through the whole epoch.
+   * Save the best segment in bestSegment.
+   */
+  while (activeSegment.end < end) {
+    /* The dmerId for the dmer at the next position */
+    U32 newDmer = ctx->dmerAt[activeSegment.end];
+    /* The entry in activeDmers for this dmerId */
+    U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer);
+    /* If the dmer isn't already present in the segment add its score. */
+    if (*newDmerOcc == 0) {
+      /* The paper suggest using the L-0.5 norm, but experiments show that it
+       * doesn't help.
+       */
+      activeSegment.score += freqs[newDmer];
+    }
+    /* Add the dmer to the segment */
+    activeSegment.end += 1;
+    *newDmerOcc += 1;
+
+    /* If the window is now too large, drop the first position */
+    if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
+      U32 delDmer = ctx->dmerAt[activeSegment.begin];
+      U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer);
+      activeSegment.begin += 1;
+      *delDmerOcc -= 1;
+      /* If this is the last occurence of the dmer, subtract its score */
+      if (*delDmerOcc == 0) {
+        COVER_map_remove(activeDmers, delDmer);
+        activeSegment.score -= freqs[delDmer];
+      }
+    }
+
+    /* If this segment is the best so far save it */
+    if (activeSegment.score > bestSegment.score) {
+      bestSegment = activeSegment;
+    }
+  }
+  {
+    /* Trim off the zero frequency head and tail from the segment. */
+    U32 newBegin = bestSegment.end;
+    U32 newEnd = bestSegment.begin;
+    U32 pos;
+    for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+      U32 freq = freqs[ctx->dmerAt[pos]];
+      if (freq != 0) {
+        newBegin = MIN(newBegin, pos);
+        newEnd = pos + 1;
+      }
+    }
+    bestSegment.begin = newBegin;
+    bestSegment.end = newEnd;
+  }
+  {
+    /* Zero out the frequency of each dmer covered by the chosen segment. */
+    U32 pos;
+    for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+      freqs[ctx->dmerAt[pos]] = 0;
+    }
+  }
+  return bestSegment;
+}
+
+/**
+ * Check the validity of the parameters.
+ * Returns non-zero if the parameters are valid and 0 otherwise.
+ */
+static int COVER_checkParameters(COVER_params_t parameters) {
+  /* k and d are required parameters */
+  if (parameters.d == 0 || parameters.k == 0) {
+    return 0;
+  }
+  /* d <= k */
+  if (parameters.d > parameters.k) {
+    return 0;
+  }
+  return 1;
+}
+
+/**
+ * Clean up a context initialized with `COVER_ctx_init()`.
+ */
+static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
+  if (!ctx) {
+    return;
+  }
+  if (ctx->suffix) {
+    free(ctx->suffix);
+    ctx->suffix = NULL;
+  }
+  if (ctx->freqs) {
+    free(ctx->freqs);
+    ctx->freqs = NULL;
+  }
+  if (ctx->dmerAt) {
+    free(ctx->dmerAt);
+    ctx->dmerAt = NULL;
+  }
+  if (ctx->offsets) {
+    free(ctx->offsets);
+    ctx->offsets = NULL;
+  }
+}
+
+/**
+ * Prepare a context for dictionary building.
+ * The context is only dependent on the parameter `d` and can used multiple
+ * times.
+ * Returns 1 on success or zero on error.
+ * The context must be destroyed with `COVER_ctx_destroy()`.
+ */
+static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
+                          const size_t *samplesSizes, unsigned nbSamples,
+                          unsigned d) {
+  const BYTE *const samples = (const BYTE *)samplesBuffer;
+  const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
+  /* Checks */
+  if (totalSamplesSize < d ||
+      totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
+    DISPLAYLEVEL(1, "Total samples size is too large, maximum size is %u MB\n",
+                 (COVER_MAX_SAMPLES_SIZE >> 20));
+    return 0;
+  }
+  /* Zero the context */
+  memset(ctx, 0, sizeof(*ctx));
+  DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples,
+               (U32)totalSamplesSize);
+  ctx->samples = samples;
+  ctx->samplesSizes = samplesSizes;
+  ctx->nbSamples = nbSamples;
+  /* Partial suffix array */
+  ctx->suffixSize = totalSamplesSize - d + 1;
+  ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+  /* Maps index to the dmerID */
+  ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+  /* The offsets of each file */
+  ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
+  if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) {
+    DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
+    COVER_ctx_destroy(ctx);
+    return 0;
+  }
+  ctx->freqs = NULL;
+  ctx->d = d;
+
+  /* Fill offsets from the samlesSizes */
+  {
+    U32 i;
+    ctx->offsets[0] = 0;
+    for (i = 1; i <= nbSamples; ++i) {
+      ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
+    }
+  }
+  DISPLAYLEVEL(2, "Constructing partial suffix array\n");
+  {
+    /* suffix is a partial suffix array.
+     * It only sorts suffixes by their first parameters.d bytes.
+     * The sort is stable, so each dmer group is sorted by position in input.
+     */
+    U32 i;
+    for (i = 0; i < ctx->suffixSize; ++i) {
+      ctx->suffix[i] = i;
+    }
+    /* qsort doesn't take an opaque pointer, so pass as a global */
+    g_ctx = ctx;
+    qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), &COVER_strict_cmp);
+  }
+  DISPLAYLEVEL(2, "Computing frequencies\n");
+  /* For each dmer group (group of positions with the same first d bytes):
+   * 1. For each position we set dmerAt[position] = dmerID.  The dmerID is
+   *    (groupBeginPtr - suffix).  This allows us to go from position to
+   *    dmerID so we can look up values in freq.
+   * 2. We calculate how many samples the dmer occurs in and save it in
+   *    freqs[dmerId].
+   */
+  COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, &COVER_cmp,
+                &COVER_group);
+  ctx->freqs = ctx->suffix;
+  ctx->suffix = NULL;
+  return 1;
+}
+
+/**
+ * Given the prepared context build the dictionary.
+ */
+static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
+                                    COVER_map_t *activeDmers, void *dictBuffer,
+                                    size_t dictBufferCapacity,
+                                    COVER_params_t parameters) {
+  BYTE *const dict = (BYTE *)dictBuffer;
+  size_t tail = dictBufferCapacity;
+  /* Divide the data up into epochs of equal size.
+   * We will select at least one segment from each epoch.
+   */
+  const U32 epochs = (U32)(dictBufferCapacity / parameters.k);
+  const U32 epochSize = (U32)(ctx->suffixSize / epochs);
+  size_t epoch;
+  DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
+               epochSize);
+  /* Loop through the epochs until there are no more segments or the dictionary
+   * is full.
+   */
+  for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
+    const U32 epochBegin = (U32)(epoch * epochSize);
+    const U32 epochEnd = epochBegin + epochSize;
+    size_t segmentSize;
+    /* Select a segment */
+    COVER_segment_t segment = COVER_selectSegment(
+        ctx, freqs, activeDmers, epochBegin, epochEnd, parameters);
+    /* Trim the segment if necessary and if it is empty then we are done */
+    segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
+    if (segmentSize == 0) {
+      break;
+    }
+    /* We fill the dictionary from the back to allow the best segments to be
+     * referenced with the smallest offsets.
+     */
+    tail -= segmentSize;
+    memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
+    DISPLAYUPDATE(
+        2, "\r%u%%       ",
+        (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
+  }
+  DISPLAYLEVEL(2, "\r%79s\r", "");
+  return tail;
+}
+
+/**
+ * Translate from COVER_params_t to ZDICT_params_t required for finalizing the
+ * dictionary.
+ */
+static ZDICT_params_t COVER_translateParams(COVER_params_t parameters) {
+  ZDICT_params_t zdictParams;
+  memset(&zdictParams, 0, sizeof(zdictParams));
+  zdictParams.notificationLevel = 1;
+  zdictParams.dictID = parameters.dictID;
+  zdictParams.compressionLevel = parameters.compressionLevel;
+  return zdictParams;
+}
+
+ZDICTLIB_API size_t COVER_trainFromBuffer(
+    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+    const size_t *samplesSizes, unsigned nbSamples, COVER_params_t parameters) {
+  BYTE *const dict = (BYTE *)dictBuffer;
+  COVER_ctx_t ctx;
+  COVER_map_t activeDmers;
+  /* Checks */
+  if (!COVER_checkParameters(parameters)) {
+    DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+    return ERROR(GENERIC);
+  }
+  if (nbSamples == 0) {
+    DISPLAYLEVEL(1, "Cover must have at least one input file\n");
+    return ERROR(GENERIC);
+  }
+  if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+    DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+                 ZDICT_DICTSIZE_MIN);
+    return ERROR(dstSize_tooSmall);
+  }
+  /* Initialize global data */
+  g_displayLevel = parameters.notificationLevel;
+  /* Initialize context and activeDmers */
+  if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
+                      parameters.d)) {
+    return ERROR(GENERIC);
+  }
+  if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
+    DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
+    COVER_ctx_destroy(&ctx);
+    return ERROR(GENERIC);
+  }
+
+  DISPLAYLEVEL(2, "Building dictionary\n");
+  {
+    const size_t tail =
+        COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer,
+                              dictBufferCapacity, parameters);
+    ZDICT_params_t zdictParams = COVER_translateParams(parameters);
+    const size_t dictionarySize = ZDICT_finalizeDictionary(
+        dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+        samplesBuffer, samplesSizes, nbSamples, zdictParams);
+    if (!ZSTD_isError(dictionarySize)) {
+      DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
+                   (U32)dictionarySize);
+    }
+    COVER_ctx_destroy(&ctx);
+    COVER_map_destroy(&activeDmers);
+    return dictionarySize;
+  }
+}
+
+/**
+ * COVER_best_t is used for two purposes:
+ * 1. Synchronizing threads.
+ * 2. Saving the best parameters and dictionary.
+ *
+ * All of the methods except COVER_best_init() are thread safe if zstd is
+ * compiled with multithreaded support.
+ */
+typedef struct COVER_best_s {
+  pthread_mutex_t mutex;
+  pthread_cond_t cond;
+  size_t liveJobs;
+  void *dict;
+  size_t dictSize;
+  COVER_params_t parameters;
+  size_t compressedSize;
+} COVER_best_t;
+
+/**
+ * Initialize the `COVER_best_t`.
+ */
+static void COVER_best_init(COVER_best_t *best) {
+  if (!best) {
+    return;
+  }
+  pthread_mutex_init(&best->mutex, NULL);
+  pthread_cond_init(&best->cond, NULL);
+  best->liveJobs = 0;
+  best->dict = NULL;
+  best->dictSize = 0;
+  best->compressedSize = (size_t)-1;
+  memset(&best->parameters, 0, sizeof(best->parameters));
+}
+
+/**
+ * Wait until liveJobs == 0.
+ */
+static void COVER_best_wait(COVER_best_t *best) {
+  if (!best) {
+    return;
+  }
+  pthread_mutex_lock(&best->mutex);
+  while (best->liveJobs != 0) {
+    pthread_cond_wait(&best->cond, &best->mutex);
+  }
+  pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Call COVER_best_wait() and then destroy the COVER_best_t.
+ */
+static void COVER_best_destroy(COVER_best_t *best) {
+  if (!best) {
+    return;
+  }
+  COVER_best_wait(best);
+  if (best->dict) {
+    free(best->dict);
+  }
+  pthread_mutex_destroy(&best->mutex);
+  pthread_cond_destroy(&best->cond);
+}
+
+/**
+ * Called when a thread is about to be launched.
+ * Increments liveJobs.
+ */
+static void COVER_best_start(COVER_best_t *best) {
+  if (!best) {
+    return;
+  }
+  pthread_mutex_lock(&best->mutex);
+  ++best->liveJobs;
+  pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Called when a thread finishes executing, both on error or success.
+ * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
+ * If this dictionary is the best so far save it and its parameters.
+ */
+static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
+                              COVER_params_t parameters, void *dict,
+                              size_t dictSize) {
+  if (!best) {
+    return;
+  }
+  {
+    size_t liveJobs;
+    pthread_mutex_lock(&best->mutex);
+    --best->liveJobs;
+    liveJobs = best->liveJobs;
+    /* If the new dictionary is better */
+    if (compressedSize < best->compressedSize) {
+      /* Allocate space if necessary */
+      if (!best->dict || best->dictSize < dictSize) {
+        if (best->dict) {
+          free(best->dict);
+        }
+        best->dict = malloc(dictSize);
+        if (!best->dict) {
+          best->compressedSize = ERROR(GENERIC);
+          best->dictSize = 0;
+          return;
+        }
+      }
+      /* Save the dictionary, parameters, and size */
+      memcpy(best->dict, dict, dictSize);
+      best->dictSize = dictSize;
+      best->parameters = parameters;
+      best->compressedSize = compressedSize;
+    }
+    pthread_mutex_unlock(&best->mutex);
+    if (liveJobs == 0) {
+      pthread_cond_broadcast(&best->cond);
+    }
+  }
+}
+
+/**
+ * Parameters for COVER_tryParameters().
+ */
+typedef struct COVER_tryParameters_data_s {
+  const COVER_ctx_t *ctx;
+  COVER_best_t *best;
+  size_t dictBufferCapacity;
+  COVER_params_t parameters;
+} COVER_tryParameters_data_t;
+
+/**
+ * Tries a set of parameters and upates the COVER_best_t with the results.
+ * This function is thread safe if zstd is compiled with multithreaded support.
+ * It takes its parameters as an *OWNING* opaque pointer to support threading.
+ */
+static void COVER_tryParameters(void *opaque) {
+  /* Save parameters as local variables */
+  COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque;
+  const COVER_ctx_t *const ctx = data->ctx;
+  const COVER_params_t parameters = data->parameters;
+  size_t dictBufferCapacity = data->dictBufferCapacity;
+  size_t totalCompressedSize = ERROR(GENERIC);
+  /* Allocate space for hash table, dict, and freqs */
+  COVER_map_t activeDmers;
+  BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+  U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+  if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
+    DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
+    goto _cleanup;
+  }
+  if (!dict || !freqs) {
+    DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
+    goto _cleanup;
+  }
+  /* Copy the frequencies because we need to modify them */
+  memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32));
+  /* Build the dictionary */
+  {
+    const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
+                                              dictBufferCapacity, parameters);
+    const ZDICT_params_t zdictParams = COVER_translateParams(parameters);
+    dictBufferCapacity = ZDICT_finalizeDictionary(
+        dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+        ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, zdictParams);
+    if (ZDICT_isError(dictBufferCapacity)) {
+      DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
+      goto _cleanup;
+    }
+  }
+  /* Check total compressed size */
+  {
+    /* Pointers */
+    ZSTD_CCtx *cctx;
+    ZSTD_CDict *cdict;
+    void *dst;
+    /* Local variables */
+    size_t dstCapacity;
+    size_t i;
+    /* Allocate dst with enough space to compress the maximum sized sample */
+    {
+      size_t maxSampleSize = 0;
+      for (i = 0; i < ctx->nbSamples; ++i) {
+        maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
+      }
+      dstCapacity = ZSTD_compressBound(maxSampleSize);
+      dst = malloc(dstCapacity);
+    }
+    /* Create the cctx and cdict */
+    cctx = ZSTD_createCCtx();
+    cdict =
+        ZSTD_createCDict(dict, dictBufferCapacity, parameters.compressionLevel);
+    if (!dst || !cctx || !cdict) {
+      goto _compressCleanup;
+    }
+    /* Compress each sample and sum their sizes (or error) */
+    totalCompressedSize = 0;
+    for (i = 0; i < ctx->nbSamples; ++i) {
+      const size_t size = ZSTD_compress_usingCDict(
+          cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
+          ctx->samplesSizes[i], cdict);
+      if (ZSTD_isError(size)) {
+        totalCompressedSize = ERROR(GENERIC);
+        goto _compressCleanup;
+      }
+      totalCompressedSize += size;
+    }
+  _compressCleanup:
+    ZSTD_freeCCtx(cctx);
+    ZSTD_freeCDict(cdict);
+    if (dst) {
+      free(dst);
+    }
+  }
+
+_cleanup:
+  COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
+                    dictBufferCapacity);
+  free(data);
+  COVER_map_destroy(&activeDmers);
+  if (dict) {
+    free(dict);
+  }
+  if (freqs) {
+    free(freqs);
+  }
+}
+
+ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
+                                                  size_t dictBufferCapacity,
+                                                  const void *samplesBuffer,
+                                                  const size_t *samplesSizes,
+                                                  unsigned nbSamples,
+                                                  COVER_params_t *parameters) {
+  /* constants */
+  const unsigned nbThreads = parameters->nbThreads;
+  const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
+  const unsigned kMaxD = parameters->d == 0 ? 16 : parameters->d;
+  const unsigned kMinK = parameters->k == 0 ? kMaxD : parameters->k;
+  const unsigned kMaxK = parameters->k == 0 ? 2048 : parameters->k;
+  const unsigned kSteps = parameters->steps == 0 ? 32 : parameters->steps;
+  const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
+  const unsigned kIterations =
+      (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
+  /* Local variables */
+  const int displayLevel = parameters->notificationLevel;
+  unsigned iteration = 1;
+  unsigned d;
+  unsigned k;
+  COVER_best_t best;
+  POOL_ctx *pool = NULL;
+  /* Checks */
+  if (kMinK < kMaxD || kMaxK < kMinK) {
+    LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
+    return ERROR(GENERIC);
+  }
+  if (nbSamples == 0) {
+    DISPLAYLEVEL(1, "Cover must have at least one input file\n");
+    return ERROR(GENERIC);
+  }
+  if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+    DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+                 ZDICT_DICTSIZE_MIN);
+    return ERROR(dstSize_tooSmall);
+  }
+  if (nbThreads > 1) {
+    pool = POOL_create(nbThreads, 1);
+    if (!pool) {
+      return ERROR(memory_allocation);
+    }
+  }
+  /* Initialization */
+  COVER_best_init(&best);
+  /* Turn down global display level to clean up display at level 2 and below */
+  g_displayLevel = parameters->notificationLevel - 1;
+  /* Loop through d first because each new value needs a new context */
+  LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
+                    kIterations);
+  for (d = kMinD; d <= kMaxD; d += 2) {
+    /* Initialize the context for this value of d */
+    COVER_ctx_t ctx;
+    LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
+    if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+      COVER_best_destroy(&best);
+      POOL_free(pool);
+      return ERROR(GENERIC);
+    }
+    /* Loop through k reusing the same context */
+    for (k = kMinK; k <= kMaxK; k += kStepSize) {
+      /* Prepare the arguments */
+      COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
+          sizeof(COVER_tryParameters_data_t));
+      LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
+      if (!data) {
+        LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
+        COVER_best_destroy(&best);
+        COVER_ctx_destroy(&ctx);
+        POOL_free(pool);
+        return ERROR(GENERIC);
+      }
+      data->ctx = &ctx;
+      data->best = &best;
+      data->dictBufferCapacity = dictBufferCapacity;
+      data->parameters = *parameters;
+      data->parameters.k = k;
+      data->parameters.d = d;
+      data->parameters.steps = kSteps;
+      /* Check the parameters */
+      if (!COVER_checkParameters(data->parameters)) {
+        DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+        free(data);
+        continue;
+      }
+      /* Call the function and pass ownership of data to it */
+      COVER_best_start(&best);
+      if (pool) {
+        POOL_add(pool, &COVER_tryParameters, data);
+      } else {
+        COVER_tryParameters(data);
+      }
+      /* Print status */
+      LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%%       ",
+                         (U32)((iteration * 100) / kIterations));
+      ++iteration;
+    }
+    COVER_best_wait(&best);
+    COVER_ctx_destroy(&ctx);
+  }
+  LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
+  /* Fill the output buffer and parameters with output of the best parameters */
+  {
+    const size_t dictSize = best.dictSize;
+    if (ZSTD_isError(best.compressedSize)) {
+      const size_t compressedSize = best.compressedSize;
+      COVER_best_destroy(&best);
+      POOL_free(pool);
+      return compressedSize;
+    }
+    *parameters = best.parameters;
+    memcpy(dictBuffer, best.dict, dictSize);
+    COVER_best_destroy(&best);
+    POOL_free(pool);
+    return dictSize;
+  }
+}

lib/dictBuilder/zdict.c

@@ -36,12 +36,11 @@
 #include <time.h>          /* clock */
 
 #include "mem.h"           /* read */
-#include "error_private.h"
 #include "fse.h"           /* FSE_normalizeCount, FSE_writeNCount */
 #define HUF_STATIC_LINKING_ONLY
-#include "huf.h"
+#include "huf.h"           /* HUF_buildCTable, HUF_writeCTable */
 #include "zstd_internal.h" /* includes zstd.h */
-#include "xxhash.h"
+#include "xxhash.h"        /* XXH64 */
 #include "divsufsort.h"
 #ifndef ZDICT_STATIC_LINKING_ONLY
 #  define ZDICT_STATIC_LINKING_ONLY
@@ -61,7 +60,7 @@
 #define NOISELENGTH 32
 
 #define MINRATIO 4
-static const int g_compressionLevel_default = 5;
+static const int g_compressionLevel_default = 6;
 static const U32 g_selectivity_default = 9;
 static const size_t g_provision_entropySize = 200;
 static const size_t g_min_fast_dictContent = 192;
@@ -307,13 +306,13 @@ static dictItem ZDICT_analyzePos(
         } while (length >=MINMATCHLENGTH);
 
         /* look backward */
-		length = MINMATCHLENGTH;
-		while ((length >= MINMATCHLENGTH) & (start > 0)) {
-			length = ZDICT_count(b + pos, b + suffix[start - 1]);
-			if (length >= LLIMIT) length = LLIMIT - 1;
-			lengthList[length]++;
-			if (length >= MINMATCHLENGTH) start--;
-		}
+        length = MINMATCHLENGTH;
+        while ((length >= MINMATCHLENGTH) & (start > 0)) {
+        	length = ZDICT_count(b + pos, b + suffix[start - 1]);
+        	if (length >= LLIMIT) length = LLIMIT - 1;
+        	lengthList[length]++;
+        	if (length >= MINMATCHLENGTH) start--;
+        }
 
         /* largest useful length */
         memset(cumulLength, 0, sizeof(cumulLength));
@@ -570,7 +569,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
             if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
     }
     cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
-    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
+    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
 
     if (cSize) {  /* if == 0; block is not compressible */
         const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
@@ -825,6 +824,55 @@ _cleanup:
 }
 
 
+
+size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
+                          const void* customDictContent, size_t dictContentSize,
+                          const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+                          ZDICT_params_t params)
+{
+    size_t hSize;
+#define HBUFFSIZE 256
+    BYTE header[HBUFFSIZE];
+    int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
+    U32 const notificationLevel = params.notificationLevel;
+
+    /* check conditions */
+    if (dictBufferCapacity < dictContentSize) return ERROR(dstSize_tooSmall);
+    if (dictContentSize < ZDICT_CONTENTSIZE_MIN) return ERROR(srcSize_wrong);
+    if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall);
+
+    /* dictionary header */
+    MEM_writeLE32(header, ZSTD_DICT_MAGIC);
+    {   U64 const randomID = XXH64(customDictContent, dictContentSize, 0);
+        U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
+        U32 const dictID = params.dictID ? params.dictID : compliantID;
+        MEM_writeLE32(header+4, dictID);
+    }
+    hSize = 8;
+
+    /* entropy tables */
+    DISPLAYLEVEL(2, "\r%70s\r", "");   /* clean display line */
+    DISPLAYLEVEL(2, "statistics ... \n");
+    {   size_t const eSize = ZDICT_analyzeEntropy(header+hSize, HBUFFSIZE-hSize,
+                                  compressionLevel,
+                                  samplesBuffer, samplesSizes, nbSamples,
+                                  customDictContent, dictContentSize,
+                                  notificationLevel);
+        if (ZDICT_isError(eSize)) return eSize;
+        hSize += eSize;
+    }
+
+    /* copy elements in final buffer ; note : src and dst buffer can overlap */
+    if (hSize + dictContentSize > dictBufferCapacity) dictContentSize = dictBufferCapacity - hSize;
+    {   size_t const dictSize = hSize + dictContentSize;
+        char* dictEnd = (char*)dictBuffer + dictSize;
+        memmove(dictEnd - dictContentSize, customDictContent, dictContentSize);
+        memcpy(dictBuffer, header, hSize);
+        return dictSize;
+    }
+}
+
+
 size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
                                                  const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                                                  ZDICT_params_t params)

lib/dictBuilder/zdict.h

@@ -19,15 +19,18 @@ extern "C" {
 #include <stddef.h>  /* size_t */
 
 
-/*======  Export for Windows  ======*/
-/*!
-*  ZSTD_DLL_EXPORT :
-*  Enable exporting of functions when building a Windows DLL
-*/
-#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZDICTLIB_API __declspec(dllexport)
+/* =====   ZDICTLIB_API : control library symbols visibility   ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#  define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
 #else
-#  define ZDICTLIB_API
+#  define ZDICTLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZDICTLIB_API ZDICTLIB_VISIBILITY
 #endif
 
 
@@ -79,27 +82,114 @@ typedef struct {
               or an error code, which can be tested by ZDICT_isError().
     note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
 */
-size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
+                                const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+                                ZDICT_params_t parameters);
+
+/*! COVER_params_t :
+    For all values 0 means default.
+    kMin and d are the only required parameters.
+*/
+typedef struct {
+    unsigned k;                  /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+    unsigned d;                  /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+    unsigned steps;              /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
+
+    unsigned nbThreads;          /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+    unsigned notificationLevel;  /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+    unsigned dictID;             /* 0 means auto mode (32-bits random value); other : force dictID value */
+    int      compressionLevel;   /* 0 means default; target a specific zstd compression level */
+} COVER_params_t;
+
+
+/*! COVER_trainFromBuffer() :
+    Train a dictionary from an array of samples using the COVER algorithm.
+    Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+    supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+    The resulting dictionary will be saved into `dictBuffer`.
+    @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+              or an error code, which can be tested with ZDICT_isError().
+    Note : COVER_trainFromBuffer() requires about 9 bytes of memory for each input byte.
+    Tips : In general, a reasonable dictionary has a size of ~ 100 KB.
+           It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`.
+           In general, it's recommended to provide a few thousands samples, but this can vary a lot.
+           It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+*/
+ZDICTLIB_API size_t COVER_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+                              const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+                              COVER_params_t parameters);
+
+/*! COVER_optimizeTrainFromBuffer() :
+    The same requirements as above hold for all the parameters except `parameters`.
+    This function tries many parameter combinations and picks the best parameters.
+    `*parameters` is filled with the best parameters found, and the dictionary
+    constructed with those parameters is stored in `dictBuffer`.
+
+    All of the parameters d, k, steps are optional.
+    If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
+    if steps is zero it defaults to its default value.
+    If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
+
+    @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+              or an error code, which can be tested with ZDICT_isError().
+              On success `*parameters` contains the parameters selected.
+    Note : COVER_optimizeTrainFromBuffer() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
+*/
+ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+                                     const void* samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
+                                     COVER_params_t *parameters);
+
+/*! ZDICT_finalizeDictionary() :
+
+    Given a custom content as a basis for dictionary, and a set of samples,
+    finalize dictionary by adding headers and statistics.
+
+    Samples must be stored concatenated in a flat buffer `samplesBuffer`,
+    supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
+
+    dictContentSize must be > ZDICT_CONTENTSIZE_MIN bytes.
+    maxDictSize must be >= dictContentSize, and must be > ZDICT_DICTSIZE_MIN bytes.
+
+    @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
+              or an error code, which can be tested by ZDICT_isError().
+    note : ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
+    note 2 : dictBuffer and customDictContent can overlap
+*/
+#define ZDICT_CONTENTSIZE_MIN 256
+#define ZDICT_DICTSIZE_MIN    512
+ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
+                                const void* customDictContent, size_t dictContentSize,
                                 const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                                 ZDICT_params_t parameters);
 
 
-/*! ZDICT_addEntropyTablesFromBuffer() :
 
-    Given a content-only dictionary (built using any 3rd party algorithm),
-    add entropy tables computed from an array of samples.
-    Samples must be stored concatenated in a flat buffer `samplesBuffer`,
-    supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
+/* Deprecation warnings */
+/* It is generally possible to disable deprecation warnings from compiler,
+   for example with -Wno-deprecated-declarations for gcc
+   or _CRT_SECURE_NO_WARNINGS in Visual.
+   Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
+#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
+#  define ZDICT_DEPRECATED(message) ZDICTLIB_API   /* disable deprecation warnings */
+#else
+#  define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#    define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
+#  elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
+#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
+#  elif (ZDICT_GCC_VERSION >= 301)
+#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
+#    define ZDICT_DEPRECATED(message) ZDICTLIB_API
+#  endif
+#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
 
-    The input dictionary content must be stored *at the end* of `dictBuffer`.
-    Its size is `dictContentSize`.
-    The resulting dictionary with added entropy tables will be *written back to `dictBuffer`*,
-    starting from its beginning.
-    @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
-*/
+ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
 size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
-                                        const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
-
+                                  const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
 
 #endif   /* ZDICT_STATIC_LINKING_ONLY */

lib/dll/example/build_package.bat

@@ -4,6 +4,7 @@ COPY tests\fullbench.c bin\example\
 COPY programs\datagen.c bin\example\
 COPY programs\datagen.h bin\example\
 COPY programs\util.h bin\example\
+COPY programs\platform.h bin\example\
 COPY lib\common\mem.h bin\example\
 COPY lib\common\zstd_errors.h bin\example\
 COPY lib\common\zstd_internal.h bin\example\

lib/dll/example/fullbench-dll.vcxproj

@@ -100,6 +100,7 @@
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -141,6 +142,7 @@
       <OptimizeReferences>true</OptimizeReferences>
       <AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

lib/dll/libzstd.def

@@ -58,6 +58,8 @@ EXPORTS
     ZSTD_getBlockSizeMax
     ZSTD_getCParams
     ZSTD_getDecompressedSize
+    ZSTD_findDecompressedSize
+    ZSTD_getFrameContentSize
     ZSTD_getErrorName
     ZSTD_getFrameParams
     ZSTD_getParams

lib/legacy/zstd_legacy.h

@@ -20,14 +20,33 @@ extern "C" {
 #include "mem.h"            /* MEM_STATIC */
 #include "error_private.h"  /* ERROR */
 #include "zstd.h"           /* ZSTD_inBuffer, ZSTD_outBuffer */
-#include "zstd_v01.h"
-#include "zstd_v02.h"
-#include "zstd_v03.h"
-#include "zstd_v04.h"
-#include "zstd_v05.h"
-#include "zstd_v06.h"
-#include "zstd_v07.h"
 
+#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
+#  undef ZSTD_LEGACY_SUPPORT
+#  define ZSTD_LEGACY_SUPPORT 8
+#endif
+
+#if (ZSTD_LEGACY_SUPPORT <= 1)
+#  include "zstd_v01.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
+#  include "zstd_v02.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
+#  include "zstd_v03.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
+#  include "zstd_v04.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
+#  include "zstd_v05.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
+#  include "zstd_v06.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
+#  include "zstd_v07.h"
+#endif
 
 /** ZSTD_isLegacy() :
     @return : > 0 if supported by legacy decoder. 0 otherwise.
@@ -40,13 +59,27 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
     magicNumberLE = MEM_readLE32(src);
     switch(magicNumberLE)
     {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
         case ZSTDv01_magicNumberLE:return 1;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
         case ZSTDv02_magicNumber : return 2;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
         case ZSTDv03_magicNumber : return 3;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case ZSTDv04_magicNumber : return 4;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case ZSTDv05_MAGICNUMBER : return 5;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case ZSTDv06_MAGICNUMBER : return 6;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case ZSTDv07_MAGICNUMBER : return 7;
+#endif
         default : return 0;
     }
 }
@@ -56,24 +89,30 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s
 {
     U32 const version = ZSTD_isLegacy(src, srcSize);
     if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
+#if (ZSTD_LEGACY_SUPPORT <= 5)
     if (version==5) {
         ZSTDv05_parameters fParams;
         size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
         if (frResult != 0) return 0;
         return fParams.srcSize;
     }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
     if (version==6) {
         ZSTDv06_frameParams fParams;
         size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
         if (frResult != 0) return 0;
         return fParams.frameContentSize;
     }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
     if (version==7) {
         ZSTDv07_frameParams fParams;
         size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
         if (frResult != 0) return 0;
         return fParams.frameContentSize;
     }
+#endif
     return 0;   /* should not be possible */
 }
 
@@ -86,14 +125,23 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
     U32 const version = ZSTD_isLegacy(src, compressedSize);
     switch(version)
     {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
         case 1 :
             return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
         case 2 :
             return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
         case 3 :
             return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
             return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
             {   size_t result;
                 ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx();
@@ -102,6 +150,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                 ZSTDv05_freeDCtx(zd);
                 return result;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
             {   size_t result;
                 ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx();
@@ -110,6 +160,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                 ZSTDv06_freeDCtx(zd);
                 return result;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
             {   size_t result;
                 ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx();
@@ -118,11 +170,50 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                 ZSTDv07_freeDCtx(zd);
                 return result;
             }
+#endif
         default :
             return ERROR(prefix_unknown);
     }
 }
 
+MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src,
+                                             size_t compressedSize)
+{
+    U32 const version = ZSTD_isLegacy(src, compressedSize);
+    switch(version)
+    {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
+        case 1 :
+            return ZSTDv01_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
+        case 2 :
+            return ZSTDv02_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
+        case 3 :
+            return ZSTDv03_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
+        case 4 :
+            return ZSTDv04_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
+        case 5 :
+            return ZSTDv05_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
+        case 6 :
+            return ZSTDv06_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
+        case 7 :
+            return ZSTDv07_findFrameCompressedSize(src, compressedSize);
+#endif
+        default :
+            return ERROR(prefix_unknown);
+    }
+}
 
 MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
 {
@@ -133,10 +224,18 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
         case 2 :
         case 3 :
             return ERROR(version_unsupported);
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext);
+#endif
     }
 }
 
@@ -152,6 +251,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
         case 2 :
         case 3 :
             return 0;
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
         {
             ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext;
@@ -161,6 +261,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
         {
             ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext;
@@ -169,6 +271,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
         {
             ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext;
@@ -177,6 +281,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
         {
             ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext;
@@ -185,6 +291,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
     }
 }
 
@@ -200,6 +307,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
         case 2 :
         case 3 :
             return ERROR(version_unsupported);
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
             {
                 ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext;
@@ -212,6 +320,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
             {
                 ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext;
@@ -224,6 +334,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
             {
                 ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext;
@@ -236,6 +348,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
             {
                 ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext;
@@ -248,6 +362,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
     }
 }
 

lib/legacy/zstd_v01.c

@@ -1992,6 +1992,37 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
     return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
+size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+    size_t remainingSize = srcSize;
+    U32 magicNumber;
+    blockProperties_t blockProperties;
+
+    /* Frame Header */
+    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    magicNumber = ZSTD_readBE32(src);
+    if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
+    ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
+
+    /* Loop on each block */
+    while (1)
+    {
+        size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
+        if (ZSTDv01_isError(blockSize)) return blockSize;
+
+        ip += ZSTD_blockHeaderSize;
+        remainingSize -= ZSTD_blockHeaderSize;
+        if (blockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        if (blockSize == 0) break;   /* bt_end */
+
+        ip += blockSize;
+        remainingSize -= blockSize;
+    }
+
+    return ip - (const BYTE*)src;
+}
 
 /*******************************
 *  Streaming Decompression API

lib/legacy/zstd_v01.h

@@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
 size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
+/**
+ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
+    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    return : the number of bytes that would be read to decompress this frame
+             or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+*/
+size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
+
 /**
 ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
 */

lib/legacy/zstd_v02.c

@@ -3378,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
+static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+
+    const BYTE* ip = (const BYTE*)src;
+    size_t remainingSize = srcSize;
+    U32 magicNumber;
+    blockProperties_t blockProperties;
+
+    /* Frame Header */
+    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    magicNumber = MEM_readLE32(src);
+    if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
+    ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
+
+    /* Loop on each block */
+    while (1)
+    {
+        size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTD_blockHeaderSize;
+        remainingSize -= ZSTD_blockHeaderSize;
+        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        if (cBlockSize == 0) break;   /* bt_end */
+
+        ip += cBlockSize;
+        remainingSize -= cBlockSize;
+    }
+
+    return ip - (const BYTE*)src;
+}
 
 /*******************************
 *  Streaming Decompression API
@@ -3492,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
 	return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
 }
 
+size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
+{
+    return ZSTD_findFrameCompressedSize(src, compressedSize);
+}
+
 ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
 {
 	return (ZSTDv02_Dctx*)ZSTD_createDCtx();

lib/legacy/zstd_v02.h

@@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
 size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
+/**
+ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
+    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    return : the number of bytes that would be read to decompress this frame
+             or an errorCode if it fails (which can be tested using ZSTDv02_isError())
+*/
+size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
+
 /**
 ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
 */

lib/legacy/zstd_v03.c

@@ -3019,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
+static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+    size_t remainingSize = srcSize;
+    U32 magicNumber;
+    blockProperties_t blockProperties;
+
+    /* Frame Header */
+    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    magicNumber = MEM_readLE32(src);
+    if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
+    ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
+
+    /* Loop on each block */
+    while (1)
+    {
+        size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTD_blockHeaderSize;
+        remainingSize -= ZSTD_blockHeaderSize;
+        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        if (cBlockSize == 0) break;   /* bt_end */
+
+        ip += cBlockSize;
+        remainingSize -= cBlockSize;
+    }
+
+    return ip - (const BYTE*)src;
+}
+
 
 /*******************************
 *  Streaming Decompression API
@@ -3133,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
 	return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
 }
 
+size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
+{
+    return ZSTD_findFrameCompressedSize(src, srcSize);
+}
+
 ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
 {
 	return (ZSTDv03_Dctx*)ZSTD_createDCtx();

lib/legacy/zstd_v03.h

@@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
 /**
+ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
+    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    return : the number of bytes that would be read to decompress this frame
+             or an errorCode if it fails (which can be tested using ZSTDv03_isError())
+*/
+size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
+
+    /**
 ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
 */
 unsigned ZSTDv03_isError(size_t code);

lib/legacy/zstd_v04.c

@@ -3012,21 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     /* Literal length */
     litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
     prevOffset = litLength ? seq->offset : seqState->prevOffset;
-    if (litLength == MaxLL)
-    {
+    if (litLength == MaxLL) {
         U32 add = *dumps++;
         if (add < 255) litLength += add;
-        else
-        {
-            litLength = MEM_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+        else {
+            litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
             dumps += 3;
         }
-        if (dumps >= de) dumps = de-1;   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { litLength = MaxLL+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
 
     /* Offset */
-    {
-        static const U32 offsetPrefix[MaxOff+1] = {
+    {   static const U32 offsetPrefix[MaxOff+1] = {
                 1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
                 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
                 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
@@ -3043,16 +3041,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
 
     /* MatchLength */
     matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
-    if (matchLength == MaxML)
-    {
+    if (matchLength == MaxML) {
         U32 add = *dumps++;
         if (add < 255) matchLength += add;
-        else
-        {
-            matchLength = MEM_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+        else {
+            matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
             dumps += 3;
         }
-        if (dumps >= de) dumps = de-1;   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { matchLength = MaxML+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
     matchLength += MINMATCH;
 
@@ -3116,8 +3113,7 @@ static size_t ZSTD_execSequence(BYTE* op,
     /* Requirement: op <= oend_8 */
 
     /* match within prefix */
-    if (sequence.offset < 8)
-    {
+    if (sequence.offset < 8) {
         /* close range match, overlap */
         const int sub2 = dec64table[sequence.offset];
         op[0] = match[0];
@@ -3127,9 +3123,7 @@ static size_t ZSTD_execSequence(BYTE* op,
         match += dec32table[sequence.offset];
         ZSTD_copy4(op+4, match);
         match -= sub2;
-    }
-    else
-    {
+    } else {
         ZSTD_copy8(op, match);
     }
     op += 8; match += 8;
@@ -3332,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
     return op-ostart;
 }
 
+static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+    size_t remainingSize = srcSize;
+    blockProperties_t blockProperties;
+
+    /* Frame Header */
+    if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
+    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+    ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
+
+    /* Loop on each block */
+    while (1)
+    {
+        size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTD_blockHeaderSize;
+        remainingSize -= ZSTD_blockHeaderSize;
+        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        if (cBlockSize == 0) break;   /* bt_end */
+
+        ip += cBlockSize;
+        remainingSize -= cBlockSize;
+    }
+
+    return ip - (const BYTE*)src;
+}
 
 /* ******************************
 *  Streaming Decompression API
@@ -3759,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
 #endif
 }
 
+size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
+{
+    return ZSTD_findFrameCompressedSize(src, srcSize);
+}
 
 size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
 

lib/legacy/zstd_v04.h

@@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
 size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
+/**
+ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
+    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    return : the number of bytes that would be read to decompress this frame
+             or an errorCode if it fails (which can be tested using ZSTDv04_isError())
+*/
+size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
+
 /**
 ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
 */

lib/legacy/zstd_v05.c

@@ -3230,7 +3230,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
             if (litLength&1) litLength>>=1, dumps += 3;
             else litLength = (U16)(litLength)>>1, dumps += 2;
         }
-        if (dumps >= de) dumps = de-1;   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { litLength = MaxLL+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
 
     /* Offset */
@@ -3263,7 +3264,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
             if (matchLength&1) matchLength>>=1, dumps += 3;
             else matchLength = (U16)(matchLength)>>1, dumps += 2;
         }
-        if (dumps >= de) dumps = de-1;   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { matchLength = MaxML+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
     matchLength += MINMATCH;
 
@@ -3581,6 +3583,35 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
 #endif
 }
 
+size_t ZSTDv05_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+    size_t remainingSize = srcSize;
+    blockProperties_t blockProperties;
+
+    /* Frame Header */
+    if (srcSize < ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong);
+    if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) return ERROR(prefix_unknown);
+    ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
+
+    /* Loop on each block */
+    while (1)
+    {
+        size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
+        if (ZSTDv05_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTDv05_blockHeaderSize;
+        remainingSize -= ZSTDv05_blockHeaderSize;
+        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        if (cBlockSize == 0) break;   /* bt_end */
+
+        ip += cBlockSize;
+        remainingSize -= cBlockSize;
+    }
+
+    return ip - (const BYTE*)src;
+}
 
 /* ******************************
 *  Streaming Decompression API

lib/legacy/zstd_v05.h

@@ -32,6 +32,13 @@ extern "C" {
 size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
                      const void* src, size_t compressedSize);
 
+/**
+ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
+    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    return : the number of bytes that would be read to decompress this frame
+             or an errorCode if it fails (which can be tested using ZSTDv05_isError())
+*/
+size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /* *************************************
 *  Helper functions

lib/legacy/zstd_v06.c

@@ -3729,6 +3729,37 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
+size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+    size_t remainingSize = srcSize;
+    blockProperties_t blockProperties = { bt_compressed, 0 };
+
+    /* Frame Header */
+    {   size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
+        if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
+        if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
+        if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
+        ip += frameHeaderSize; remainingSize -= frameHeaderSize;
+    }
+
+    /* Loop on each block */
+    while (1) {
+        size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
+        if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTDv06_blockHeaderSize;
+        remainingSize -= ZSTDv06_blockHeaderSize;
+        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        if (cBlockSize == 0) break;   /* bt_end */
+
+        ip += cBlockSize;
+        remainingSize -= cBlockSize;
+    }
+
+    return ip - (const BYTE*)src;
+}
 
 /*_******************************
 *  Streaming Decompression API
@@ -4077,7 +4108,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
                     zbd->inBuff = (char*)malloc(blockSize);
                     if (zbd->inBuff == NULL) return ERROR(memory_allocation);
                 }
-                {   size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
+                {   size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize + WILDCOPY_OVERLENGTH * 2;
                     if (zbd->outBuffSize < neededOutSize) {
                         free(zbd->outBuff);
                         zbd->outBuffSize = neededOutSize;

lib/legacy/zstd_v06.h

@@ -41,6 +41,13 @@ extern "C" {
 ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
                                     const void* src, size_t compressedSize);
 
+/**
+ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
+    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    return : the number of bytes that would be read to decompress this frame
+             or an errorCode if it fails (which can be tested using ZSTDv06_isError())
+*/
+size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /* *************************************
 *  Helper functions

lib/legacy/zstd_v07.c

@@ -13,12 +13,14 @@
 #include <string.h>     /* memcpy */
 #include <stdlib.h>     /* malloc, free, qsort */
 
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"      /* XXH64_* */
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY    /* XXH64_state_t */
+#endif
+#include "xxhash.h"                  /* XXH64_* */
 #include "zstd_v07.h"
 
-#define FSEv07_STATIC_LINKING_ONLY  /* FSEv07_MIN_TABLELOG */
-#define HUFv07_STATIC_LINKING_ONLY  /* HUFv07_TABLELOG_ABSOLUTEMAX */
+#define FSEv07_STATIC_LINKING_ONLY   /* FSEv07_MIN_TABLELOG */
+#define HUFv07_STATIC_LINKING_ONLY   /* HUFv07_TABLELOG_ABSOLUTEMAX */
 #define ZSTDv07_STATIC_LINKING_ONLY
 
 #include "error_private.h"
@@ -3968,6 +3970,41 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
+size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+    size_t remainingSize = srcSize;
+
+    /* check */
+    if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
+
+    /* Frame Header */
+    {   size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
+        if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
+        if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown);
+        if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
+        ip += frameHeaderSize; remainingSize -= frameHeaderSize;
+    }
+
+    /* Loop on each block */
+    while (1) {
+        blockProperties_t blockProperties;
+        size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
+        if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTDv07_blockHeaderSize;
+        remainingSize -= ZSTDv07_blockHeaderSize;
+
+        if (blockProperties.blockType == bt_end) break;
+
+        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        ip += cBlockSize;
+        remainingSize -= cBlockSize;
+    }
+
+    return ip - (const BYTE*)src;
+}
 
 /*_******************************
 *  Streaming Decompression API
@@ -4134,9 +4171,9 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si
     }
 
     if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
-    dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
-    dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
-    dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
+    dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
+    dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
+    dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
     dictPtr += 12;
 
     dctx->litEntropy = dctx->fseEntropy = 1;
@@ -4448,7 +4485,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
                     zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
                     if (zbd->inBuff == NULL) return ERROR(memory_allocation);
                 }
-                {   size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
+                {   size_t const neededOutSize = zbd->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
                     if (zbd->outBuffSize < neededOutSize) {
                         zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
                         zbd->outBuffSize = neededOutSize;
@@ -4501,7 +4538,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
                     if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; }   /* this was just a header */
                     zbd->outEnd = zbd->outStart +  decodedSize;
                     zbd->stage = ZBUFFds_flush;
-                    // break; /* ZBUFFds_flush follows */
+                    /* break; */
+                    /* pass-through */
             }   }
 
         case ZBUFFds_flush:

lib/legacy/zstd_v07.h

@@ -48,6 +48,14 @@ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize);
 ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
                                     const void* src, size_t compressedSize);
 
+/**
+ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
+    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    return : the number of bytes that would be read to decompress this frame
+             or an errorCode if it fails (which can be tested using ZSTDv07_isError())
+*/
+size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t compressedSize);
+
 /*======  Helper functions  ======*/
 ZSTDLIBv07_API unsigned    ZSTDv07_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code);     /*!< provides readable string from an error code */

lib/zstd.h

@@ -20,13 +20,16 @@ extern "C" {
 
 /* =====   ZSTDLIB_API : control library symbols visibility   ===== */
 #if defined(__GNUC__) && (__GNUC__ >= 4)
-#  define ZSTDLIB_API __attribute__ ((visibility ("default")))
-#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZSTDLIB_API __declspec(dllexport)
-#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
 #else
-#  define ZSTDLIB_API
+#  define ZSTDLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDLIB_API ZSTDLIB_VISIBILITY
 #endif
 
 
@@ -36,7 +39,7 @@ extern "C" {
   zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
   at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
   decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
-  Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
@@ -53,7 +56,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    1
-#define ZSTD_VERSION_RELEASE  2
+#define ZSTD_VERSION_RELEASE  4
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -77,7 +80,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
                                   int compressionLevel);
 
 /*! ZSTD_decompress() :
-    `compressedSize` : must be the _exact_ size of a single compressed frame.
+    `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
@@ -86,6 +89,15 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                               const void* src, size_t compressedSize);
 
 /*! ZSTD_getDecompressedSize() :
+*   NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
+*   ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
+*   frame, but distinguishes empty frames from frames with an unknown size, or errors.
+*
+*   Additionally, ZSTD_findDecompressedSize can be used instead.  It can handle multiple
+*   concatenated frames in one buffer, and so is more general.
+*   As a result however, it requires more computation and entire frames to be passed to it,
+*   as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
+*
 *   'src' is the start of a zstd compressed frame.
 *   @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
 *    note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
@@ -127,7 +139,11 @@ ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
     Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
-/*= Decompression context */
+/*= Decompression context
+*   When decompressing many times,
+*   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+*   This will make workload friendlier for system's memory.
+*   Use one context per thread for parallel execution in multi-threaded environments. */
 typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
@@ -170,8 +186,8 @@ typedef struct ZSTD_CDict_s ZSTD_CDict;
 *   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
 *   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
 *   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-*   `dict` can be released after ZSTD_CDict creation. */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+*   `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
 /*! ZSTD_freeCDict() :
 *   Function frees memory allocated by ZSTD_createCDict(). */
@@ -191,8 +207,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
 
 /*! ZSTD_createDDict() :
 *   Create a digested dictionary, ready to start decompression operation without startup delay.
-*   `dict` can be released after creation. */
-ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+*   dictBuffer can be released after DDict creation, as its content is copied inside DDict */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
 
 /*! ZSTD_freeDDict() :
 *   Function frees memory allocated with ZSTD_createDDict() */
@@ -265,9 +281,11 @@ typedef struct ZSTD_outBuffer_s {
 * *******************************************************************/
 
 typedef struct ZSTD_CStream_s ZSTD_CStream;
+/*===== ZSTD_CStream management functions =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
 
+/*===== Streaming compression functions =====*/
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -301,9 +319,11 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 * *******************************************************************************/
 
 typedef struct ZSTD_DStream_s ZSTD_DStream;
+/*===== ZSTD_DStream management functions =====*/
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 
+/*===== Streaming decompression functions =====*/
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
@@ -325,12 +345,15 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
  * ***************************************************************************************/
 
 /* --- Constants ---*/
-#define ZSTD_MAGICNUMBER            0xFD2FB528   /* v0.8 */
+#define ZSTD_MAGICNUMBER            0xFD2FB528   /* >= v0.8.0 */
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
 
-#define ZSTD_WINDOWLOG_MAX_32  25
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
+#define ZSTD_WINDOWLOG_MAX_32  27
 #define ZSTD_WINDOWLOG_MAX_64  27
-#define ZSTD_WINDOWLOG_MAX    ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MAX    ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
 #define ZSTD_WINDOWLOG_MIN     10
 #define ZSTD_HASHLOG_MAX       ZSTD_WINDOWLOG_MAX
 #define ZSTD_HASHLOG_MIN        6
@@ -345,8 +368,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 #define ZSTD_TARGETLENGTH_MAX 999
 
 #define ZSTD_FRAMEHEADERSIZE_MAX 18    /* for static allocation */
+#define ZSTD_FRAMEHEADERSIZE_MIN  6
 static const size_t ZSTD_frameHeaderSize_prefix = 5;
-static const size_t ZSTD_frameHeaderSize_min = 6;
+static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
 static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
 static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable frame length */
 
@@ -365,9 +389,9 @@ typedef struct {
 } ZSTD_compressionParameters;
 
 typedef struct {
-    unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
-    unsigned checksumFlag;    /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
-    unsigned noDictIDFlag;    /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
+    unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+    unsigned checksumFlag;    /**< 1: generate a 32-bits checksum at end of frame, for error detection */
+    unsigned noDictIDFlag;    /**< 1: no dictID will be saved into frame header (if dictionary compression) */
 } ZSTD_frameParameters;
 
 typedef struct {
@@ -380,6 +404,54 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
 
+/***************************************
+*  Compressed size functions
+***************************************/
+
+/*! ZSTD_findFrameCompressedSize() :
+ *  `src` should point to the start of a ZSTD encoded frame or skippable frame
+ *  `srcSize` must be at least as large as the frame
+ *  @return : the compressed size of the frame pointed to by `src`, suitable to pass to
+ *      `ZSTD_decompress` or similar, or an error code if given invalid input. */
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
+/***************************************
+*  Decompressed size functions
+***************************************/
+/*! ZSTD_getFrameContentSize() :
+*   `src` should point to the start of a ZSTD encoded frame
+*   `srcSize` must be at least as large as the frame header.  A value greater than or equal
+*       to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
+*   @return : decompressed size of the frame pointed to be `src` if known, otherwise
+*             - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+*             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+
+/*! ZSTD_findDecompressedSize() :
+*   `src` should point the start of a series of ZSTD encoded and/or skippable frames
+*   `srcSize` must be the _exact_ size of this series
+*       (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
+*   @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
+*             - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+*             - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+*
+*    note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+*             When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+*             In which case, it's necessary to use streaming mode to decompress data.
+*             Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+*             (For example, data may be necessarily cut into blocks <= 16 KB).
+*    note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+*    note 3 : decompressed size can be very large (64-bits value),
+*             potentially larger than what local system can handle as a single memory segment.
+*             In which case, it's necessary to use streaming mode to decompress data.
+*    note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+*             Always ensure result fits within application's authorized limits.
+*             Each application can set its own limits.
+*    note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+*             read each contained frame header.  This is efficient as most of the data is skipped,
+*             however it does mean that all frame data must be present and valid. */
+ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+
 
 /***************************************
 *  Advanced compression functions
@@ -397,9 +469,24 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
  *  Gives the amount of memory used by a given ZSTD_CCtx */
 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 
+typedef enum {
+    ZSTD_p_forceWindow,   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
+    ZSTD_p_forceRawDict   /* Force loading dictionary in "content-only" mode (no header analysis) */
+} ZSTD_CCtxParameter;
+/*! ZSTD_setCCtxParameter() :
+ *  Set advanced parameters, selected through enum ZSTD_CCtxParameter
+ *  @result : 0, or an error code (which can be tested with ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
+
+/*! ZSTD_createCDict_byReference() :
+ *  Create a digested dictionary for compression
+ *  Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ *  It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
 /*! ZSTD_createCDict_advanced() :
  *  Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
                                                   ZSTD_parameters params, ZSTD_customMem customMem);
 
 /*! ZSTD_sizeof_CDict() :
@@ -455,6 +542,17 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
  *  Gives the amount of memory used by a given ZSTD_DCtx */
 ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
 
+/*! ZSTD_createDDict_byReference() :
+ *  Create a digested dictionary, ready to start decompression operation without startup delay.
+ *  Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ *  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_createDDict_advanced() :
+ *  Create a ZSTD_DDict using external alloc and free, optionally by reference */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+                                                  unsigned byReference, ZSTD_customMem customMem);
+
 /*! ZSTD_sizeof_DDict() :
  *  Gives the amount of memory used by a given ZSTD_DDict */
 ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
@@ -463,13 +561,13 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  *  Provides the dictID stored within dictionary.
  *  if @return == 0, the dictionary is not conformant with Zstandard specification.
  *  It can still be loaded, but as a content-only dictionary. */
-unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
 
 /*! ZSTD_getDictID_fromDDict() :
  *  Provides the dictID of the dictionary loaded into `ddict`.
  *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
  *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
 
 /*! ZSTD_getDictID_fromFrame() :
  *  Provides the dictID required to decompressed the frame stored within `src`.
@@ -481,7 +579,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
  *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
  *  - This is not a Zstandard frame.
  *  When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
-unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 
 /********************************************************************
@@ -490,19 +588,19 @@ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 /*=====   Advanced Streaming compression functions  =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct */
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
+ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
 ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 
 
 /*=====   Advanced Streaming decompression functions  =====*/
-typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
 ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
@@ -542,17 +640,18 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
     In which case, it will "discard" the relevant memory section from its history.
 
   Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
-  It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
-  Without last block mark, frames will be considered unfinished (broken) by decoders.
+  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+  Without last block mark, frames will be considered unfinished (corrupted) by decoders.
 
-  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
 */
 
 /*=====   Buffer-less streaming compression functions  =====*/
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
+ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
 ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
@@ -612,6 +711,9 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   c) Frame Content - any content (User Data) of length equal to Frame Size
   For skippable frames ZSTD_decompressContinue() always returns 0.
   For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+    Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
+           For purposes of decompression, it is valid in both cases to skip the frame using
+           ZSTD_findFrameCompressedSize to find its size in bytes.
   It also returns Frame Size as fparamsPtr->frameContentSize.
 */
 

programs/.gitignore

@@ -8,6 +8,7 @@ zstd-decompress
 *.o
 *.ko
 default.profraw
+have_zlib
 
 # Executables
 *.exe

programs/BUCK

@@ -0,0 +1,63 @@
+cxx_binary(
+    name='zstd',
+    headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
+    srcs=glob(['*.c'], excludes=['datagen.c']),
+    deps=[
+        ':datagen',
+        ':util',
+        '//lib:zstd',
+        '//lib:zdict',
+        '//lib:mem',
+        '//lib:xxhash',
+    ],
+)
+
+cxx_binary(
+    name='zstdmt',
+    headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
+    srcs=glob(['*.c'], excludes=['datagen.c']),
+    deps=[
+        ':datagen',
+        ':util',
+        '//lib:zstd',
+        '//lib:zdict',
+        '//lib:mem',
+        '//lib:xxhash',
+    ],
+    preprocessor_flags=['-DZSTD_MULTITHREAD'],
+    linker_flags=['-lpthread'],
+)
+
+cxx_binary(
+    name='gzstd',
+    headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
+    srcs=glob(['*.c'], excludes=['datagen.c']),
+    deps=[
+        ':datagen',
+        ':util',
+        '//lib:zstd',
+        '//lib:zdict',
+        '//lib:mem',
+        '//lib:xxhash',
+    ],
+    preprocessor_flags=['-DZSTD_GZDECOMPRESS'],
+    linker_flags=['-lz'],
+)
+
+cxx_library(
+    name='datagen',
+    visibility=['PUBLIC'],
+    header_namespace='',
+    exported_headers=['datagen.h'],
+    srcs=['datagen.c'],
+    deps=['//lib:mem'],
+)
+
+
+cxx_library(
+    name='util',
+    visibility=['PUBLIC'],
+    header_namespace='',
+    exported_headers=['util.h', 'platform.h'],
+    deps=['//lib:mem'],
+)

programs/Makefile

@@ -1,7 +1,9 @@
 # ##########################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
+# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+#
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
@@ -14,11 +16,6 @@
 # zstd-decompress : decompressor-only version of zstd
 # ##########################################################################
 
-DESTDIR?=
-PREFIX ?= /usr/local
-BINDIR  = $(PREFIX)/bin
-MANDIR  = $(PREFIX)/share/man/man1
-
 ZSTDDIR = ../lib
 
 ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version "), 1)
@@ -27,31 +24,36 @@ else
 ALIGN_LOOP =
 endif
 
-CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+           -I$(ZSTDDIR)/dictBuilder \
+           -DXXH_NAMESPACE=ZSTD_   # because xxhash.o already compiled with this macro from library
 CFLAGS  ?= -O3
-CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-          -Wpointer-arith
-CFLAGS  += $(MOREFLAGS)
+DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+          -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+          -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
+ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
 ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
 
-ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
-CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
+ZSTD_LEGACY_SUPPORT ?= 4
 ZSTDLEGACY_FILES:=
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+	ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+endif
+	CPPFLAGS += -I$(ZSTDDIR)/legacy
 else
-ZSTD_LEGACY_SUPPORT:=1
-CPPFLAGS  += -I$(ZSTDDIR)/legacy
-ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c
 endif
 
+ZSTDLIB_FILES := $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))
+ZSTDLIB_OBJ   := $(patsubst %.c,%.o,$(ZSTDLIB_FILES))
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
@@ -67,27 +69,58 @@ else
 EXT =
 endif
 
+# zlib detection
+NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
+VOID = /dev/null
+HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
+ifeq ($(HAVE_ZLIB), 1)
+ZLIB_MSG := ==> building zstd with .gz compression support
+ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
+ZLIBLD = -lz
+else
+ZLIB_MSG := $(NO_ZLIB_MSG)
+endif
+# lzma detection
+NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
+HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(){}' | $(CC) -o have_lzma -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
+ifeq ($(HAVE_LZMA), 1)
+LZMA_MSG := ==> building zstd with .xz/.lzma compression support
+LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
+LZMALD = -llzma
+else
+LZMA_MSG := $(NO_LZMA_MSG)
+endif
+
 
 .PHONY: default all clean clean_decomp_o install uninstall generate_res
 
-default: zstd
+default: zstd-release
 
 all: zstd
 
 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
 
-zstd  : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
-        zstdcli.c fileio.c bench.c datagen.c dibio.c
+zstd : CPPFLAGS += $(ZLIBCPP)
+zstd : LDFLAGS += $(ZLIBLD)
+zstd : LZMA_MSG := $(NO_LZMA_MSG)
+zstd-nogz : ZLIB_MSG := $(NO_ZLIB_MSG)
+zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG)
+xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP)
+xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD)
+zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+	@echo "$(ZLIB_MSG)"
+	@echo "$(LZMA_MSG)"
 ifneq (,$(filter Windows%,$(OS)))
 	windres/generate_res.bat
 endif
-	$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
+	$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
 
+zstd-release: DEBUGFLAGS :=
+zstd-release: zstd
 
 zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd32 : $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
-        zstdcli.c fileio.c bench.c datagen.c dibio.c
+zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
 ifneq (,$(filter Windows%,$(OS)))
 	windres/generate_res.bat
 endif
@@ -109,61 +142,74 @@ zstd-pgo : clean zstd
 	$(RM) $(ZSTDDECOMP_O)
 	$(MAKE) zstd MOREFLAGS=-fprofile-use
 
-zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
+zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT)
 
-zstd-small: clean_decomp_o
-	ZSTD_LEGACY_SUPPORT=0 CFLAGS="-Os -s" $(MAKE) zstd-frugal
+zstd-small:
+	CFLAGS="-Os -s" $(MAKE) zstd-frugal
 
-zstd-decompress-clean: $(ZSTDDECOMP_O) $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
-	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT)
-
-zstd-decompress: clean_decomp_o
-	ZSTD_LEGACY_SUPPORT=0 $(MAKE) zstd-decompress-clean
+zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
 
 zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
 
-gzstd: clean_decomp_o
-	@echo "int main(){}" | $(CC) -o have_zlib -x c - -lz && echo found zlib || echo did not found zlib
-	@if [ -s have_zlib ]; then \
-        echo building gzstd with .gz decompression support \
-        && rm have_zlib$(EXT) \
-        && CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd; \
-    else \
-        echo "WARNING : no zlib, building gzstd with only .zst files support : NO .gz SUPPORT !!!" \
-        && $(MAKE) zstd; \
-    fi
+zstdmt: CPPFLAGS += -DZSTD_MULTITHREAD
+ifeq (,$(filter Windows%,$(OS)))
+zstdmt: LDFLAGS += -lpthread
+endif
+zstdmt: zstd
 
 generate_res:
 	windres/generate_res.bat
 
 clean:
-	$(MAKE) -C ../lib clean
+	$(MAKE) -C $(ZSTDDIR) clean
 	@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
 	@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
         zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
-        *.gcda default.profraw
+        *.gcda default.profraw have_zlib$(EXT)
 	@echo Cleaning completed
 
 clean_decomp_o:
 	@$(RM) $(ZSTDDECOMP_O)
 
 
-#----------------------------------------------------------------------------------
-#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
-#----------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD))
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+PREFIX  ?= /usr/local
+DESTDIR ?=
+BINDIR  ?= $(PREFIX)/bin
+
+ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
+MANDIR  ?= $(PREFIX)/man/man1
+else
+MANDIR  ?= $(PREFIX)/share/man/man1
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL) -m 755
+INSTALL_SCRIPT  ?= $(INSTALL) -m 755
+INSTALL_MAN     ?= $(INSTALL) -m 644
+
 install: zstd
 	@echo Installing binaries
-	@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
-	@install -m 755 zstd $(DESTDIR)$(BINDIR)/zstd
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
+	@$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
-	@install -m 755 zstdless $(DESTDIR)$(BINDIR)/zstdless
-	@install -m 755 zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
+	@$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
+	@$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
 	@echo Installing man pages
-	@install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
+	@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
 	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
 	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
 	@echo zstd installation completed

programs/bench.c

@@ -8,13 +8,31 @@
  */
 
 
+
+/* **************************************
+*  Tuning parameters
+****************************************/
+#ifndef BMK_TIMETEST_DEFAULT_S   /* default minimum time per test */
+#define BMK_TIMETEST_DEFAULT_S 3
+#endif
+
+
+/* **************************************
+*  Compiler Warnings
+****************************************/
+#ifdef _MSC_VER
+#  pragma warning(disable : 4127)                /* disable: C4127: conditional expression is constant */
+#endif
+
+
 /* *************************************
 *  Includes
 ***************************************/
-#include "util.h"        /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
+#include "platform.h"    /* Large Files support */
+#include "util.h"        /* UTIL_getFileSize, UTIL_sleep */
 #include <stdlib.h>      /* malloc, free */
 #include <string.h>      /* memset */
-#include <stdio.h>       /* fprintf, fopen, ftello64 */
+#include <stdio.h>       /* fprintf, fopen */
 #include <time.h>        /* clock_t, clock, CLOCKS_PER_SEC */
 
 #include "mem.h"
@@ -22,6 +40,7 @@
 #include "zstd.h"
 #include "datagen.h"     /* RDG_genBuffer */
 #include "xxhash.h"
+#include "zstdmt_compress.h"
 
 
 /* *************************************
@@ -33,7 +52,6 @@
 #  define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
 #endif
 
-#define NBSECONDS             3
 #define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
 #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
 #define COOLPERIOD_SEC        10
@@ -82,8 +100,6 @@ static clock_t g_time = 0;
 /* *************************************
 *  Benchmark Parameters
 ***************************************/
-static U32 g_nbSeconds = NBSECONDS;
-static size_t g_blockSize = 0;
 static int g_additionalParam = 0;
 static U32 g_decodeOnly = 0;
 
@@ -91,19 +107,30 @@ void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
 void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
 
-void BMK_SetNbSeconds(unsigned nbSeconds)
+static U32 g_nbSeconds = BMK_TIMETEST_DEFAULT_S;
+void BMK_setNbSeconds(unsigned nbSeconds)
 {
     g_nbSeconds = nbSeconds;
-    DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
+    DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression - \n", g_nbSeconds);
 }
 
-void BMK_SetBlockSize(size_t blockSize)
+static size_t g_blockSize = 0;
+void BMK_setBlockSize(size_t blockSize)
 {
     g_blockSize = blockSize;
-    DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
+    if (g_blockSize) DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
+}
+
+void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); }
+
+static U32 g_nbThreads = 1;
+void BMK_setNbThreads(unsigned nbThreads) {
+#ifndef ZSTD_MULTITHREAD
+    if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
+#endif
+    g_nbThreads = nbThreads;
 }
 
-void BMK_setDecodeOnly(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); }
 
 /* ********************************************************
 *  Bench functions
@@ -125,7 +152,8 @@ typedef struct {
 static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const char* displayName, int cLevel,
                         const size_t* fileSizes, U32 nbFiles,
-                        const void* dictBuffer, size_t dictBufferSize)
+                        const void* dictBuffer, size_t dictBufferSize,
+                        ZSTD_compressionParameters *comprParams)
 {
     size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
     size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
@@ -134,13 +162,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
     void* const compressedBuffer = malloc(maxCompressedSize);
     void* resultBuffer = malloc(srcSize);
+    ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
     ZSTD_CCtx* const ctx = ZSTD_createCCtx();
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
     size_t const loadedCompressedSize = srcSize;
     size_t cSize = 0;
     double ratio = 0.;
     U32 nbBlocks;
-    UTIL_time_t ticksPerSecond;
+    UTIL_freq_t ticksPerSecond;
 
     /* checks */
     if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
@@ -155,7 +184,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         U64 dSize64 = 0;
         U32 fileNb;
         for (fileNb=0; fileNb<nbFiles; fileNb++) {
-            U64 const fSize64 = ZSTD_getDecompressedSize(srcPtr, fileSizes[fileNb]);
+            U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
             if (fSize64==0) EXM_THROW(32, "Impossible to determine original size ");
             dSize64 += fSize64;
             srcPtr += fileSizes[fileNb];
@@ -188,7 +217,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                 blockTable[nbBlocks].cRoom = g_decodeOnly ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
                 blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom;
                 blockTable[nbBlocks].resPtr = (void*)resPtr;
-                blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_getDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
+                blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
                 srcPtr += thisBlockSize;
                 cPtr += blockTable[nbBlocks].cRoom;
                 resPtr += thisBlockSize;
@@ -212,7 +241,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         UTIL_getTime(&coolTime);
         DISPLAYLEVEL(2, "\r%79s\r", "");
         while (!cCompleted || !dCompleted) {
-            UTIL_time_t clockStart;
 
             /* overheat protection */
             if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
@@ -222,6 +250,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             }
 
             if (!g_decodeOnly) {
+                UTIL_time_t clockStart;
                 /* Compression */
                 DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
                 if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
@@ -231,12 +260,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                 UTIL_getTime(&clockStart);
 
                 if (!cCompleted) {   /* still some time to do compression tests */
-                    ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
+                    ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
                     ZSTD_customMem const cmem = { NULL, NULL, NULL };
                     U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
                     U32 nbLoops = 0;
-                    ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
+                    ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
                     if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
+                    if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
+                    if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
+                    if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
+                    if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
+                    if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
+                    if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
+                    if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
                     do {
                         U32 blockNb;
                         size_t rSize;
@@ -247,9 +283,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                                                 blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
                                                 cdict);
                             } else {
-                                rSize = ZSTD_compressCCtx (ctx,
+#ifdef ZSTD_MULTITHREAD         /* note : limitation : MT single-pass does not support compression with dictionary */
+                                rSize = ZSTDMT_compressCCtx(mtctx,
                                                 blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
+                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
+                                                cLevel);
+#else
+                                rSize = ZSTD_compress_advanced (ctx,
+                                                blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
+                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams);
+#endif
                             }
                             if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
                             blockTable[blockNb].cSize = rSize;
@@ -257,9 +300,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         nbLoops++;
                     } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
                     ZSTD_freeCDict(cdict);
-                    {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
-                        if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
-                        totalCTime += clockSpan;
+                    {   U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+                        if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops;
+                        totalCTime += clockSpanMicro;
                         cCompleted = (totalCTime >= maxTime);
                 }   }
 
@@ -274,20 +317,23 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                 memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
             }
 
-            (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
-#if 1
+#if 0       /* disable decompression test */
+            dCompleted=1;
+            (void)totalDTime; (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
+#else
             /* Decompression */
             if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
 
             UTIL_sleepMilli(1); /* give processor time to other processes */
             UTIL_waitForNextTick(ticksPerSecond);
-            UTIL_getTime(&clockStart);
 
             if (!dCompleted) {
                 U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
                 U32 nbLoops = 0;
+                UTIL_time_t clockStart;
                 ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
                 if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
+                UTIL_getTime(&clockStart);
                 do {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
@@ -296,8 +342,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                             blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
                             ddict);
                         if (ZSTD_isError(regenSize)) {
-                            DISPLAY("ZSTD_decompress_usingDDict() failed on block %u : %s  \n",
-                                      blockNb, ZSTD_getErrorName(regenSize));
+                            DISPLAY("ZSTD_decompress_usingDDict() failed on block %u of size %u : %s  \n",
+                                      blockNb, (U32)blockTable[blockNb].cSize, ZSTD_getErrorName(regenSize));
                             clockLoop = 0;   /* force immediate test end */
                             break;
                         }
@@ -306,9 +352,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     nbLoops++;
                 } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
                 ZSTD_freeDDict(ddict);
-                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
-                    if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
-                    totalDTime += clockSpan;
+                {   U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+                    if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops;
+                    totalDTime += clockSpanMicro;
                     dCompleted = (totalDTime >= maxTime);
             }   }
 
@@ -335,6 +381,17 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                             pos = (U32)(u - bacc);
                             bNb = pos / (128 KB);
                             DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
+                            if (u>5) {
+                                int n;
+                                for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
+                                DISPLAY(" :%02X:  ", ((const BYTE*)srcBuffer)[u]);
+                                for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
+                                DISPLAY(" \n");
+                                for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
+                                DISPLAY(" :%02X:  ", ((const BYTE*)resultBuffer)[u]);
+                                for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
+                                DISPLAY(" \n");
+                            }
                             break;
                         }
                         if (u==srcSize-1) {  /* should never happen */
@@ -360,6 +417,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     free(blockTable);
     free(compressedBuffer);
     free(resultBuffer);
+    ZSTDMT_freeCCtx(mtctx);
     ZSTD_freeCCtx(ctx);
     ZSTD_freeDCtx(dctx);
     return 0;
@@ -387,7 +445,8 @@ static size_t BMK_findMaxMem(U64 requiredMem)
 static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
                             const char* displayName, int cLevel, int cLevelLast,
                             const size_t* fileSizes, unsigned nbFiles,
-                            const void* dictBuffer, size_t dictBufferSize)
+                            const void* dictBuffer, size_t dictBufferSize,
+                            ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
 {
     int l;
 
@@ -395,7 +454,10 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     if (!pch) pch = strrchr(displayName, '/'); /* Linux */
     if (pch) displayName = pch+1;
 
-    SET_HIGH_PRIORITY;
+    if (setRealTimePrio) {
+        DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
+        SET_REALTIME_PRIORITY;
+    }
 
     if (g_displayLevel == 1 && !g_additionalParam)
         DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
@@ -406,7 +468,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
         BMK_benchMem(srcBuffer, benchedSize,
                      displayName, l,
                      fileSizes, nbFiles,
-                     dictBuffer, dictBufferSize);
+                     dictBuffer, dictBufferSize, compressionParams);
     }
 }
 
@@ -443,8 +505,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
     if (totalSize == 0) EXM_THROW(12, "no data to bench");
 }
 
-static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
-                               const char* dictFileName, int cLevel, int cLevelLast)
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
+                               int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
 {
     void* srcBuffer;
     size_t benchedSize;
@@ -483,7 +545,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
         BMK_benchCLevel(srcBuffer, benchedSize,
                         displayName, cLevel, cLevelLast,
                         fileSizes, nbFiles,
-                        dictBuffer, dictBufferSize);
+                        dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
     }
 
     /* clean up */
@@ -493,7 +555,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
 }
 
 
-static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
 {
     char name[20] = {0};
     size_t benchedSize = 10000000;
@@ -507,26 +569,27 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
 
     /* Bench */
     snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
-    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
+    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
 
     /* clean up */
     free(srcBuffer);
 }
 
 
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
-                   const char* dictFileName, int cLevel, int cLevelLast)
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
+                   int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
 {
     double const compressibility = (double)g_compressibilityDefault / 100;
 
+    if (cLevel < 1) cLevel = 1;   /* minimum compression level */
     if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
     if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
     if (cLevelLast < cLevel) cLevelLast = cLevel;
     if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
 
     if (nbFiles == 0)
-        BMK_syntheticTest(cLevel, cLevelLast, compressibility);
+        BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
     else
-        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
+        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
     return 0;
 }

programs/bench.h

@@ -12,15 +12,18 @@
 #define BENCH_H_121279284357
 
 #include <stddef.h>   /* size_t */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
+#include "zstd.h"     /* ZSTD_compressionParameters */
 
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
-                   const char* dictFileName, int cLevel, int cLevelLast);
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
+                   int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio);
 
 /* Set Parameters */
-void BMK_SetNbSeconds(unsigned nbLoops);
-void BMK_SetBlockSize(size_t blockSize);
-void BMK_setAdditionalParam(int additionalParam);
+void BMK_setNbSeconds(unsigned nbLoops);
+void BMK_setBlockSize(size_t blockSize);
+void BMK_setNbThreads(unsigned nbThreads);
 void BMK_setNotificationLevel(unsigned level);
-void BMK_setDecodeOnly(unsigned decodeFlag);
+void BMK_setAdditionalParam(int additionalParam);
+void BMK_setDecodeOnlyMode(unsigned decodeFlag);
 
 #endif   /* BENCH_H_121279284357 */

programs/datagen.c

@@ -9,35 +9,16 @@
 
 
 
-/* *************************************
-*  Compiler Options
-***************************************/
-#if defined(_MSC_VER)
-#  define _CRT_SECURE_NO_WARNINGS    /* removes Visual warning on strerror() */
-#  define _CRT_SECURE_NO_DEPRECATE   /* removes VS2005 warning on strerror() */
-#endif
-
 /*-************************************
 *  Dependencies
 **************************************/
+#include "platform.h"  /* SET_BINARY_MODE */
 #include <stdlib.h>    /* malloc, free */
 #include <stdio.h>     /* FILE, fwrite, fprintf */
 #include <string.h>    /* memcpy */
 #include "mem.h"       /* U32 */
 
 
-/*-************************************
-*  OS-specific Includes
-**************************************/
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-#  include <fcntl.h>   /* _O_BINARY */
-#  include <io.h>      /* _setmode, _isatty */
-#  define SET_BINARY_MODE(file) {int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
-#else
-#  define SET_BINARY_MODE(file)
-#endif
-
-
 /*-************************************
 *  Macros
 **************************************/

programs/dibio.c

@@ -9,10 +9,19 @@
 
 
 
+/* **************************************
+*  Compiler Warnings
+****************************************/
+#ifdef _MSC_VER
+#  pragma warning(disable : 4127)                /* disable: C4127: conditional expression is constant */
+#endif
+
+
 /*-*************************************
 *  Includes
 ***************************************/
-#include "util.h"           /* Compiler options, UTIL_GetFileSize, UTIL_getTotalFileSize */
+#include "platform.h"       /* Large Files support */
+#include "util.h"           /* UTIL_getFileSize, UTIL_getTotalFileSize */
 #include <stdlib.h>         /* malloc, free */
 #include <string.h>         /* memset */
 #include <stdio.h>          /* fprintf, fopen, ftello64 */
@@ -31,7 +40,9 @@
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#define MEMMULT 11
+#define SAMPLESIZE_MAX (128 KB)
+#define MEMMULT 11    /* rough estimation : memory cost to analyze 1 byte of sample */
+#define COVER_MEMMULT 9    /* rough estimation : memory cost to analyze 1 byte of sample */
 static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
 
 #define NOISELENGTH 32
@@ -97,7 +108,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
     for (n=0; n<nbFiles; n++) {
         const char* const fileName = fileNamesTable[n];
         unsigned long long const fs64 = UTIL_getFileSize(fileName);
-        size_t const fileSize = (size_t) MIN(fs64, 128 KB);
+        size_t const fileSize = (size_t) MIN(fs64, SAMPLESIZE_MAX);
         if (fileSize > *bufferSizePtr-pos) break;
         {   FILE* const f = fopen(fileName, "rb");
             if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
@@ -108,10 +119,36 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
             fileSizes[n] = fileSize;
             fclose(f);
     }   }
+    DISPLAYLEVEL(2, "\r%79s\r", "");
     *bufferSizePtr = pos;
     return n;
 }
 
+#define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static U32 DiB_rand(U32* src)
+{
+    static const U32 prime1 = 2654435761U;
+    static const U32 prime2 = 2246822519U;
+    U32 rand32 = *src;
+    rand32 *= prime1;
+    rand32 ^= prime2;
+    rand32  = DiB_rotl32(rand32, 13);
+    *src = rand32;
+    return rand32 >> 5;
+}
+
+static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
+  /* Initialize the pseudorandom number generator */
+  U32 seed = 0xFD2FB528;
+  unsigned i;
+  for (i = nbFiles - 1; i > 0; --i) {
+    unsigned const j = DiB_rand(&seed) % (i + 1);
+    const char* tmp = fileNamesTable[j];
+    fileNamesTable[j] = fileNamesTable[i];
+    fileNamesTable[i] = tmp;
+  }
+}
+
 
 /*-********************************************************
 *  Dictionary training functions
@@ -163,6 +200,21 @@ static void DiB_saveDict(const char* dictFileName,
 }
 
 
+static int g_tooLargeSamples = 0;
+static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+    U64 total = 0;
+    unsigned n;
+    for (n=0; n<nbFiles; n++) {
+        U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
+        U64 const cappedFileSize = MIN(fileSize, SAMPLESIZE_MAX);
+        total += cappedFileSize;
+        g_tooLargeSamples |= (fileSize > 2*SAMPLESIZE_MAX);
+    }
+    return total;
+}
+
+
 /*! ZDICT_trainFromBuffer_unsafe() :
     Strictly Internal use only !!
     Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
@@ -177,20 +229,29 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
 
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles,
-                       ZDICT_params_t params)
+                       ZDICT_params_t *params, COVER_params_t *coverParams,
+                       int optimizeCover)
 {
     void* const dictBuffer = malloc(maxDictSize);
     size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
-    unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
-    size_t const maxMem =  DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
-    size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
+    unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles);
+    size_t const memMult = params ? MEMMULT : COVER_MEMMULT;
+    size_t const maxMem =  DiB_findMaxMem(totalSizeToLoad * memMult) / memMult;
+    size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
     void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
     int result = 0;
 
     /* Checks */
+    if (params) g_displayLevel = params->notificationLevel;
+    else if (coverParams) g_displayLevel = coverParams->notificationLevel;
+    else EXM_THROW(13, "Neither dictionary algorith selected");   /* should not happen */
     if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles");   /* should not happen */
-    g_displayLevel = params.notificationLevel;
-    if (nbFiles < 5) {
+    if (g_tooLargeSamples) {
+        DISPLAYLEVEL(2, "!  Warning : some samples are very large \n");
+        DISPLAYLEVEL(2, "!  Note that dictionary is only useful for small files or beginning of large files. \n");
+        DISPLAYLEVEL(2, "!  As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX);
+    }
+    if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) {
         DISPLAYLEVEL(2, "!  Warning : nb of samples too low for proper processing ! \n");
         DISPLAYLEVEL(2, "!  Please provide _one file per sample_. \n");
         DISPLAYLEVEL(2, "!  Do not concatenate samples together into a single file, \n");
@@ -203,12 +264,29 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
         DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
 
     /* Load input buffer */
+    DISPLAYLEVEL(3, "Shuffling input files\n");
+    DiB_shuffle(fileNamesTable, nbFiles);
     nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles);
-    DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH);   /* guard band, for end of buffer condition */
 
-    {   size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
-                            srcBuffer, fileSizes, nbFiles,
-                            params);
+    {
+        size_t dictSize;
+        if (params) {
+            DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH);   /* guard band, for end of buffer condition */
+            dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
+                                                    srcBuffer, fileSizes, nbFiles,
+                                                    *params);
+        } else if (optimizeCover) {
+            dictSize = COVER_optimizeTrainFromBuffer(
+                dictBuffer, maxDictSize, srcBuffer, fileSizes, nbFiles,
+                coverParams);
+            if (!ZDICT_isError(dictSize)) {
+              DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps);
+            }
+        } else {
+            dictSize = COVER_trainFromBuffer(dictBuffer, maxDictSize,
+                                             srcBuffer, fileSizes, nbFiles,
+                                             *coverParams);
+        }
         if (ZDICT_isError(dictSize)) {
             DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize));   /* should not happen */
             result = 1;

programs/dibio.h

@@ -32,7 +32,7 @@
 */
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles,
-                       ZDICT_params_t parameters);
-
+                       ZDICT_params_t *params, COVER_params_t *coverParams,
+                       int optimizeCover);
 
 #endif

programs/fileio.c

@@ -7,11 +7,12 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 
+
 /* *************************************
 *  Compiler Options
 ***************************************/
 #ifdef _MSC_VER   /* Visual */
-#  define _CRT_SECURE_NO_WARNINGS  /* removes Visual warning on strerror() */
+#  pragma warning(disable : 4127)  /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4204)  /* non-constant aggregate initializer */
 #endif
 #if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
@@ -22,7 +23,8 @@
 /*-*************************************
 *  Includes
 ***************************************/
-#include "util.h"       /* Compiler options, UTIL_GetFileSize, _LARGEFILE64_SOURCE */
+#include "platform.h"   /* Large Files support, SET_BINARY_MODE */
+#include "util.h"       /* UTIL_getFileSize */
 #include <stdio.h>      /* fprintf, fopen, fread, _fileno, stdin, stdout */
 #include <stdlib.h>     /* malloc, free */
 #include <string.h>     /* strcmp, strlen */
@@ -33,24 +35,17 @@
 #include "fileio.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
 #include "zstd.h"
-#ifdef ZSTD_GZDECOMPRESS
-#include "zlib.h"
-#if !defined(z_const)
-    #define z_const
+#ifdef ZSTD_MULTITHREAD
+#  include "zstdmt_compress.h"
 #endif
+#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
+#  include <zlib.h>
+#  if !defined(z_const)
+#    define z_const
+#  endif
 #endif
-
-
-/*-*************************************
-*  OS-specific Includes
-***************************************/
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-#  include <fcntl.h>    /* _O_BINARY */
-#  include <io.h>       /* _setmode, _isatty */
-#  define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); }
-#else
-#  include <unistd.h>   /* isatty */
-#  define SET_BINARY_MODE(file)
+#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
+#  include <lzma.h>
 #endif
 
 
@@ -79,30 +74,62 @@
 #define MAX_DICT_SIZE (8 MB)   /* protection against large input (attack scenario) */
 
 #define FNSPACE 30
-#define GZ_EXTENSION ".gz"
 
 
 /*-*************************************
 *  Macros
 ***************************************/
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
 static U32 g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
             if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
             { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stdout); } }
+            if (g_displayLevel>=4) fflush(stdout); } } }
 static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
 static clock_t g_time = 0;
 
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
+/* ************************************************************
+* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
+***************************************************************/
+#if defined(_MSC_VER) && _MSC_VER >= 1400
+#   define LONG_SEEK _fseeki64
+#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
+#  define LONG_SEEK fseeko
+#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__)
+#   define LONG_SEEK fseeko64
+#elif defined(_WIN32) && !defined(__DJGPP__)
+#   include <windows.h>
+    static int LONG_SEEK(FILE* file, __int64 offset, int origin) {
+        LARGE_INTEGER off;
+        DWORD method;
+        off.QuadPart = offset;
+        if (origin == SEEK_END)
+            method = FILE_END;
+        else if (origin == SEEK_CUR)
+            method = FILE_CURRENT;
+        else
+            method = FILE_BEGIN;
+
+        if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method))
+            return 0;
+        else
+            return -1;
+    }
+#else
+#   define LONG_SEEK fseek
+#endif
+
 
 /*-*************************************
 *  Local Parameters - Not thread safe
 ***************************************/
+static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
+void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
 static U32 g_overwrite = 0;
 void FIO_overwriteMode(void) { g_overwrite=1; }
 static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
@@ -115,7 +142,30 @@ static U32 g_removeSrcFile = 0;
 void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
 static U32 g_memLimit = 0;
 void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
-
+static U32 g_nbThreads = 1;
+void FIO_setNbThreads(unsigned nbThreads) {
+#ifndef ZSTD_MULTITHREAD
+    if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
+#endif
+    g_nbThreads = nbThreads;
+}
+static U32 g_blockSize = 0;
+void FIO_setBlockSize(unsigned blockSize) {
+    if (blockSize && g_nbThreads==1)
+        DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
+#ifdef ZSTD_MULTITHREAD
+    if (blockSize-1 < ZSTDMT_SECTION_SIZE_MIN-1)   /* intentional underflow */
+        DISPLAYLEVEL(2, "Note : minimum block size is %u KB \n", (ZSTDMT_SECTION_SIZE_MIN>>10));
+#endif
+    g_blockSize = blockSize;
+}
+#define FIO_OVERLAP_LOG_NOTSET 9999
+static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
+void FIO_setOverlapLog(unsigned overlapLog){
+    if (overlapLog && g_nbThreads==1)
+        DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
+    g_overlapLog = overlapLog;
+}
 
 
 /*-*************************************
@@ -150,6 +200,10 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
         f = stdin;
         SET_BINARY_MODE(stdin);
     } else {
+        if (!UTIL_isRegFile(srcFileName)) {
+            DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName);
+            return NULL;
+        }
         f = fopen(srcFileName, "rb");
         if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
     }
@@ -238,22 +292,36 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
 *  Compression
 ************************************************************************/
 typedef struct {
+    FILE* srcFile;
+    FILE* dstFile;
     void*  srcBuffer;
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
+#ifdef ZSTD_MULTITHREAD
+    ZSTDMT_CCtx* cctx;
+#else
     ZSTD_CStream* cctx;
-    FILE* dstFile;
-    FILE* srcFile;
+#endif
 } cRess_t;
 
-static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize)
-{
+static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
+                                    U64 srcSize, int srcRegFile,
+                                    ZSTD_compressionParameters* comprParams) {
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
+#ifdef ZSTD_MULTITHREAD
+    ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
+    if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+    if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
+        ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);   /* use complete window for overlap */
+    if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
+        ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
+#else
     ress.cctx = ZSTD_createCStream();
     if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+#endif
     ress.srcBufferSize = ZSTD_CStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
     ress.dstBufferSize = ZSTD_CStreamOutSize();
@@ -265,11 +333,24 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 sr
         size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
         if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
         {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
-            params.fParams.contentSizeFlag = 1;
+            params.fParams.contentSizeFlag = srcRegFile;
             params.fParams.checksumFlag = g_checksumFlag;
             params.fParams.noDictIDFlag = !g_dictIDFlag;
+            if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
+            if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
+            if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
+            if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
+            if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
+            if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
+            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
+#ifdef ZSTD_MULTITHREAD
+            {   size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
+                if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
+                ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
+#else
             {   size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
                 if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
+#endif
         }   }
         free(dictBuffer);
     }
@@ -281,17 +362,146 @@ static void FIO_freeCResources(cRess_t ress)
 {
     free(ress.srcBuffer);
     free(ress.dstBuffer);
+#ifdef ZSTD_MULTITHREAD
+    ZSTDMT_freeCCtx(ress.cctx);
+#else
     ZSTD_freeCStream(ress.cctx);   /* never fails */
+#endif
 }
 
 
+#ifdef ZSTD_GZCOMPRESS
+static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize)
+{
+    unsigned long long inFileSize = 0, outFileSize = 0;
+    z_stream strm;
+    int ret;
+
+    if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION;
+
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+
+    ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
+    if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
+
+    strm.next_in = 0;
+    strm.avail_in = 0;
+    strm.next_out = (Bytef*)ress->dstBuffer;
+    strm.avail_out = (uInt)ress->dstBufferSize;
+
+    while (1) {
+        if (strm.avail_in == 0) {
+            size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
+            if (inSize == 0) break;
+            inFileSize += inSize;
+            strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+            strm.avail_in = (uInt)inSize;
+        }
+        ret = deflate(&strm, Z_NO_FLUSH);
+        if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
+        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+            if (decompBytes) {
+                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(73, "Write error : cannot write to output file");
+                outFileSize += decompBytes;
+                strm.next_out = (Bytef*)ress->dstBuffer;
+                strm.avail_out = (uInt)ress->dstBufferSize;
+            }
+        }
+        if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100)
+        else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100);
+    }
+
+    while (1) {
+        ret = deflate(&strm, Z_FINISH);
+        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+            if (decompBytes) {
+                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(75, "Write error : cannot write to output file");
+                outFileSize += decompBytes;
+                strm.next_out = (Bytef*)ress->dstBuffer;
+                strm.avail_out = (uInt)ress->dstBufferSize;
+            }
+        }
+        if (ret == Z_STREAM_END) break;
+        if (ret != Z_BUF_ERROR) EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
+    }
+
+    ret = deflateEnd(&strm);
+    if (ret != Z_OK) EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
+    *readsize = inFileSize;
+
+    return outFileSize;
+}
+#endif
+
+
+#ifdef ZSTD_LZMACOMPRESS
+static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize, int plain_lzma)
+{
+    unsigned long long inFileSize = 0, outFileSize = 0;
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_action action = LZMA_RUN;
+    lzma_ret ret;
+
+    if (compressionLevel < 0) compressionLevel = 0;
+    if (compressionLevel > 9) compressionLevel = 9;
+
+    if (plain_lzma) {
+        lzma_options_lzma opt_lzma;
+        if (lzma_lzma_preset(&opt_lzma, compressionLevel)) EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
+        ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
+        if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
+    } else {
+        ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
+        if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
+    }
+
+    strm.next_in = 0;
+    strm.avail_in = 0;
+    strm.next_out = ress->dstBuffer;
+    strm.avail_out = ress->dstBufferSize;
+
+    while (1) {
+        if (strm.avail_in == 0) {
+            size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
+            if (inSize == 0) action = LZMA_FINISH;
+            inFileSize += inSize;
+            strm.next_in = ress->srcBuffer;
+            strm.avail_in = inSize;
+        }
+
+        ret = lzma_code(&strm, action);
+
+        if (ret != LZMA_OK && ret != LZMA_STREAM_END) EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
+        {   size_t const compBytes = ress->dstBufferSize - strm.avail_out;
+            if (compBytes) {
+                if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) EXM_THROW(73, "Write error : cannot write to output file");
+                outFileSize += compBytes;
+                strm.next_out = ress->dstBuffer;
+                strm.avail_out = ress->dstBufferSize;
+            }
+        }
+        if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100)
+        else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100);
+        if (ret == LZMA_STREAM_END) break;
+    }
+
+    lzma_end(&strm);
+    *readsize = inFileSize;
+
+    return outFileSize;
+}
+#endif
+
+
 /*! FIO_compressFilename_internal() :
  *  same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
  *  @return : 0 : compression completed correctly,
  *            1 : missing or pb opening srcFileName
  */
 static int FIO_compressFilename_internal(cRess_t ress,
-                                         const char* dstFileName, const char* srcFileName)
+                                         const char* dstFileName, const char* srcFileName, int compressionLevel)
 {
     FILE* const srcFile = ress.srcFile;
     FILE* const dstFile = ress.dstFile;
@@ -299,8 +509,34 @@ static int FIO_compressFilename_internal(cRess_t ress,
     U64 compressedfilesize = 0;
     U64 const fileSize = UTIL_getFileSize(srcFileName);
 
+    switch (g_compressionType) {
+        case FIO_zstdCompression:
+            break;
+        case FIO_gzipCompression:
+#ifdef ZSTD_GZCOMPRESS
+            compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+#else
+            (void)compressionLevel;
+            EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName);
+#endif
+            goto finish;
+        case FIO_xzCompression:
+        case FIO_lzmaCompression:
+#ifdef ZSTD_LZMACOMPRESS
+            compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression);
+#else
+            (void)compressionLevel;
+            EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", srcFileName);
+#endif
+            goto finish;
+    }
+
     /* init */
+#ifdef ZSTD_MULTITHREAD
+    {   size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize);
+#else
     {   size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize);
+#endif
         if (ZSTD_isError(resetError)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(resetError));
     }
 
@@ -310,35 +546,49 @@ static int FIO_compressFilename_internal(cRess_t ress,
         size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
         if (inSize==0) break;
         readsize += inSize;
-        DISPLAYUPDATE(2, "\rRead : %u MB  ", (U32)(readsize>>20));
 
-        /* Compress using buffered streaming */
         {   ZSTD_inBuffer  inBuff = { ress.srcBuffer, inSize, 0 };
-            ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
-            { size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
-              if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); }
-            if (inBuff.pos != inBuff.size)
-                /* inBuff should be entirely consumed since buffer sizes are recommended ones */
-                EXM_THROW(24, "Compression error : input block not fully consumed");
+            while (inBuff.pos != inBuff.size) {   /* note : is there any possibility of endless loop ? for example, if outBuff is not large enough ? */
+                ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
+#ifdef ZSTD_MULTITHREAD
+                size_t const result = ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff);
+#else
+                size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
+#endif
+                if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result));
 
-            /* Write cBlock */
-            { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
-              if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
-            compressedfilesize += outBuff.pos;
-        }
-        DISPLAYUPDATE(2, "\rRead : %u MB  ==> %.2f%%   ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
+                /* Write compressed stream */
+                if (outBuff.pos) {
+                    size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
+                    if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
+                    compressedfilesize += outBuff.pos;
+        }   }   }
+#ifdef ZSTD_MULTITHREAD
+        if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
+        else DISPLAYUPDATE(2, "\rRead : %u / %u MB", (U32)(readsize>>20), (U32)(fileSize>>20));
+#else
+        if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(readsize>>20), (double)compressedfilesize/readsize*100)
+        else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(readsize>>20), (U32)(fileSize>>20), (double)compressedfilesize/readsize*100);
+#endif
     }
 
     /* End of Frame */
-    {   ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
-        size_t const result = ZSTD_endStream(ress.cctx, &outBuff);
-        if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
-
-        { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
-          if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
-        compressedfilesize += outBuff.pos;
+    {   size_t result = 1;
+        while (result!=0) {   /* note : is there any possibility of endless loop ? */
+            ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
+#ifdef ZSTD_MULTITHREAD
+            result = ZSTDMT_endStream(ress.cctx, &outBuff);
+#else
+            result = ZSTD_endStream(ress.cctx, &outBuff);
+#endif
+            if (ZSTD_isError(result)) EXM_THROW(26, "Compression error during frame end : %s", ZSTD_getErrorName(result));
+            { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
+              if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
+            compressedfilesize += outBuff.pos;
+        }
     }
 
+finish:
     /* Status */
     DISPLAYLEVEL(2, "\r%79s\r", "");
     DISPLAYLEVEL(2,"%-20s :%6.2f%%   (%6llu => %6llu bytes, %s) \n", srcFileName,
@@ -356,7 +606,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
  *            1 : missing or pb opening srcFileName
  */
 static int FIO_compressFilename_srcFile(cRess_t ress,
-                                        const char* dstFileName, const char* srcFileName)
+                                        const char* dstFileName, const char* srcFileName, int compressionLevel)
 {
     int result;
 
@@ -365,13 +615,17 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
         DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
         return 1;
     }
+
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (!ress.srcFile) return 1;   /* srcFile could not be opened */
 
-    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName);
+    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);
-    if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
+    if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) {
+        if (remove(srcFileName))
+            EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
+    }
     return result;
 }
 
@@ -381,7 +635,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
  *            1 : pb
  */
 static int FIO_compressFilename_dstFile(cRess_t ress,
-                                        const char* dstFileName, const char* srcFileName)
+                                        const char* dstFileName, const char* srcFileName, int compressionLevel)
 {
     int result;
     stat_t statbuf;
@@ -391,7 +645,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
     if (ress.dstFile==NULL) return 1;  /* could not open dstFileName */
 
     if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
-    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName);
+    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel);
 
     if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; }  /* error closing dstFile */
     if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); }  /* remove operation artefact */
@@ -401,13 +655,14 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
 
 
 int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
-                         const char* dictFileName, int compressionLevel)
+                         const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
 {
     clock_t const start = clock();
     U64 const srcSize = UTIL_getFileSize(srcFileName);
+    int const regFile = UTIL_isRegFile(srcFileName);
 
-    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
-    int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
+    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
+    int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
 
     double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
     DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
@@ -419,14 +674,16 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
 
 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
                                   const char* suffix,
-                                  const char* dictFileName, int compressionLevel)
+                                  const char* dictFileName, int compressionLevel,
+                                  ZSTD_compressionParameters* comprParams)
 {
     int missed_files = 0;
     size_t dfnSize = FNSPACE;
     char*  dstFileName = (char*)malloc(FNSPACE);
     size_t const suffixSize = suffix ? strlen(suffix) : 0;
     U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
-    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
+    int const regFile = (nbFiles != 1) ? 0 : UTIL_isRegFile(inFileNamesTable[0]);
+    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
 
     /* init */
     if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
@@ -438,7 +695,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
         ress.dstFile = stdout;
         SET_BINARY_MODE(stdout);
         for (u=0; u<nbFiles; u++)
-            missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u]);
+            missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u], compressionLevel);
         if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close stdout");
     } else {
         unsigned u;
@@ -447,7 +704,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
             if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
             strcpy(dstFileName, inFileNamesTable[u]);
             strcat(dstFileName, suffix);
-            missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u]);
+            missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
     }   }
 
     /* Close & Free */
@@ -484,7 +741,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
     /* Allocation */
     ress.dctx = ZSTD_createDStream();
     if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
-    ZSTD_setDStreamParameter(ress.dctx, ZSTDdsp_maxWindowSize, g_memLimit);
+    ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit);
     ress.srcBufferSize = ZSTD_DStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
     ress.dstBufferSize = ZSTD_DStreamOutSize();
@@ -529,7 +786,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
 
     /* avoid int overflow */
     if (storedSkips > 1 GB) {
-        int const seekResult = fseek(file, 1 GB, SEEK_CUR);
+        int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
         if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
         storedSkips -= 1 GB;
     }
@@ -545,7 +802,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
         storedSkips += (unsigned)(nb0T * sizeof(size_t));
 
         if (nb0T != seg0SizeT) {   /* not all 0s */
-            int const seekResult = fseek(file, storedSkips, SEEK_CUR);
+            int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
             if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
             storedSkips = 0;
             seg0SizeT -= nb0T;
@@ -565,7 +822,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
             for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
             storedSkips += (unsigned) (restPtr - restStart);
             if (restPtr != restEnd) {
-                int seekResult = fseek(file, storedSkips, SEEK_CUR);
+                int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
                 if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
                 storedSkips = 0;
                 {   size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
@@ -578,11 +835,11 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
 static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
 {
     if (storedSkips-->0) {   /* implies g_sparseFileSupport>0 */
-        int const seekResult = fseek(file, storedSkips, SEEK_CUR);
-        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
+        int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
+        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)");
         {   const char lastZeroByte[1] = { 0 };
             size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
-            if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
+            if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero");
     }   }
 }
 
@@ -635,7 +892,7 @@ unsigned long long FIO_decompressFrame(dRess_t* ress,
         if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
 
         /* Write block */
-        storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);       
+        storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
         frameSize += outBuff.pos;
         DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)((alreadyDecoded+frameSize)>>20) );
 
@@ -665,29 +922,85 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
 {
     unsigned long long outFileSize = 0;
     z_stream strm;
+    int flush = Z_NO_FLUSH;
+    int ret;
 
     strm.zalloc = Z_NULL;
     strm.zfree = Z_NULL;
     strm.opaque = Z_NULL;
     strm.next_in = 0;
-    strm.avail_in = Z_NULL;
+    strm.avail_in = 0;
     if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0;  /* see http://www.zlib.net/manual.html */
 
+    strm.next_out = (Bytef*)ress->dstBuffer;
+    strm.avail_out = (uInt)ress->dstBufferSize;
+    strm.avail_in = (uInt)ress->srcBufferLoaded;
+    strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+
+    for ( ; ; ) {
+        if (strm.avail_in == 0) {
+            ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
+            if (ress->srcBufferLoaded == 0) flush = Z_FINISH;
+            strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+            strm.avail_in = (uInt)ress->srcBufferLoaded;
+        }
+        ret = inflate(&strm, flush);
+        if (ret == Z_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+        if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
+        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+            if (decompBytes) {
+                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
+                outFileSize += decompBytes;
+                strm.next_out = (Bytef*)ress->dstBuffer;
+                strm.avail_out = (uInt)ress->dstBufferSize;
+            }
+        }
+        if (ret == Z_STREAM_END) break;
+    }
+
+    if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
+    ress->srcBufferLoaded = strm.avail_in;
+    ret = inflateEnd(&strm);
+    if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d", srcFileName, ret);
+    return outFileSize;
+}
+#endif
+
+
+#ifdef ZSTD_LZMADECOMPRESS
+static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
+{
+    unsigned long long outFileSize = 0;
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_action action = LZMA_RUN;
+    lzma_ret ret;
+
+    strm.next_in = 0;
+    strm.avail_in = 0;
+    if (plain_lzma) {
+        ret = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */
+    } else {
+        ret = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */
+    }
+
+    if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d", srcFileName, ret);
+
     strm.next_out = ress->dstBuffer;
     strm.avail_out = ress->dstBufferSize;
     strm.avail_in = ress->srcBufferLoaded;
-    strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+    strm.next_in = ress->srcBuffer;
 
     for ( ; ; ) {
-        int ret;
         if (strm.avail_in == 0) {
             ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
-            if (ress->srcBufferLoaded == 0) break;
-            strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+            if (ress->srcBufferLoaded == 0) action = LZMA_FINISH;
+            strm.next_in = ress->srcBuffer;
             strm.avail_in = ress->srcBufferLoaded;
         }
-        ret = inflate(&strm, Z_NO_FLUSH);
-        if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
+        ret = lzma_code(&strm, action);
+
+        if (ret == LZMA_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+        if (ret != LZMA_OK && ret != LZMA_STREAM_END) { DISPLAY("zstd: %s: lzma_code decoding error %d \n", srcFileName, ret); return 0; }
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
                 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
@@ -696,12 +1009,12 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
                 strm.avail_out = ress->dstBufferSize;
             }
         }
-        if (ret == Z_STREAM_END) break;
+        if (ret == LZMA_STREAM_END) break;
     }
 
     if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
     ress->srcBufferLoaded = strm.avail_in;
-    inflateEnd(&strm);
+    lzma_end(&strm);
     return outFileSize;
 }
 #endif
@@ -724,7 +1037,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
     }
 
     srcFile = FIO_openSrcFile(srcFileName);
-    if (srcFile==0) return 1;
+    if (srcFile==NULL) return 1;
 
     /* for each frame */
     for ( ; ; ) {
@@ -739,7 +1052,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
         }
         readSomething = 1;   /* there is at least >= 4 bytes in srcFile */
         if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
-        if (buf[0] == 31 && buf[1] == 139) { /* gz header */
+        if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
 #ifdef ZSTD_GZDECOMPRESS
             unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
             if (result == 0) return 1;
@@ -747,6 +1060,16 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 #else
             DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
             return 1;
+#endif
+        } else if ((buf[0] == 0xFD && buf[1] == 0x37)  /* xz magic number */
+                || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
+#ifdef ZSTD_LZMADECOMPRESS
+            unsigned long long const result = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
+            if (result == 0) return 1;
+            filesize += result;
+#else
+            DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName);
+            return 1;
 #endif
         } else {
             if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
@@ -769,7 +1092,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 
     /* Close file */
     if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName);  /* error should never happen */
-    if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); };
+    if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); };
     return 0;
 }
 
@@ -835,32 +1158,31 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
             missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]);
         if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout");
     } else {
-        size_t const suffixSize = strlen(suffix);
-        size_t const gzipSuffixSize = strlen(GZ_EXTENSION);
+        size_t suffixSize;
         size_t dfnSize = FNSPACE;
         unsigned u;
         char* dstFileName = (char*)malloc(FNSPACE);
         if (dstFileName==NULL) EXM_THROW(73, "not enough memory for dstFileName");
         for (u=0; u<nbFiles; u++) {   /* create dstFileName */
             const char* const srcFileName = srcNamesTable[u];
+            const char* const suffixPtr = strrchr(srcFileName, '.');
             size_t const sfnSize = strlen(srcFileName);
-            const char* const suffixPtr = srcFileName + sfnSize - suffixSize;
-            const char* const gzipSuffixPtr = srcFileName + sfnSize - gzipSuffixSize;
+            if (!suffixPtr) {
+                DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", srcFileName);
+                skippedFiles++;
+                continue;
+            }
+            suffixSize = strlen(suffixPtr);
             if (dfnSize+suffixSize <= sfnSize+1) {
                 free(dstFileName);
                 dfnSize = sfnSize + 20;
                 dstFileName = (char*)malloc(dfnSize);
                 if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName");
             }
-            if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
-                if (sfnSize <= gzipSuffixSize || strcmp(gzipSuffixPtr, GZ_EXTENSION) != 0) {
-                    DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s expected) -- ignored \n", srcFileName, suffix, GZ_EXTENSION);
-                    skippedFiles++;
-                    continue;
-                } else {
-                    memcpy(dstFileName, srcFileName, sfnSize - gzipSuffixSize);
-                    dstFileName[sfnSize-gzipSuffixSize] = '\0';
-                }
+            if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION))) {
+                DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s expected) -- ignored \n", srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION);
+                skippedFiles++;
+                continue;
             } else {
                 memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
                 dstFileName[sfnSize-suffixSize] = '\0';

programs/fileio.h

@@ -11,10 +11,14 @@
 #ifndef FILEIO_H_23981798732
 #define FILEIO_H_23981798732
 
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
+#include "zstd.h"                  /* ZSTD_* */
+
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+
 /* *************************************
 *  Special i/o constants
 **************************************/
@@ -25,11 +29,22 @@ extern "C" {
 #else
 #  define nulmark "/dev/null"
 #endif
+#define LZMA_EXTENSION  ".lzma"
+#define XZ_EXTENSION    ".xz"
+#define GZ_EXTENSION    ".gz"
+#define ZSTD_EXTENSION  ".zst"
+
+
+/*-*************************************
+*  Types
+***************************************/
+typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression } FIO_compressionType_t;
 
 
 /*-*************************************
 *  Parameters
 ***************************************/
+void FIO_setCompressionType(FIO_compressionType_t compressionType);
 void FIO_overwriteMode(void);
 void FIO_setNotificationLevel(unsigned level);
 void FIO_setSparseWrite(unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
@@ -37,6 +52,9 @@ void FIO_setDictIDFlag(unsigned dictIDFlag);
 void FIO_setChecksumFlag(unsigned checksumFlag);
 void FIO_setRemoveSrcFile(unsigned flag);
 void FIO_setMemLimit(unsigned memLimit);
+void FIO_setNbThreads(unsigned nbThreads);
+void FIO_setBlockSize(unsigned blockSize);
+void FIO_setOverlapLog(unsigned overlapLog);
 
 
 /*-*************************************
@@ -44,7 +62,8 @@ void FIO_setMemLimit(unsigned memLimit);
 ***************************************/
 /** FIO_compressFilename() :
     @return : 0 == ok;  1 == pb with src file. */
-int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel);
+int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName,
+                          int compressionLevel, ZSTD_compressionParameters* comprParams);
 
 /** FIO_decompressFilename() :
     @return : 0 == ok;  1 == pb with src file. */
@@ -58,7 +77,8 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
     @return : nb of missing files */
 int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
                                   const char* suffix,
-                                  const char* dictFileName, int compressionLevel);
+                                  const char* dictFileName, int compressionLevel,
+                                  ZSTD_compressionParameters* comprParams);
 
 /** FIO_decompressMultipleFilenames() :
     @return : nb of missing or skipped files */

programs/platform.h

@@ -0,0 +1,137 @@
+/**
+ * platform.h - compiler and OS detection
+ *
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef PLATFORM_H_MODULE
+#define PLATFORM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+
+/* **************************************
+*  Compiler Options
+****************************************/
+#if defined(_MSC_VER)
+#  define _CRT_SECURE_NO_WARNINGS   /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
+#  define _CRT_SECURE_NO_DEPRECATE  /* VS2005 - must be declared before <io.h> and <windows.h> */
+#  if (_MSC_VER <= 1800)            /* (1800 = Visual Studio 2013) */
+#    define snprintf sprintf_s      /* snprintf unsupported by Visual <= 2013 */
+#  endif
+#endif
+
+
+/* **************************************
+*  Detect 64-bit OS
+*  http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
+****************************************/
+#if defined __ia64 || defined _M_IA64                                                                               /* Intel Itanium */ \
+  || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__                                                /* POWER 64-bit */  \
+  || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__  /* SPARC 64-bit */  \
+  || defined __x86_64__s || defined _M_X64                                                                          /* x86 64-bit */    \
+  || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__                                           /* ARM 64-bit */    \
+  || (defined __mips  && (__mips == 64 || __mips == 4 || __mips == 3))                                              /* MIPS 64-bit */   \
+  || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */               \
+  || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */
+#  if !defined(__64BIT__)
+#    define __64BIT__  1
+#  endif
+#endif
+
+
+/* *********************************************************
+*  Turn on Large Files support (>4GB) for 32-bit Linux/Unix
+***********************************************************/
+#if !defined(__64BIT__) || defined(__MINGW32__)       /* No point defining Large file for 64 bit but MinGW-w64 requires it */
+#  if !defined(_FILE_OFFSET_BITS)
+#    define _FILE_OFFSET_BITS 64                      /* turn off_t into a 64-bit type for ftello, fseeko */
+#  endif
+#  if !defined(_LARGEFILE_SOURCE)                     /* obsolete macro, replaced with _FILE_OFFSET_BITS */
+#    define _LARGEFILE_SOURCE 1                       /* Large File Support extension (LFS) - fseeko, ftello */
+#  endif
+#  if defined(_AIX) || defined(__hpux)
+#    define _LARGE_FILES                              /* Large file support on 32-bits AIX and HP-UX */
+#  endif
+#endif
+
+
+/* ************************************************************
+*  Detect POSIX version
+*  PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
+*  PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
+*  PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
+***************************************************************/
+#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
+   || defined(__midipix__) || defined(__VMS))
+#  if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
+     || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* BSD distros */
+#    define PLATFORM_POSIX_VERSION 200112L
+#  else
+#    if defined(__linux__) || defined(__linux)
+#      ifndef _POSIX_C_SOURCE
+#        define _POSIX_C_SOURCE 200112L  /* use feature test macro */
+#      endif
+#    endif
+#    include <unistd.h>  /* declares _POSIX_VERSION */
+#    if defined(_POSIX_VERSION)  /* POSIX compliant */
+#      define PLATFORM_POSIX_VERSION _POSIX_VERSION
+#    else
+#      define PLATFORM_POSIX_VERSION 0
+#    endif
+#  endif
+#endif
+#if !defined(PLATFORM_POSIX_VERSION)
+#  define PLATFORM_POSIX_VERSION -1
+#endif
+
+
+/*-*********************************************
+*  Detect if isatty() and fileno() are available
+************************************************/
+#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
+#  include <unistd.h>   /* isatty */
+#  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#  include <io.h>       /* _isatty */
+#  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+#  define IS_CONSOLE(stdStream) 0
+#endif
+
+
+/******************************
+*  OS-specific Includes
+******************************/
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
+#  include <fcntl.h>   /* _O_BINARY */
+#  include <io.h>      /* _setmode, _fileno, _get_osfhandle */
+#  if !defined(__DJGPP__)
+#    include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
+#    include <winioctl.h> /* FSCTL_SET_SPARSE */
+#    define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
+#    define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
+#  else
+#    define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
+#    define SET_SPARSE_FILE_MODE(file)
+#  endif
+#else
+#  define SET_BINARY_MODE(file)
+#  define SET_SPARSE_FILE_MODE(file)
+#endif
+
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* PLATFORM_H_MODULE */

programs/util.h

@@ -1,4 +1,6 @@
 /**
+ * util.h - utility functions
+ * 
  * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
@@ -14,50 +16,71 @@
 extern "C" {
 #endif
 
-/* **************************************
-*  Compiler Options
-****************************************/
-#if defined(__INTEL_COMPILER)
-#  pragma warning(disable : 177)    /* disable: message #177: function was declared but never referenced */
-#endif
-#if defined(_MSC_VER)
-#  define _CRT_SECURE_NO_WARNINGS    /* Disable some Visual warning messages for fopen, strncpy */
-#  define _CRT_SECURE_NO_DEPRECATE   /* VS2005 */
-#  pragma warning(disable : 4127)    /* disable: C4127: conditional expression is constant */
-#if _MSC_VER <= 1800                 /* (1800 = Visual Studio 2013) */
-#  define snprintf sprintf_s       /* snprintf unsupported by Visual <= 2013 */
-#endif
-#endif
-
-
-/* Unix Large Files support (>4GB) */
-#if !defined(__LP64__)                                  /* No point defining Large file for 64 bit */
-#   define _FILE_OFFSET_BITS 64                         /* turn off_t into a 64-bit type for ftello, fseeko */
-#   if defined(__sun__) && !defined(_LARGEFILE_SOURCE)  /* Sun Solaris 32-bits requires specific definitions */
-#      define _LARGEFILE_SOURCE                         /* fseeko, ftello */
-#   elif !defined(_LARGEFILE64_SOURCE)
-#      define _LARGEFILE64_SOURCE                       /* off64_t, fseeko64, ftello64 */
-#   endif
-#endif
 
 
 /*-****************************************
 *  Dependencies
 ******************************************/
-#include <stdlib.h>     /* features.h with _POSIX_C_SOURCE, malloc */
-#include <stdio.h>      /* fprintf */
-#include <sys/types.h>  /* stat, utime */
-#include <sys/stat.h>   /* stat */
+#include "platform.h"     /* PLATFORM_POSIX_VERSION */
+#include <stdlib.h>       /* malloc */
+#include <stddef.h>       /* size_t, ptrdiff_t */
+#include <stdio.h>        /* fprintf */
+#include <sys/types.h>    /* stat, utime */
+#include <sys/stat.h>     /* stat */
 #if defined(_MSC_VER)
-#  include <sys/utime.h>   /* utime */
-#  include <io.h>          /* _chmod */
+#  include <sys/utime.h>  /* utime */
+#  include <io.h>         /* _chmod */
 #else
 #  include <unistd.h>     /* chown, stat */
 #  include <utime.h>      /* utime */
 #endif
-#include <time.h>       /* time */
+#include <time.h>         /* time */
 #include <errno.h>
-#include "mem.h"        /* U32, U64 */
+#include "mem.h"          /* U32, U64 */
+
+
+/* ************************************************************
+* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
+***************************************************************/
+#if defined(_MSC_VER) && (_MSC_VER >= 1400)
+#   define UTIL_fseek _fseeki64
+#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
+#  define UTIL_fseek fseeko
+#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS)
+#   define UTIL_fseek fseeko64
+#else
+#   define UTIL_fseek fseek
+#endif
+
+
+/*-****************************************
+*  Sleep functions: Windows - Posix - others
+******************************************/
+#if defined(_WIN32)
+#  include <windows.h>
+#  define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
+#  define UTIL_sleep(s) Sleep(1000*s)
+#  define UTIL_sleepMilli(milli) Sleep(milli)
+#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
+#  include <unistd.h>
+#  include <sys/resource.h> /* setpriority */
+#  include <time.h>         /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
+#  if defined(PRIO_PROCESS)
+#    define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
+#  else
+#    define SET_REALTIME_PRIORITY /* disabled */
+#  endif
+#  define UTIL_sleep(s) sleep(s)
+#  if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L)  /* nanosleep requires POSIX.1-2001 */
+#      define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
+#  else
+#      define UTIL_sleepMilli(milli) /* disabled */
+#  endif
+#else
+#  define SET_REALTIME_PRIORITY      /* disabled */
+#  define UTIL_sleep(s)          /* disabled */
+#  define UTIL_sleepMilli(milli) /* disabled */
+#endif
 
 
 /* *************************************
@@ -69,6 +92,9 @@ extern "C" {
 /*-****************************************
 *  Compiler specifics
 ******************************************/
+#if defined(__INTEL_COMPILER)
+#  pragma warning(disable : 177)    /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */
+#endif
 #if defined(__GNUC__)
 #  define UTIL_STATIC static __attribute__((unused))
 #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -80,56 +106,44 @@ extern "C" {
 #endif
 
 
-/*-****************************************
-*  Sleep functions: Windows - Posix - others
-******************************************/
-#if defined(_WIN32)
-#  include <windows.h>
-#  define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
-#  define UTIL_sleep(s) Sleep(1000*s)
-#  define UTIL_sleepMilli(milli) Sleep(milli)
-#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__)))
-#  include <unistd.h>
-#  include <sys/resource.h> /* setpriority */
-#  include <time.h>         /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
-#  if defined(PRIO_PROCESS)
-#    define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
-#  else
-#    define SET_HIGH_PRIORITY /* disabled */
-#  endif
-#  define UTIL_sleep(s) sleep(s)
-#  if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L))
-#      define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
-#  else
-#      define UTIL_sleepMilli(milli) /* disabled */
-#  endif
-#else
-#  define SET_HIGH_PRIORITY      /* disabled */
-#  define UTIL_sleep(s)          /* disabled */
-#  define UTIL_sleepMilli(milli) /* disabled */
-#endif
-
-
 /*-****************************************
 *  Time functions
 ******************************************/
-#if !defined(_WIN32)
-   typedef clock_t UTIL_time_t;
-   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; }
-   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
-   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-#else
+#if defined(_WIN32)   /* Windows */
+   typedef LARGE_INTEGER UTIL_freq_t;
    typedef LARGE_INTEGER UTIL_time_t;
-   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
+   UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
    UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
-   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
-   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+#elif defined(__APPLE__) && defined(__MACH__)
+   #include <mach/mach_time.h>
+   typedef mach_timebase_info_data_t UTIL_freq_t;
+   typedef U64 UTIL_time_t;
+   UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* rate) { mach_timebase_info(rate); }
+   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = mach_absolute_time(); }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); }
+#elif (PLATFORM_POSIX_VERSION >= 200112L)
+    #include <sys/times.h>   /* times */
+   typedef U64 UTIL_freq_t;
+   typedef U64 UTIL_time_t;
+   UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); }
+   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
+#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+   typedef clock_t UTIL_freq_t;
+   typedef clock_t UTIL_time_t;
+   UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { *ticksPerSecond=0; }
+   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
 #endif
 
 
 /* returns time span in microseconds */
-UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond )
+UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_freq_t ticksPerSecond )
 {
     UTIL_time_t clockEnd;
     UTIL_getTime(&clockEnd);
@@ -137,7 +151,7 @@ UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPe
 }
 
 
-UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
+UTIL_STATIC void UTIL_waitForNextTick(UTIL_freq_t ticksPerSecond)
 {
     UTIL_time_t clockStart, clockEnd;
     UTIL_getTime(&clockStart);
@@ -153,7 +167,7 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
 ******************************************/
 #if defined(_MSC_VER)
 	#define chmod _chmod
-	typedef struct _stat64 stat_t;
+	typedef struct __stat64 stat_t;
 #else
     typedef struct stat stat_t;
 #endif
@@ -193,13 +207,39 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
 }
 
 
+UTIL_STATIC int UTIL_isRegFile(const char* infilename)
+{
+    stat_t statbuf;
+    return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
+}
+
+
+UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
+{
+    int r;
+    stat_t statbuf;
+#if defined(_MSC_VER)
+    r = _stat64(infilename, &statbuf);
+    if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+    r = stat(infilename, &statbuf);
+    if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+    return 0;
+}
+
+
 UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
 {
     int r;
 #if defined(_MSC_VER)
-    struct _stat64 statbuf;
+    struct __stat64 statbuf;
     r = _stat64(infilename, &statbuf);
     if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
+#elif defined(__MINGW32__) && defined (__MSVCRT__)
+    struct _stati64 statbuf;
+    r = _stati64(infilename, &statbuf);
+    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
 #else
     struct stat statbuf;
     r = stat(infilename, &statbuf);
@@ -219,37 +259,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi
 }
 
 
-UTIL_STATIC int UTIL_doesFileExists(const char* infilename)
-{
-    int r;
-#if defined(_MSC_VER)
-    struct _stat64 statbuf;
-    r = _stat64(infilename, &statbuf);
-    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
-#else
-    struct stat statbuf;
-    r = stat(infilename, &statbuf);
-    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
-#endif
-    return 1;
-}
-
-
-UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
-{
-    int r;
-#if defined(_MSC_VER)
-    struct _stat64 statbuf;
-    r = _stat64(infilename, &statbuf);
-    if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
-#else
-    struct stat statbuf;
-    r = stat(infilename, &statbuf);
-    if (!r && S_ISDIR(statbuf.st_mode)) return 1;
-#endif
-    return 0;
-}
-
 /*
  * A modified version of realloc().
  * If UTIL_realloc() fails the original block is freed.
@@ -270,7 +279,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
 {
     char* path;
     int dirLength, fnameLength, pathLength, nbFiles = 0;
-    WIN32_FIND_DATA cFile;
+    WIN32_FIND_DATAA cFile;
     HANDLE hFile;
 
     dirLength = (int)strlen(dirName);
@@ -282,7 +291,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
     path[dirLength+1] = '*';
     path[dirLength+2] = 0;
 
-    hFile=FindFirstFile(path, &cFile);
+    hFile=FindFirstFileA(path, &cFile);
     if (hFile == INVALID_HANDLE_VALUE) {
         fprintf(stderr, "Cannot open directory '%s'\n", dirName);
         return 0;
@@ -319,18 +328,16 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
             }
         }
         free(path);
-    } while (FindNextFile(hFile, &cFile));
+    } while (FindNextFileA(hFile, &cFile));
 
     FindClose(hFile);
     return nbFiles;
 }
 
-#elif (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) || \
-      (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) || \
-       defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
-      (defined(__linux__) && defined(_POSIX_C_SOURCE)) /* opendir */
+#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L)  /* opendir, readdir require POSIX.1-2001 */
 #  define UTIL_HAS_CREATEFILELIST
 #  include <dirent.h>       /* opendir, readdir */
+#  include <string.h>       /* strerror, memcpy */
 
 UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
 {
@@ -392,7 +399,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
 UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
 {
     (void)bufStart; (void)bufEnd; (void)pos;
-    fprintf(stderr, "Directory %s ignored (zstd compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
+    fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
     return 0;
 }
 

programs/zstd.1

@@ -69,6 +69,8 @@ from standard input if it is a terminal.
 .PP
 Unless
 .B \-\-stdout
+or
+.B \-o
 is specified,
 .I files
 are written to a new file whose name is derived from the source
@@ -159,7 +161,8 @@ No files are created or removed.
  # compression level [1-19] (default:3)
 .TP
 .BR \--ultra
- unlocks high compression levels 20+ (maximum 22), using a lot more memory
+ unlocks high compression levels 20+ (maximum 22), using a lot more memory.
+Note that decompression will also require more memory when using these levels.
 .TP
 .B \-D file
  use `file` as Dictionary to compress or decompress FILE(s)
@@ -248,6 +251,30 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
 .B \-s#
  dictionary selectivity level (default: 9)
  the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size.
+.TP
+.B \--cover=k=#,d=#
+ Use alternate dictionary builder algorithm named cover with parameters \fIk\fR and \fId\fR with \fId\fR <= \fIk\fR.
+ Selects segments of size \fIk\fR with the highest score to put in the dictionary.
+ The score of a segment is computed by the sum of the frequencies of all the subsegments of of size \fId\fR.
+ Generally \fId\fR should be in the range [6, 24].
+ Good values for \fIk\fR vary widely based on the input data, but a safe range is [32, 2048].
+ Example: \fB--train --cover=k=64,d=8 FILEs\fR.
+.TP
+.B \--optimize-cover[=steps=#,k=#,d=#]
+ If \fIsteps\fR is not specified, the default value of 32 is used.
+ If \fIk\fR is not specified, \fIsteps\fR values in [16, 2048] are checked for each value of \fId\fR.
+ If \fId\fR is not specified, the values checked are [6, 8, ..., 16].
+
+ Runs the cover dictionary builder for each parameter set saves the optimal parameters and dictionary.
+ Prints the optimal parameters and writes the optimal dictionary to the output file.
+ Supports multithreading if \fBzstd\fR is compiled with threading support.
+
+ The parameter \fIk\fR is more sensitve than \fId\fR, and is faster to optimize over.
+ Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set.
+ Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024]).
+ \fBzstd --train --optimize-cover FILEs
+ \fBzstd --train --optimize-cover=d=d,steps=512 FILEs
+.TP
 
 .SH BENCHMARK
 .TP
@@ -262,7 +289,117 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
 .TP
 .B \-B#
  cut file into independent blocks of size # (default: no block)
+.B \--priority=rt
+ set process priority to real-time
 
+.SH ADVANCED COMPRESSION OPTIONS
+.TP
+.B \--zstd[=\fIoptions\fR]
+.PD
+\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options.
+The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level.
+The list of available \fIoptions\fR:
+.RS
+
+.TP
+.BI strategy= strat
+.PD 0
+.TP
+.BI strat= strat
+.PD
+Specify a strategy used by a match finder.
+.IP ""
+There are 8 strategies numbered from 0 to 7, from faster to stronger:
+0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2.
+.IP ""
+
+.TP
+.BI windowLog= wlog
+.PD 0
+.TP
+.BI wlog= wlog
+.PD
+Specify the maximum number of bits for a match distance.
+.IP ""
+The higher number of bits increases the chance to find a match what usually improves compression ratio.
+It also increases memory requirements for compressor and decompressor.
+.IP ""
+The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
+.IP ""
+
+.TP
+.BI hashLog= hlog
+.PD 0
+.TP
+.BI hlog= hlog
+.PD
+Specify the maximum number of bits for a hash table.
+.IP ""
+The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression.
+.IP ""
+The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
+
+.TP
+.BI chainLog= clog
+.PD 0
+.TP
+.BI clog= clog
+.PD
+Specify the maximum number of bits for a hash chain or a binary tree.
+.IP ""
+The higher number of bits increases the chance to find a match what usually improves compression ratio.
+It also slows down compression speed and increases memory requirements for compression.
+This option is ignored for the ZSTD_fast strategy.
+.IP ""
+The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation.
+.IP ""
+
+.TP
+.BI searchLog= slog
+.PD 0
+.TP
+.BI slog= slog
+.PD
+Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale.
+.IP ""
+The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed.
+.IP ""
+The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation.
+.IP ""
+
+.TP
+.BI searchLength= slen
+.PD 0
+.TP
+.BI slen= slen
+.PD
+Specify the minimum searched length of a match in a hash table.
+.IP ""
+The bigger search length usually decreases compression ratio but improves decompression speed.
+.IP ""
+The minimum \fIslen\fR is 3 and the maximum is 7.
+.IP ""
+
+.TP
+.BI targetLength= tlen
+.PD 0
+.TP
+.BI tlen= tlen
+.PD
+Specify the minimum match length that causes a match finder to interrupt searching of better matches.
+.IP ""
+The bigger minimum match length usually improves compression ratio but decreases compression speed.
+This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies.
+.IP ""
+The minimum \fItlen\fR is 4 and the maximum is 999.
+.IP ""
+
+.PP
+.B An example
+.br
+The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB:
+.IP ""
+\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6
 
 .SH BUGS
 Report bugs at:- https://github.com/facebook/zstd/issues

programs/zstdcli.c

@@ -20,10 +20,12 @@
 #endif
 
 
+
 /*-************************************
 *  Dependencies
 **************************************/
-#include "util.h"     /* Compiler options, UTIL_HAS_CREATEFILELIST */
+#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
+#include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
 #include <string.h>   /* strcmp, strlen */
 #include <errno.h>    /* errno */
 #include "fileio.h"
@@ -37,21 +39,6 @@
 #include "zstd.h"     /* ZSTD_VERSION_STRING */
 
 
-/*-************************************
-*  OS-specific Includes
-**************************************/
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-#  include <io.h>       /* _isatty */
-#  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
-#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
-      defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
-#  include <unistd.h>   /* isatty */
-#  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
-#else
-#  define IS_CONSOLE(stdStream) 0
-#endif
-
-
 /*-************************************
 *  Constants
 **************************************/
@@ -62,9 +49,13 @@
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
 
-#define ZSTD_EXTENSION ".zst"
-#define ZSTD_CAT "zstdcat"
 #define ZSTD_UNZSTD "unzstd"
+#define ZSTD_CAT "zstdcat"
+#define ZSTD_GZ "gzip"
+#define ZSTD_GUNZIP "gunzip"
+#define ZSTD_GZCAT "gzcat"
+#define ZSTD_LZMA "lzma"
+#define ZSTD_XZ "xz"
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
@@ -76,6 +67,8 @@ static const char*    g_defaultDictName = "dictionary";
 static const unsigned g_defaultMaxDictSize = 110 KB;
 static const int      g_defaultDictCLevel = 3;
 static const unsigned g_defaultSelectivityLevel = 9;
+#define OVERLAP_LOG_DEFAULT 9999
+static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
 
 
 /*-************************************
@@ -124,12 +117,23 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
 #ifdef UTIL_HAS_CREATEFILELIST
-    DISPLAY( " -r     : operate recursively on directories\n");
+    DISPLAY( " -r     : operate recursively on directories \n");
 #endif
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
     DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
-    DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
+    DISPLAY( "--[no-]check : integrity check (default:enabled) \n");
+#ifdef ZSTD_MULTITHREAD
+    DISPLAY( " -T#    : use # threads for compression (default:1) \n");
+    DISPLAY( " -B#    : select size of independent sections (default:0==automatic) \n");
+#endif
+#ifdef ZSTD_GZCOMPRESS
+    DISPLAY( "--format=gzip : compress files to the .gz format \n");
+#endif
+#ifdef ZSTD_LZMACOMPRESS
+    DISPLAY( "--format=xz : compress files to the .xz format \n");
+    DISPLAY( "--format=lzma : compress files to the .lzma format \n");
+#endif
 #endif
 #ifndef ZSTD_NODECOMPRESS
     DISPLAY( "--test  : test compressed file integrity \n");
@@ -141,6 +145,8 @@ static int usage_advanced(const char* programName)
     DISPLAY( "\n");
     DISPLAY( "Dictionary builder :\n");
     DISPLAY( "--train ## : create a dictionary from a training set of files \n");
+    DISPLAY( "--cover=k=#,d=# : use the cover algorithm with parameters k and d \n");
+    DISPLAY( "--optimize-cover[=steps=#,k=#,d=#] : optimize cover parameters with optional parameters\n");
     DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
     DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
     DISPLAY( " -s#    : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
@@ -153,6 +159,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -e#    : test all compression levels from -bX to # (default: 1)\n");
     DISPLAY( " -i#    : minimum evaluation time in seconds (default : 3s)\n");
     DISPLAY( " -B#    : cut file into independent blocks of size # (default: no block)\n");
+    DISPLAY( "--priority=rt : set process priority to real-time\n");
 #endif
     return 0;
 }
@@ -193,7 +200,7 @@ static unsigned readU32FromChar(const char** stringPtr)
 }
 
 /** longCommandWArg() :
- *  check is *stringPtr is the same as longCommand.
+ *  check if *stringPtr is the same as longCommand.
  *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
  *  @return 0 and doesn't modify *stringPtr otherwise.
  */
@@ -205,6 +212,56 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
     return result;
 }
 
+
+#ifndef ZSTD_NODICT
+/**
+ * parseCoverParameters() :
+ * reads cover parameters from *stringPtr (e.g. "--cover=smoothing=100,kmin=48,kstep=4,kmax=64,d=8") into *params
+ * @return 1 means that cover parameters were correct
+ * @return 0 in case of malformed parameters
+ */
+static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *params)
+{
+    memset(params, 0, sizeof(*params));
+    for (; ;) {
+        if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        return 0;
+    }
+    if (stringPtr[0] != 0) return 0;
+    DISPLAYLEVEL(4, "k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
+    return 1;
+}
+#endif
+
+
+/** parseCompressionParameters() :
+ *  reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
+ *  @return 1 means that compression parameters were correct
+ *  @return 0 in case of malformed parameters
+ */
+static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
+{
+    for ( ; ;) {
+        if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        return 0;
+    }
+
+    if (stringPtr[0] != 0) return 0; /* check the end of string */
+    DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
+    DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy);
+    return 1;
+}
+
+
 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
@@ -221,8 +278,13 @@ int main(int argCount, const char* argv[])
         nextArgumentIsDictID=0,
         nextArgumentsAreFiles=0,
         ultra=0,
-        lastCommand = 0;
+        lastCommand = 0,
+        nbThreads = 1,
+        setRealTimePrio = 0;
+    unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
+    size_t blockSize = 0;
     zstd_operation_mode operation = zom_compress;
+    ZSTD_compressionParameters compressionParams;
     int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
     int cLevelLast = 1;
     unsigned recursive = 0;
@@ -232,6 +294,7 @@ int main(int argCount, const char* argv[])
     const char* programName = argv[0];
     const char* outFileName = NULL;
     const char* dictFileName = NULL;
+    const char* suffix = ZSTD_EXTENSION;
     unsigned maxDictSize = g_defaultMaxDictSize;
     unsigned dictID = 0;
     int dictCLevel = g_defaultDictCLevel;
@@ -241,6 +304,10 @@ int main(int argCount, const char* argv[])
     char* fileNamesBuf = NULL;
     unsigned fileNamesNb;
 #endif
+#ifndef ZSTD_NODICT
+    COVER_params_t coverParams;
+    int cover = 0;
+#endif
 
     /* init */
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
@@ -259,6 +326,12 @@ int main(int argCount, const char* argv[])
     /* preset behaviors */
     if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
     if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }
+    if (!strcmp(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); }    /* behave like gzip */
+    if (!strcmp(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); }                                          /* behave like gunzip */
+    if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }  /* behave like gzcat */
+    if (!strcmp(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }    /* behave like lzma */
+    if (!strcmp(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }    /* behave like xz */
+    memset(&compressionParams, 0, sizeof(compressionParams));
 
     /* command switches */
     for (argNb=1; argNb<argCount; argNb++) {
@@ -302,11 +375,35 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
                     if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
                     if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
+                    if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
+#ifdef ZSTD_GZCOMPRESS
+                    if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
+#endif
+#ifdef ZSTD_LZMACOMPRESS
+                    if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression);  continue; }
+                    if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression);  continue; }
+#endif
 
                     /* long commands with arguments */
+#ifndef ZSTD_NODICT
+                    if (longCommandWArg(&argument, "--cover=")) {
+                      cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName));
+                      continue;
+                    }
+                    if (longCommandWArg(&argument, "--optimize-cover")) {
+                      cover=2;
+                      /* Allow optional arguments following an = */
+                      if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
+                      else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
+                      else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
+                      continue;
+                    }
+#endif
                     if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
+                    if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; }
+                    if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
                     /* fall-through, will trigger bad_usage() later on */
                 }
 
@@ -314,7 +411,7 @@ int main(int argCount, const char* argv[])
                 while (argument[0]!=0) {
                     if (lastCommand) {
                         DISPLAY("error : command must be followed by argument \n");
-                        return 1;
+                        CLEAN_RETURN(1);
                     }
 #ifndef ZSTD_NOCOMPRESS
                     /* compression Level */
@@ -337,7 +434,7 @@ int main(int argCount, const char* argv[])
                          /* Decoding */
                     case 'd':
 #ifndef ZSTD_NOBENCH
-                            if (operation==zom_bench) { BMK_setDecodeOnly(1); argument++; break; }  /* benchmark decode (hidden option) */
+                            if (operation==zom_bench) { BMK_setDecodeOnlyMode(1); argument++; break; }  /* benchmark decode (hidden option) */
 #endif
                             operation=zom_decompress; argument++; break;
 
@@ -381,34 +478,38 @@ int main(int argCount, const char* argv[])
 
 #ifndef ZSTD_NOBENCH
                         /* Benchmark */
-                    case 'b': operation=zom_bench; argument++; break;
+                    case 'b':
+                        operation=zom_bench;
+                        argument++;
+                        break;
 
                         /* range bench (benchmark only) */
                     case 'e':
-                            /* compression Level */
-                            argument++;
-                            cLevelLast = readU32FromChar(&argument);
-                            break;
+                        /* compression Level */
+                        argument++;
+                        cLevelLast = readU32FromChar(&argument);
+                        break;
 
                         /* Modify Nb Iterations (benchmark only) */
                     case 'i':
                         argument++;
-                        {   U32 const iters = readU32FromChar(&argument);
-                            BMK_setNotificationLevel(displayLevel);
-                            BMK_SetNbSeconds(iters);
-                        }
+                        bench_nbSeconds = readU32FromChar(&argument);
                         break;
 
                         /* cut input into blocks (benchmark only) */
                     case 'B':
                         argument++;
-                        {   size_t const bSize = readU32FromChar(&argument);
-                            BMK_setNotificationLevel(displayLevel);
-                            BMK_SetBlockSize(bSize);
-                        }
+                        blockSize = readU32FromChar(&argument);
                         break;
+
 #endif   /* ZSTD_NOBENCH */
 
+                        /* nb of threads (hidden option) */
+                    case 'T':
+                        argument++;
+                        nbThreads = readU32FromChar(&argument);
+                        break;
+
                         /* Dictionary Selection level */
                     case 's':
                         argument++;
@@ -466,10 +567,19 @@ int main(int argCount, const char* argv[])
         filenameTable[filenameIdx++] = argument;
     }
 
-    if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; }  /* forgotten argument */
+    if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); }  /* forgotten argument */
 
     /* Welcome message (if verbose) */
     DISPLAYLEVEL(3, WELCOME_MESSAGE);
+#ifdef _POSIX_C_SOURCE
+    DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
+#endif
+#ifdef _POSIX_VERSION
+    DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
+#endif
+#ifdef PLATFORM_POSIX_VERSION
+    DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
+#endif
 
 #ifdef UTIL_HAS_CREATEFILELIST
     if (recursive) {  /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
@@ -488,21 +598,33 @@ int main(int argCount, const char* argv[])
     if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
         BMK_setNotificationLevel(displayLevel);
-        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
+        BMK_setBlockSize(blockSize);
+        BMK_setNbThreads(nbThreads);
+        BMK_setNbSeconds(bench_nbSeconds);
+        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
 #endif
+        (void)bench_nbSeconds;
         goto _end;
     }
 
     /* Check if dictionary builder is selected */
     if (operation==zom_train) {
 #ifndef ZSTD_NODICT
-        ZDICT_params_t dictParams;
-        memset(&dictParams, 0, sizeof(dictParams));
-        dictParams.compressionLevel = dictCLevel;
-        dictParams.selectivityLevel = dictSelect;
-        dictParams.notificationLevel = displayLevel;
-        dictParams.dictID = dictID;
-        DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
+        if (cover) {
+            coverParams.nbThreads = nbThreads;
+            coverParams.compressionLevel = dictCLevel;
+            coverParams.notificationLevel = displayLevel;
+            coverParams.dictID = dictID;
+            DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, cover - 1);
+        } else {
+            ZDICT_params_t dictParams;
+            memset(&dictParams, 0, sizeof(dictParams));
+            dictParams.compressionLevel = dictCLevel;
+            dictParams.selectivityLevel = dictSelect;
+            dictParams.notificationLevel = displayLevel;
+            dictParams.dictID = dictID;
+            DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0);
+        }
 #endif
         goto _end;
     }
@@ -531,7 +653,7 @@ int main(int argCount, const char* argv[])
     }   }
 #endif
 
-    /* No warning message in pipe mode (stdin + stdout) or multi-files mode */
+    /* No status message in pipe mode (stdin - stdout) or multi-files mode */
     if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
     if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1;
 
@@ -539,10 +661,13 @@ int main(int argCount, const char* argv[])
     FIO_setNotificationLevel(displayLevel);
     if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
+        FIO_setNbThreads(nbThreads);
+        FIO_setBlockSize((U32)blockSize);
+        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
         if ((filenameIdx==1) && outFileName)
-          operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
+          operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
         else
-          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel);
+          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : suffix, dictFileName, cLevel, &compressionParams);
 #else
         DISPLAY("Compression not supported\n");
 #endif

programs/zstdless

@@ -1 +1,2 @@
-zstdcat $@ | less
+#!/bin/sh
+zstdcat "$@" | less

tests/.gitignore

@@ -3,15 +3,25 @@ fullbench
 fullbench32
 fuzzer
 fuzzer32
+fuzzer-dll
 zbufftest
 zbufftest32
+zbufftest-dll
 zstreamtest
 zstreamtest32
+zstreamtest_asan
+zstreamtest_tsan
+zstreamtest-dll
 datagen
 paramgrill
 paramgrill32
 roundTripCrash
 longmatch
+symbols
+legacy
+decodecorpus
+pool
+invalidDictionaries
 
 # Tmp test directory
 zstdtest

tests/Makefile

@@ -2,6 +2,8 @@
 # Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
+# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+#
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
@@ -18,46 +20,52 @@
 # zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
 # ##########################################################################
 
-DESTDIR?=
-PREFIX ?= /usr/local
-BINDIR  = $(PREFIX)/bin
-MANDIR  = $(PREFIX)/share/man/man1
 ZSTDDIR = ../lib
 PRGDIR  = ../programs
 PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
 
-CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR)
-CFLAGS ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
-CFLAGS += $(MOREFLAGS)
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CFLAGS  ?= -O3
+CFLAGS  += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+           -Wstrict-prototypes -Wundef -Wformat-security
+CFLAGS  += $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
+ZSTDCOMP_FILES   := $(ZSTDDIR)/compress/*.c
 ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
-ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 
+ZSTD_OBJ  := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES)) )
+ZBUFF_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZBUFF_FILES)) )
+ZDICT_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZDICT_FILES)) )
 
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  =
 else
 EXT =
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  = -pthread
 endif
+MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
 
 VOID = /dev/null
-ZSTREAM_TESTTIME = -T2mn
-FUZZERTEST= -T5mn
-ZSTDRTTEST= --test-large-data
+ZSTREAM_TESTTIME ?= -T2mn
+FUZZERTEST ?= -T5mn
+ZSTDRTTEST = --test-large-data
+DECODECORPUS_TESTTIME ?= -T30
 
-.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest
+.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
 
 default: fullbench
 
@@ -65,7 +73,7 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
 
 all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
 
-
+dll: fuzzer-dll zstreamtest-dll zbufftest-dll
 
 zstd:
 	$(MAKE) -C $(PRGDIR) $@
@@ -93,11 +101,16 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 
-fuzzer  : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
+fuzzer   : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
-	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
+	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+
+fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
 zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
@@ -109,11 +122,35 @@ zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
 zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-zstreamtest  : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
+zbufftest-dll : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
+zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
-zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-	$(CC) -m32  $(FLAGS) $^ -o $@$(EXT)
+ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
+zstreamtest : CPPFLAGS += $(MULTITHREAD_CPP)
+zstreamtest : LDFLAGS += $(MULTITHREAD_LD)
+zstreamtest : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
+zstreamtest32 : CFLAGS += -m32
+zstreamtest32 : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+
+zstreamtest_asan : CFLAGS += -fsanitize=address
+zstreamtest_asan : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+
+zstreamtest_tsan : CFLAGS += -fsanitize=thread
+zstreamtest_tsan : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+
+zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
 	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
@@ -127,30 +164,56 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
 longmatch  : $(ZSTD_FILES) longmatch.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
+invalidDictionaries  : $(ZSTD_FILES) invalidDictionaries.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=4
+legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
+legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+decodecorpus	: $(filter-out $(ZSTDDIR)/compress/zstd_compress.c, $(wildcard $(ZSTD_FILES))) decodecorpus.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT) -lm
+
+symbols  : symbols.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+ifneq (,$(filter Windows%,$(OS)))
+	cp $(ZSTDDIR)/dll/libzstd.dll .
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll
+else
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so
+endif
+
+pool  : pool.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c
+	$(CC)    $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+
 namespaceTest:
 	if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
 	$(RM) $@
 
-versionsTest:
+versionsTest: clean
 	$(PYTHON) test-zstd-versions.py
 
 clean:
-	$(MAKE) -C ../lib clean
+	$(MAKE) -C $(ZSTDDIR) clean
 	@$(RM) -fR $(TESTARTEFACT)
 	@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
         $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
         fullbench$(EXT) fullbench32$(EXT) \
         fullbench-lib$(EXT) fullbench-dll$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
-		zstreamtest$(EXT) zstreamtest32$(EXT) \
-        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT)
+        fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
+        zstreamtest$(EXT) zstreamtest32$(EXT) \
+        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
+        symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT) \
+	decodecorpus$(EXT)
 	@echo Cleaning completed
 
 
 #----------------------------------------------------------------------------------
-#make valgrindTest is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
+#make valgrindTest is validated only for Linux, OSX, BSD, Hurd and Solaris targets
 #----------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 HOST_OS = POSIX
 
 valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
@@ -175,15 +238,29 @@ HOST_OS = MSYS
 endif
 
 
-#------------------------------------------------------------------------
-#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
-#------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+#make tests validated only for MSYS, Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+
+DIFF:=diff
+ifneq (,$(filter $(shell uname),SunOS))
+DIFF:=gdiff
+endif
+
 zstd-playTests: datagen
 	file $(ZSTD)
 	ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
 
-test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch
+shortest: ZSTDRTTEST=
+shortest: test-zstd
+
+fuzztest: test-fuzzer test-zstream test-decodecorpus
+
+test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
+ifeq ($(QEMU_SYS),)
+test: test-pool
+endif
 
 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
 
@@ -206,7 +283,7 @@ test-gzstd: gzstd
 	$(PRGDIR)/zstd -d README.md.gz -o README2.md
 	$(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz
 	$(PRGDIR)/zstd -d zstd_gz.zst gz_zstd.gz
-	diff -q zstd_gz gz_zstd
+	$(DIFF) -q zstd_gz gz_zstd
 	echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst
 	echo Hello World GZIP | gzip -c - >hello.gz
 	echo Hello World TEXT >hello.txt
@@ -243,4 +320,19 @@ test-zstream32: zstreamtest32
 test-longmatch: longmatch
 	$(QEMU_SYS) ./longmatch
 
+test-invalidDictionaries: invalidDictionaries
+	$(QEMU_SYS) ./invalidDictionaries
+
+test-symbols: symbols
+	$(QEMU_SYS) ./symbols
+
+test-legacy: legacy
+	$(QEMU_SYS) ./legacy
+
+test-decodecorpus: decodecorpus
+	$(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME)
+
+test-pool: pool
+	$(QEMU_SYS) ./pool
+
 endif

tests/README.md

@@ -10,12 +10,14 @@ This directory contains the following programs and scripts:
 - `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
 - `zbufftest`  : Test tool to check ZBUFF (a buffered streaming API) integrity
 - `zstreamtest` : Fuzzer test tool for zstd streaming API
+- `legacy` : Test tool to test decoding of legacy zstd frames
+- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations
 
 
 #### `test-zstd-versions.py` - script for testing zstd interoperability between versions
 
 This script creates `versionsTest` directory to which zstd repository is cloned.
-Then all taged (released) versions of zstd are compiled.
+Then all tagged (released) versions of zstd are compiled.
 In the following step interoperability between zstd versions is checked.
 
 
@@ -64,3 +66,25 @@ optional arguments:
   --sleepTime SLEEPTIME
                         frequency of repository checking in seconds
 ```
+
+#### `decodecorpus` - tool to generate Zstandard frames for decoder testing
+Command line tool to generate test .zst files.
+
+This tool will generate .zst files with checksums,
+as well as optionally output the corresponding correct uncompressed data for
+extra verfication.
+
+Example:
+```
+./decodecorpus -ptestfiles -otestfiles -n10000 -s5
+```
+will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory,
+with the zstd checksum field set,
+as well as the 10,000 original files for more detailed comparison of decompression results.
+
+```
+./decodecorpus -t -T1mn
+```
+will choose a random seed, and for 1 minute,
+generate random test frames and ensure that the
+zstd library correctly decompresses them in both simple and streaming modes.

tests/decodecorpus.c

@@ -0,0 +1,1449 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include <limits.h>
+#include <math.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "zstd.h"
+#include "zstd_internal.h"
+#include "mem.h"
+
+// Direct access to internal compression functions is required
+#include "zstd_compress.c"
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"     /* XXH64 */
+
+#ifndef MIN
+    #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef MAX_PATH
+    #ifdef PATH_MAX
+        #define MAX_PATH PATH_MAX
+    #else
+        #define MAX_PATH 256
+    #endif
+#endif
+
+/*-************************************
+*  DISPLAY Macros
+**************************************/
+#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 0;
+
+#define DISPLAYUPDATE(...)                                                     \
+    do {                                                                       \
+        if ((clockSpan(g_displayClock) > g_refreshRate) ||                     \
+            (g_displayLevel >= 4)) {                                           \
+            g_displayClock = clock();                                          \
+            DISPLAY(__VA_ARGS__);                                              \
+            if (g_displayLevel >= 4) fflush(stderr);                           \
+        }                                                                      \
+    } while (0)
+static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
+static clock_t g_displayClock = 0;
+
+static clock_t clockSpan(clock_t cStart)
+{
+    return clock() - cStart;   /* works even when overflow; max span ~ 30mn */
+}
+
+#define CHECKERR(code)                                                         \
+    do {                                                                       \
+        if (ZSTD_isError(code)) {                                              \
+            DISPLAY("Error occurred while generating data: %s\n",              \
+                    ZSTD_getErrorName(code));                                  \
+            exit(1);                                                           \
+        }                                                                      \
+    } while (0)
+
+/*-*******************************************************
+*  Random function
+*********************************************************/
+#define CLAMP(x, a, b) ((x) < (a) ? (a) : ((x) > (b) ? (b) : (x)))
+
+static unsigned RAND(unsigned* src)
+{
+#define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+    static const U32 prime1 = 2654435761U;
+    static const U32 prime2 = 2246822519U;
+    U32 rand32 = *src;
+    rand32 *= prime1;
+    rand32 += prime2;
+    rand32  = RAND_rotl32(rand32, 13);
+    *src = rand32;
+    return RAND_rotl32(rand32, 27);
+#undef RAND_rotl32
+}
+
+#define DISTSIZE (8192)
+
+/* Write `size` bytes into `ptr`, all of which are less than or equal to `maxSymb` */
+static void RAND_bufferMaxSymb(U32* seed, void* ptr, size_t size, int maxSymb)
+{
+    size_t i;
+    BYTE* op = ptr;
+
+    for (i = 0; i < size; i++) {
+        op[i] = RAND(seed) % (maxSymb + 1);
+    }
+}
+
+/* Write `size` random bytes into `ptr` */
+static void RAND_buffer(U32* seed, void* ptr, size_t size)
+{
+    size_t i;
+    BYTE* op = ptr;
+
+    for (i = 0; i + 4 <= size; i += 4) {
+        MEM_writeLE32(op + i, RAND(seed));
+    }
+    for (; i < size; i++) {
+        op[i] = RAND(seed) & 0xff;
+    }
+}
+
+/* Write `size` bytes into `ptr` following the distribution `dist` */
+static void RAND_bufferDist(U32* seed, BYTE* dist, void* ptr, size_t size)
+{
+    size_t i;
+    BYTE* op = ptr;
+
+    for (i = 0; i < size; i++) {
+        op[i] = dist[RAND(seed) % DISTSIZE];
+    }
+}
+
+/* Generate a random distribution where the frequency of each symbol follows a
+ * geometric distribution defined by `weight`
+ * `dist` should have size at least `DISTSIZE` */
+static void RAND_genDist(U32* seed, BYTE* dist, double weight)
+{
+    size_t i = 0;
+    size_t statesLeft = DISTSIZE;
+    BYTE symb = RAND(seed) % 256;
+    BYTE step = (RAND(seed) % 256) | 1; /* force it to be odd so it's relatively prime to 256 */
+
+    while (i < DISTSIZE) {
+        size_t states = ((size_t)(weight * statesLeft)) + 1;
+        size_t j;
+        for (j = 0; j < states && i < DISTSIZE; j++, i++) {
+            dist[i] = symb;
+        }
+
+        symb += step;
+        statesLeft -= states;
+    }
+}
+
+/* Generates a random number in the range [min, max) */
+static inline U32 RAND_range(U32* seed, U32 min, U32 max)
+{
+    return (RAND(seed) % (max-min)) + min;
+}
+
+#define ROUND(x) ((U32)(x + 0.5))
+
+/* Generates a random number in an exponential distribution with mean `mean` */
+static double RAND_exp(U32* seed, double mean)
+{
+    double const u = RAND(seed) / (double) UINT_MAX;
+    return log(1-u) * (-mean);
+}
+
+/*-*******************************************************
+*  Constants and Structs
+*********************************************************/
+const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"};
+
+#define MAX_DECOMPRESSED_SIZE_LOG 20
+#define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG)
+
+#define MAX_WINDOW_LOG 22 /* Recommended support is 8MB, so limit to 4MB + mantissa */
+#define MAX_BLOCK_SIZE (128ULL * 1024)
+
+#define MIN_SEQ_LEN (3)
+#define MAX_NB_SEQ ((MAX_BLOCK_SIZE + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
+
+BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
+BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
+BYTE LITERAL_BUFFER[MAX_BLOCK_SIZE];
+
+seqDef SEQUENCE_BUFFER[MAX_NB_SEQ];
+BYTE SEQUENCE_LITERAL_BUFFER[MAX_BLOCK_SIZE]; /* storeSeq expects a place to copy literals to */
+BYTE SEQUENCE_LLCODE[MAX_BLOCK_SIZE];
+BYTE SEQUENCE_MLCODE[MAX_BLOCK_SIZE];
+BYTE SEQUENCE_OFCODE[MAX_BLOCK_SIZE];
+
+unsigned WKSP[1024];
+
+typedef struct {
+    size_t contentSize; /* 0 means unknown (unless contentSize == windowSize == 0) */
+    unsigned windowSize; /* contentSize >= windowSize means single segment */
+} frameHeader_t;
+
+/* For repeat modes */
+typedef struct {
+    U32 rep[ZSTD_REP_NUM];
+
+    int hufInit;
+    /* the distribution used in the previous block for repeat mode */
+    BYTE hufDist[DISTSIZE];
+    U32 hufTable [256]; /* HUF_CElt is an incomplete type */
+
+    int fseInit;
+    FSE_CTable offcodeCTable  [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+    FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+    FSE_CTable litlengthCTable  [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+
+    /* Symbols that were present in the previous distribution, for use with
+     * set_repeat */
+    BYTE litlengthSymbolSet[36];
+    BYTE offsetSymbolSet[29];
+    BYTE matchlengthSymbolSet[53];
+} cblockStats_t;
+
+typedef struct {
+    void* data;
+    void* dataStart;
+    void* dataEnd;
+
+    void* src;
+    void* srcStart;
+    void* srcEnd;
+
+    frameHeader_t header;
+
+    cblockStats_t stats;
+    cblockStats_t oldStats; /* so they can be rolled back if uncompressible */
+} frame_t;
+
+/*-*******************************************************
+*  Generator Functions
+*********************************************************/
+
+struct {
+    int contentSize; /* force the content size to be present */
+} opts; /* advanced options on generation */
+
+/* Generate and write a random frame header */
+static void writeFrameHeader(U32* seed, frame_t* frame)
+{
+    BYTE* const op = frame->data;
+    size_t pos = 0;
+    frameHeader_t fh;
+
+    BYTE windowByte = 0;
+
+    int singleSegment = 0;
+    int contentSizeFlag = 0;
+    int fcsCode = 0;
+
+    memset(&fh, 0, sizeof(fh));
+
+    /* generate window size */
+    {
+        /* Follow window algorithm from specification */
+        int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10);
+        int const mantissa = RAND(seed) % 8;
+        windowByte = (exponent << 3) | mantissa;
+        fh.windowSize = (1U << (exponent + 10));
+        fh.windowSize += fh.windowSize / 8 * mantissa;
+    }
+
+    {
+        /* Generate random content size */
+        size_t highBit;
+        if (RAND(seed) & 7) {
+            /* do content of at least 128 bytes */
+            highBit = 1ULL << RAND_range(seed, 7, MAX_DECOMPRESSED_SIZE_LOG);
+        } else if (RAND(seed) & 3) {
+            /* do small content */
+            highBit = 1ULL << RAND_range(seed, 0, 7);
+        } else {
+            /* 0 size frame */
+            highBit = 0;
+        }
+        fh.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0;
+
+        /* provide size sometimes */
+        contentSizeFlag = opts.contentSize | (RAND(seed) & 1);
+
+        if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) {
+            /* do single segment sometimes */
+            fh.windowSize = fh.contentSize;
+            singleSegment = 1;
+        }
+    }
+
+    if (contentSizeFlag) {
+        /* Determine how large fcs field has to be */
+        int minFcsCode = (fh.contentSize >= 256) +
+                               (fh.contentSize >= 65536 + 256) +
+                               (fh.contentSize > 0xFFFFFFFFU);
+        if (!singleSegment && !minFcsCode) {
+            minFcsCode = 1;
+        }
+        fcsCode = minFcsCode + (RAND(seed) % (4 - minFcsCode));
+        if (fcsCode == 1 && fh.contentSize < 256) fcsCode++;
+    }
+
+    /* write out the header */
+    MEM_writeLE32(op + pos, ZSTD_MAGICNUMBER);
+    pos += 4;
+
+    {
+        BYTE const frameHeaderDescriptor =
+                (fcsCode << 6) | (singleSegment << 5) | (1 << 2);
+        op[pos++] = frameHeaderDescriptor;
+    }
+
+    if (!singleSegment) {
+        op[pos++] = windowByte;
+    }
+
+    if (contentSizeFlag) {
+        switch (fcsCode) {
+        default: /* Impossible */
+        case 0: op[pos++] = fh.contentSize; break;
+        case 1: MEM_writeLE16(op + pos, fh.contentSize - 256); pos += 2; break;
+        case 2: MEM_writeLE32(op + pos, fh.contentSize); pos += 4; break;
+        case 3: MEM_writeLE64(op + pos, fh.contentSize); pos += 8; break;
+        }
+    }
+
+    DISPLAYLEVEL(2, " frame content size:\t%zu\n", fh.contentSize);
+    DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize);
+    DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag);
+    DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment);
+
+    frame->data = op + pos;
+    frame->header = fh;
+}
+
+/* Write a literal block in either raw or RLE form, return the literals size */
+static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t contentSize)
+{
+    BYTE* op = (BYTE*)frame->data;
+    int const type = RAND(seed) % 2;
+    int const sizeFormatDesc = RAND(seed) % 8;
+    size_t litSize;
+    size_t maxLitSize = MIN(contentSize, MAX_BLOCK_SIZE);
+
+    if (sizeFormatDesc == 0) {
+        /* Size_FormatDesc = ?0 */
+        maxLitSize = MIN(maxLitSize, 31);
+    } else if (sizeFormatDesc <= 4) {
+        /* Size_FormatDesc = 01 */
+        maxLitSize = MIN(maxLitSize, 4095);
+    } else {
+        /* Size_Format = 11 */
+        maxLitSize = MIN(maxLitSize, 1048575);
+    }
+
+    litSize = RAND(seed) % (maxLitSize + 1);
+    if (frame->src == frame->srcStart && litSize == 0) {
+        litSize = 1; /* no empty literals if there's nothing preceding this block */
+    }
+    if (litSize + 3 > contentSize) {
+        litSize = contentSize; /* no matches shorter than 3 are allowed */
+    }
+    /* use smallest size format that fits */
+    if (litSize < 32) {
+        op[0] = (type | (0 << 2) | (litSize << 3)) & 0xff;
+        op += 1;
+    } else if (litSize < 4096) {
+        op[0] = (type | (1 << 2) | (litSize << 4)) & 0xff;
+        op[1] = (litSize >> 4) & 0xff;
+        op += 2;
+    } else {
+        op[0] = (type | (3 << 2) | (litSize << 4)) & 0xff;
+        op[1] = (litSize >> 4) & 0xff;
+        op[2] = (litSize >> 12) & 0xff;
+        op += 3;
+    }
+
+    if (type == 0) {
+        /* Raw literals */
+        DISPLAYLEVEL(4, "   raw literals\n");
+
+        RAND_buffer(seed, LITERAL_BUFFER, litSize);
+        memcpy(op, LITERAL_BUFFER, litSize);
+        op += litSize;
+    } else {
+        /* RLE literals */
+        BYTE const symb = RAND(seed) % 256;
+
+        DISPLAYLEVEL(4, "   rle literals: 0x%02x\n", (U32)symb);
+
+        memset(LITERAL_BUFFER, symb, litSize);
+        op[0] = symb;
+        op++;
+    }
+
+    frame->data = op;
+
+    return litSize;
+}
+
+/* Generate a Huffman header for the given source */
+static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t dstSize,
+                                 const void* src, size_t srcSize)
+{
+    BYTE* const ostart = (BYTE*)dst;
+    BYTE* op = ostart;
+
+    unsigned huffLog = 11;
+    U32 maxSymbolValue = 255;
+
+    U32 count[HUF_SYMBOLVALUE_MAX+1];
+
+    /* Scan input and build symbol stats */
+    {   size_t const largest = FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP);
+        if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 0; }   /* single symbol, rle */
+        if (largest <= (srcSize >> 7)+1) return 0;   /* Fast heuristic : not compressible enough */
+    }
+
+    /* Build Huffman Tree */
+    /* Max Huffman log is 11, min is highbit(maxSymbolValue)+1 */
+    huffLog = RAND_range(seed, ZSTD_highbit32(maxSymbolValue)+1, huffLog+1);
+    DISPLAYLEVEL(6, "     huffman log: %u\n", huffLog);
+    {   size_t const maxBits = HUF_buildCTable_wksp (hufTable, count, maxSymbolValue, huffLog, WKSP, sizeof(WKSP));
+        CHECKERR(maxBits);
+        huffLog = (U32)maxBits;
+    }
+
+    /* Write table description header */
+    {   size_t const hSize = HUF_writeCTable (op, dstSize, hufTable, maxSymbolValue, huffLog);
+        if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
+        op += hSize;
+    }
+
+    return op - ostart;
+}
+
+/* Write a Huffman coded literals block and return the litearls size */
+static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t contentSize)
+{
+    BYTE* origop = (BYTE*)frame->data;
+    BYTE* opend = (BYTE*)frame->dataEnd;
+    BYTE* op;
+    BYTE* const ostart = origop;
+    int const sizeFormat = RAND(seed) % 4;
+    size_t litSize;
+    size_t hufHeaderSize = 0;
+    size_t compressedSize = 0;
+    size_t maxLitSize = MIN(contentSize-3, MAX_BLOCK_SIZE);
+
+    symbolEncodingType_e hType;
+
+    if (contentSize < 64) {
+        /* make sure we get reasonably-sized literals for compression */
+        return ERROR(GENERIC);
+    }
+
+    DISPLAYLEVEL(4, "   compressed literals\n");
+
+    switch (sizeFormat) {
+    case 0: /* fall through, size is the same as case 1 */
+    case 1:
+        maxLitSize = MIN(maxLitSize, 1023);
+        origop += 3;
+        break;
+    case 2:
+        maxLitSize = MIN(maxLitSize, 16383);
+        origop += 4;
+        break;
+    case 3:
+        maxLitSize = MIN(maxLitSize, 262143);
+        origop += 5;
+        break;
+    default:; /* impossible */
+    }
+
+    do {
+        op = origop;
+        do {
+            litSize = RAND(seed) % (maxLitSize + 1);
+        } while (litSize < 32); /* avoid small literal sizes */
+        if (litSize + 3 > contentSize) {
+            litSize = contentSize; /* no matches shorter than 3 are allowed */
+        }
+
+        /* most of the time generate a new distribution */
+        if ((RAND(seed) & 3) || !frame->stats.hufInit) {
+            do {
+                if (RAND(seed) & 3) {
+                    /* add 10 to ensure some compressability */
+                    double const weight = ((RAND(seed) % 90) + 10) / 100.0;
+
+                    DISPLAYLEVEL(5, "    distribution weight: %d%%\n",
+                                 (int)(weight * 100));
+
+                    RAND_genDist(seed, frame->stats.hufDist, weight);
+                } else {
+                    /* sometimes do restricted range literals to force
+                     * non-huffman headers */
+                    DISPLAYLEVEL(5, "    small range literals\n");
+                    RAND_bufferMaxSymb(seed, frame->stats.hufDist, DISTSIZE,
+                                       15);
+                }
+                RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER,
+                                litSize);
+
+                /* generate the header from the distribution instead of the
+                 * actual data to avoid bugs with symbols that were in the
+                 * distribution but never showed up in the output */
+                hufHeaderSize = writeHufHeader(
+                        seed, (HUF_CElt*)frame->stats.hufTable, op, opend - op,
+                        frame->stats.hufDist, DISTSIZE);
+                CHECKERR(hufHeaderSize);
+                /* repeat until a valid header is written */
+            } while (hufHeaderSize == 0);
+            op += hufHeaderSize;
+            hType = set_compressed;
+
+            frame->stats.hufInit = 1;
+        } else {
+            /* repeat the distribution/table from last time */
+            DISPLAYLEVEL(5, "    huffman repeat stats\n");
+            RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER,
+                            litSize);
+            hufHeaderSize = 0;
+            hType = set_repeat;
+        }
+
+        do {
+            compressedSize =
+                    sizeFormat == 0
+                            ? HUF_compress1X_usingCTable(
+                                      op, opend - op, LITERAL_BUFFER, litSize,
+                                      (HUF_CElt*)frame->stats.hufTable)
+                            : HUF_compress4X_usingCTable(
+                                      op, opend - op, LITERAL_BUFFER, litSize,
+                                      (HUF_CElt*)frame->stats.hufTable);
+            CHECKERR(compressedSize);
+            /* this only occurs when it could not compress or similar */
+        } while (compressedSize <= 0);
+
+        op += compressedSize;
+
+        compressedSize += hufHeaderSize;
+        DISPLAYLEVEL(5, "    regenerated size: %zu\n", litSize);
+        DISPLAYLEVEL(5, "    compressed size: %zu\n", compressedSize);
+        if (compressedSize >= litSize) {
+            DISPLAYLEVEL(5, "     trying again\n");
+            /* if we have to try again, reset the stats so we don't accidentally
+             * try to repeat a distribution we just made */
+            frame->stats = frame->oldStats;
+        } else {
+            break;
+        }
+    } while (1);
+
+    /* write header */
+    switch (sizeFormat) {
+    case 0: /* fall through, size is the same as case 1 */
+    case 1: {
+        U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) |
+                           ((U32)compressedSize << 14);
+        MEM_writeLE24(ostart, header);
+        break;
+    }
+    case 2: {
+        U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) |
+                           ((U32)compressedSize << 18);
+        MEM_writeLE32(ostart, header);
+        break;
+    }
+    case 3: {
+        U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) |
+                           ((U32)compressedSize << 22);
+        MEM_writeLE32(ostart, header);
+        ostart[4] = (BYTE)(compressedSize >> 10);
+        break;
+    }
+    default:; /* impossible */
+    }
+
+    frame->data = op;
+    return litSize;
+}
+
+static size_t writeLiteralsBlock(U32* seed, frame_t* frame, size_t contentSize)
+{
+    /* only do compressed for larger segments to avoid compressibility issues */
+    if (RAND(seed) & 7 && contentSize >= 64) {
+        return writeLiteralsBlockCompressed(seed, frame, contentSize);
+    } else {
+        return writeLiteralsBlockSimple(seed, frame, contentSize);
+    }
+}
+
+static inline void initSeqStore(seqStore_t *seqStore) {
+    seqStore->sequencesStart = SEQUENCE_BUFFER;
+    seqStore->litStart = SEQUENCE_LITERAL_BUFFER;
+    seqStore->llCode = SEQUENCE_LLCODE;
+    seqStore->mlCode = SEQUENCE_MLCODE;
+    seqStore->ofCode = SEQUENCE_OFCODE;
+
+    ZSTD_resetSeqStore(seqStore);
+}
+
+/* Randomly generate sequence commands */
+static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
+                                size_t contentSize, size_t literalsSize)
+{
+    /* The total length of all the matches */
+    size_t const remainingMatch = contentSize - literalsSize;
+    size_t excessMatch = 0;
+    U32 numSequences = 0;
+  
+    U32 i;
+
+
+    const BYTE* literals = LITERAL_BUFFER;
+    BYTE* srcPtr = frame->src;
+
+    if (literalsSize != contentSize) {
+        /* each match must be at least MIN_SEQ_LEN, so this is the maximum
+         * number of sequences we can have */
+        U32 const maxSequences = (U32)remainingMatch / MIN_SEQ_LEN;
+        numSequences = (RAND(seed) % maxSequences) + 1;
+
+        /* the extra match lengths we have to allocate to each sequence */
+        excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN;
+    }
+
+    DISPLAYLEVEL(5, "    total match lengths: %zu\n", remainingMatch);
+
+    for (i = 0; i < numSequences; i++) {
+        /* Generate match and literal lengths by exponential distribution to
+         * ensure nice numbers */
+        U32 matchLen =
+                MIN_SEQ_LEN +
+                ROUND(RAND_exp(seed, excessMatch / (double)(numSequences - i)));
+        U32 literalLen =
+                (RAND(seed) & 7)
+                        ? ROUND(RAND_exp(seed,
+                                         literalsSize /
+                                                 (double)(numSequences - i)))
+                        : 0;
+        /* actual offset, code to send, and point to copy up to when shifting
+         * codes in the repeat offsets history */
+        U32 offset, offsetCode, repIndex;
+
+        /* bounds checks */
+        matchLen = MIN(matchLen, excessMatch + MIN_SEQ_LEN);
+        literalLen = MIN(literalLen, literalsSize);
+        if (i == 0 && srcPtr == frame->srcStart && literalLen == 0) literalLen = 1;
+        if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + excessMatch;
+
+        memcpy(srcPtr, literals, literalLen);
+        srcPtr += literalLen;
+
+        do {
+            if (RAND(seed) & 7) {
+                /* do a normal offset */
+                offset = (RAND(seed) %
+                          MIN(frame->header.windowSize,
+                              (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) +
+                         1;
+                offsetCode = offset + ZSTD_REP_MOVE;
+                repIndex = 2;
+            } else {
+                /* do a repeat offset */
+                offsetCode = RAND(seed) % 3;
+                if (literalLen > 0) {
+                    offset = frame->stats.rep[offsetCode];
+                    repIndex = offsetCode;
+                } else {
+                    /* special case */
+                    offset = offsetCode == 2 ? frame->stats.rep[0] - 1
+                                           : frame->stats.rep[offsetCode + 1];
+                    repIndex = MIN(2, offsetCode + 1);
+                }
+            }
+        } while (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart) || offset == 0);
+
+        {   size_t j;
+            for (j = 0; j < matchLen; j++) {
+                *srcPtr = *(srcPtr-offset);
+                srcPtr++;
+            }
+        }
+
+        {   int r;
+            for (r = repIndex; r > 0; r--) {
+                frame->stats.rep[r] = frame->stats.rep[r - 1];
+            }
+            frame->stats.rep[0] = offset;
+        }
+
+        DISPLAYLEVEL(6, "      LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen);
+        DISPLAYLEVEL(7, " srcPos: %8tu seqNb: %3u",
+                     (BYTE*)srcPtr - (BYTE*)frame->srcStart, i);
+        DISPLAYLEVEL(6, "\n");
+        if (offsetCode < 3) {
+            DISPLAYLEVEL(7, "        repeat offset: %d\n", repIndex);
+        }
+        /* use libzstd sequence handling */
+        ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode,
+                      matchLen - MINMATCH);
+
+        literalsSize -= literalLen;
+        excessMatch -= (matchLen - MIN_SEQ_LEN);
+        literals += literalLen;
+    }
+
+    memcpy(srcPtr, literals, literalsSize);
+    srcPtr += literalsSize;
+    DISPLAYLEVEL(6, "      excess literals: %5zu", literalsSize);
+    DISPLAYLEVEL(7, " srcPos: %8tu", (BYTE*)srcPtr - (BYTE*)frame->srcStart);
+    DISPLAYLEVEL(6, "\n");
+
+    return numSequences;
+}
+
+static void initSymbolSet(const BYTE* symbols, size_t len, BYTE* set, BYTE maxSymbolValue)
+{
+    size_t i;
+
+    memset(set, 0, (size_t)maxSymbolValue+1);
+
+    for (i = 0; i < len; i++) {
+        set[symbols[i]] = 1;
+    }
+}
+
+static int isSymbolSubset(const BYTE* symbols, size_t len, const BYTE* set, BYTE maxSymbolValue)
+{
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        if (symbols[i] > maxSymbolValue || !set[symbols[i]]) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
+                             size_t nbSeq)
+{
+    /* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */
+    U32 count[MaxSeq+1];
+    S16 norm[MaxSeq+1];
+    FSE_CTable* CTable_LitLength = frame->stats.litlengthCTable;
+    FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable;
+    FSE_CTable* CTable_MatchLength = frame->stats.matchlengthCTable;
+    U32 LLtype, Offtype, MLtype;   /* compressed, raw or rle */
+    const seqDef* const sequences = seqStorePtr->sequencesStart;
+    const BYTE* const ofCodeTable = seqStorePtr->ofCode;
+    const BYTE* const llCodeTable = seqStorePtr->llCode;
+    const BYTE* const mlCodeTable = seqStorePtr->mlCode;
+    BYTE* const oend = (BYTE*)frame->dataEnd;
+    BYTE* op = (BYTE*)frame->data;
+    BYTE* seqHead;
+    BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)];
+
+    /* literals compressing block removed so that can be done separately */
+
+    /* Sequences Header */
+    if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall);
+    if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq;
+    else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
+    else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
+
+    /* seqHead : flags for FSE encoding type */
+    seqHead = op++;
+
+    if (nbSeq==0) {
+        frame->data = op;
+
+        return 0;
+    }
+
+    /* convert length/distances into codes */
+    ZSTD_seqToCodes(seqStorePtr);
+
+    /* CTable for Literal Lengths */
+    {   U32 max = MaxLL;
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP);
+        if (mostFrequent == nbSeq) {
+            /* do RLE if we have the chance */
+            *op++ = llCodeTable[0];
+            FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
+            LLtype = set_rle;
+        } else if (frame->stats.fseInit && !(RAND(seed) & 3) &&
+                   isSymbolSubset(llCodeTable, nbSeq,
+                                  frame->stats.litlengthSymbolSet, 35)) {
+            /* maybe do repeat mode if we're allowed to */
+            LLtype = set_repeat;
+        } else if (!(RAND(seed) & 3)) {
+            /* maybe use the default distribution */
+            FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
+            LLtype = set_basic;
+        } else {
+            /* fall back on a full table */
+            size_t nbSeq_1 = nbSeq;
+            const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
+            if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; }
+            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+              if (FSE_isError(NCountSize)) return ERROR(GENERIC);
+              op += NCountSize; }
+            FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
+            LLtype = set_compressed;
+    }   }
+
+    /* CTable for Offsets */
+    /* see Literal Lengths for descriptions of mode choices */
+    {   U32 max = MaxOff;
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP);
+        if (mostFrequent == nbSeq) {
+            *op++ = ofCodeTable[0];
+            FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
+            Offtype = set_rle;
+        } else if (frame->stats.fseInit && !(RAND(seed) & 3) &&
+                   isSymbolSubset(ofCodeTable, nbSeq,
+                                  frame->stats.offsetSymbolSet, 28)) {
+            Offtype = set_repeat;
+        } else if (!(RAND(seed) & 3)) {
+            FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
+            Offtype = set_basic;
+        } else {
+            size_t nbSeq_1 = nbSeq;
+            const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
+            if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; }
+            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+              if (FSE_isError(NCountSize)) return ERROR(GENERIC);
+              op += NCountSize; }
+            FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
+            Offtype = set_compressed;
+    }   }
+
+    /* CTable for MatchLengths */
+    /* see Literal Lengths for descriptions of mode choices */
+    {   U32 max = MaxML;
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP);
+        if (mostFrequent == nbSeq) {
+            *op++ = *mlCodeTable;
+            FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
+            MLtype = set_rle;
+        } else if (frame->stats.fseInit && !(RAND(seed) & 3) &&
+                   isSymbolSubset(mlCodeTable, nbSeq,
+                                  frame->stats.matchlengthSymbolSet, 52)) {
+            MLtype = set_repeat;
+        } else if (!(RAND(seed) & 3)) {
+            /* sometimes do default distribution */
+            FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
+            MLtype = set_basic;
+        } else {
+            /* fall back on table */
+            size_t nbSeq_1 = nbSeq;
+            const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
+            if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; }
+            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+              if (FSE_isError(NCountSize)) return ERROR(GENERIC);
+              op += NCountSize; }
+            FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
+            MLtype = set_compressed;
+    }   }
+    frame->stats.fseInit = 1;
+    initSymbolSet(llCodeTable, nbSeq, frame->stats.litlengthSymbolSet, 35);
+    initSymbolSet(ofCodeTable, nbSeq, frame->stats.offsetSymbolSet, 28);
+    initSymbolSet(mlCodeTable, nbSeq, frame->stats.matchlengthSymbolSet, 52);
+
+    DISPLAYLEVEL(5, "    LL type: %d OF type: %d ML type: %d\n", LLtype, Offtype, MLtype);
+
+    *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
+
+    /* Encoding Sequences */
+    {   BIT_CStream_t blockStream;
+        FSE_CState_t  stateMatchLength;
+        FSE_CState_t  stateOffsetBits;
+        FSE_CState_t  stateLitLength;
+
+        CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */
+
+        /* first symbols */
+        FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
+        FSE_initCState2(&stateOffsetBits,  CTable_OffsetBits,  ofCodeTable[nbSeq-1]);
+        FSE_initCState2(&stateLitLength,   CTable_LitLength,   llCodeTable[nbSeq-1]);
+        BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
+        if (MEM_32bits()) BIT_flushBits(&blockStream);
+        BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
+        if (MEM_32bits()) BIT_flushBits(&blockStream);
+        BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        BIT_flushBits(&blockStream);
+
+        {   size_t n;
+            for (n=nbSeq-2 ; n<nbSeq ; n--) {      /* intentional underflow */
+                BYTE const llCode = llCodeTable[n];
+                BYTE const ofCode = ofCodeTable[n];
+                BYTE const mlCode = mlCodeTable[n];
+                U32  const llBits = LL_bits[llCode];
+                U32  const ofBits = ofCode;                                     /* 32b*/  /* 64b*/
+                U32  const mlBits = ML_bits[mlCode];
+                                                                                /* (7)*/  /* (7)*/
+                FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode);       /* 15 */  /* 15 */
+                FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode);      /* 24 */  /* 24 */
+                if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
+                FSE_encodeSymbol(&blockStream, &stateLitLength, llCode);        /* 16 */  /* 33 */
+                if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog)))
+                    BIT_flushBits(&blockStream);                                /* (7)*/
+                BIT_addBits(&blockStream, sequences[n].litLength, llBits);
+                if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
+                BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
+                if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
+                BIT_addBits(&blockStream, sequences[n].offset, ofBits);         /* 31 */
+                BIT_flushBits(&blockStream);                                    /* (7)*/
+        }   }
+
+        FSE_flushCState(&blockStream, &stateMatchLength);
+        FSE_flushCState(&blockStream, &stateOffsetBits);
+        FSE_flushCState(&blockStream, &stateLitLength);
+
+        {   size_t const streamSize = BIT_closeCStream(&blockStream);
+            if (streamSize==0) return ERROR(dstSize_tooSmall);   /* not enough space */
+            op += streamSize;
+    }   }
+
+    frame->data = op;
+
+    return 0;
+}
+
+static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize,
+                                  size_t literalsSize)
+{
+    seqStore_t seqStore;
+    size_t numSequences;
+
+
+    initSeqStore(&seqStore);
+
+    /* randomly generate sequences */
+    numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize);
+    /* write them out to the frame data */
+    CHECKERR(writeSequences(seed, frame, &seqStore, numSequences));
+
+    return numSequences;
+}
+
+static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize)
+{
+    BYTE* const blockStart = (BYTE*)frame->data;
+    size_t literalsSize;
+    size_t nbSeq;
+
+    DISPLAYLEVEL(4, "  compressed block:\n");
+
+    literalsSize = writeLiteralsBlock(seed, frame, contentSize);
+
+    DISPLAYLEVEL(4, "   literals size: %zu\n", literalsSize);
+
+    nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize);
+
+    DISPLAYLEVEL(4, "   number of sequences: %zu\n", nbSeq);
+
+    return (BYTE*)frame->data - blockStart;
+}
+
+static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
+                       int lastBlock)
+{
+    int const blockTypeDesc = RAND(seed) % 8;
+    size_t blockSize;
+    int blockType;
+
+    BYTE *const header = (BYTE*)frame->data;
+    BYTE *op = header + 3;
+
+    DISPLAYLEVEL(3, " block:\n");
+    DISPLAYLEVEL(3, "  block content size: %zu\n", contentSize);
+    DISPLAYLEVEL(3, "  last block: %s\n", lastBlock ? "yes" : "no");
+
+    if (blockTypeDesc == 0) {
+        /* Raw data frame */
+
+        RAND_buffer(seed, frame->src, contentSize);
+        memcpy(op, frame->src, contentSize);
+
+        op += contentSize;
+        blockType = 0;
+        blockSize = contentSize;
+    } else if (blockTypeDesc == 1) {
+        /* RLE */
+        BYTE const symbol = RAND(seed) & 0xff;
+
+        op[0] = symbol;
+        memset(frame->src, symbol, contentSize);
+
+        op++;
+        blockType = 1;
+        blockSize = contentSize;
+    } else {
+        /* compressed, most common */
+        size_t compressedSize;
+        blockType = 2;
+
+        frame->oldStats = frame->stats;
+
+        frame->data = op;
+        compressedSize = writeCompressedBlock(seed, frame, contentSize);
+        if (compressedSize > contentSize) {
+            blockType = 0;
+            memcpy(op, frame->src, contentSize);
+
+            op += contentSize;
+            blockSize = contentSize; /* fall back on raw block if data doesn't
+                                        compress */
+
+            frame->stats = frame->oldStats; /* don't update the stats */
+        } else {
+            op += compressedSize;
+            blockSize = compressedSize;
+        }
+    }
+    frame->src = (BYTE*)frame->src + contentSize;
+
+    DISPLAYLEVEL(3, "  block type: %s\n", BLOCK_TYPES[blockType]);
+    DISPLAYLEVEL(3, "  block size field: %zu\n", blockSize);
+
+    header[0] = (lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff;
+    MEM_writeLE16(header + 1, blockSize >> 5);
+
+    frame->data = op;
+}
+
+static void writeBlocks(U32* seed, frame_t* frame)
+{
+    size_t contentLeft = frame->header.contentSize;
+    size_t const maxBlockSize = MIN(MAX_BLOCK_SIZE, frame->header.windowSize);
+    while (1) {
+        /* 1 in 4 chance of ending frame */
+        int const lastBlock = contentLeft > maxBlockSize ? 0 : !(RAND(seed) & 3);
+        size_t blockContentSize;
+        if (lastBlock) {
+            blockContentSize = contentLeft;
+        } else {
+            if (contentLeft > 0 && (RAND(seed) & 7)) {
+                /* some variable size blocks */
+                blockContentSize = RAND(seed) % (MIN(maxBlockSize, contentLeft)+1);
+            } else if (contentLeft > maxBlockSize && (RAND(seed) & 1)) {
+                /* some full size blocks */
+                blockContentSize = maxBlockSize;
+            } else {
+                /* some empty blocks */
+                blockContentSize = 0;
+            }
+        }
+
+        writeBlock(seed, frame, blockContentSize, lastBlock);
+
+        contentLeft -= blockContentSize;
+        if (lastBlock) break;
+    }
+}
+
+static void writeChecksum(frame_t* frame)
+{
+    /* write checksum so implementations can verify their output */
+    U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0);
+    DISPLAYLEVEL(2, "  checksum: %08x\n", (U32)digest);
+    MEM_writeLE32(frame->data, (U32)digest);
+    frame->data = (BYTE*)frame->data + 4;
+}
+
+static void outputBuffer(const void* buf, size_t size, const char* const path)
+{
+    /* write data out to file */
+    const BYTE* ip = (const BYTE*)buf;
+    FILE* out;
+    if (path) {
+        out = fopen(path, "wb");
+    } else {
+        out = stdout;
+    }
+    if (!out) {
+        fprintf(stderr, "Failed to open file at %s: ", path);
+        perror(NULL);
+        exit(1);
+    }
+
+    {
+        size_t fsize = size;
+        size_t written = 0;
+        while (written < fsize) {
+            written += fwrite(ip + written, 1, fsize - written, out);
+            if (ferror(out)) {
+                fprintf(stderr, "Failed to write to file at %s: ", path);
+                perror(NULL);
+                exit(1);
+            }
+        }
+    }
+
+    if (path) {
+        fclose(out);
+    }
+}
+
+static void initFrame(frame_t* fr)
+{
+    memset(fr, 0, sizeof(*fr));
+    fr->data = fr->dataStart = FRAME_BUFFER;
+    fr->dataEnd = FRAME_BUFFER + sizeof(FRAME_BUFFER);
+    fr->src = fr->srcStart = CONTENT_BUFFER;
+    fr->srcEnd = CONTENT_BUFFER + sizeof(CONTENT_BUFFER);
+
+    /* init repeat codes */
+    fr->stats.rep[0] = 1;
+    fr->stats.rep[1] = 4;
+    fr->stats.rep[2] = 8;
+}
+
+/* Return the final seed */
+static U32 generateFrame(U32 seed, frame_t* fr)
+{
+    /* generate a complete frame */
+    DISPLAYLEVEL(1, "frame seed: %u\n", seed);
+
+    initFrame(fr);
+
+    writeFrameHeader(&seed, fr);
+    writeBlocks(&seed, fr);
+    writeChecksum(fr);
+
+    return seed;
+}
+
+/*-*******************************************************
+*  Test Mode
+*********************************************************/
+
+BYTE DECOMPRESSED_BUFFER[MAX_DECOMPRESSED_SIZE];
+
+static size_t testDecodeSimple(frame_t* fr)
+{
+    /* test decoding the generated data with the simple API */
+    size_t const ret = ZSTD_decompress(DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE,
+                           fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart);
+
+    if (ZSTD_isError(ret)) return ret;
+
+    if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart,
+               (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) {
+        return ERROR(corruption_detected);
+    }
+
+    return ret;
+}
+
+static size_t testDecodeStreaming(frame_t* fr)
+{
+    /* test decoding the generated data with the streaming API */
+    ZSTD_DStream* zd = ZSTD_createDStream();
+    ZSTD_inBuffer in;
+    ZSTD_outBuffer out;
+    size_t ret;
+
+    if (!zd) return ERROR(memory_allocation);
+
+    in.src = fr->dataStart;
+    in.pos = 0;
+    in.size = (BYTE*)fr->data - (BYTE*)fr->dataStart;
+
+    out.dst = DECOMPRESSED_BUFFER;
+    out.pos = 0;
+    out.size = ZSTD_DStreamOutSize();
+
+    ZSTD_initDStream(zd);
+    while (1) {
+        ret = ZSTD_decompressStream(zd, &out, &in);
+        if (ZSTD_isError(ret)) goto cleanup; /* error */
+        if (ret == 0) break; /* frame is done */
+
+        /* force decoding to be done in chunks */
+        out.size += MIN(ZSTD_DStreamOutSize(), MAX_DECOMPRESSED_SIZE - out.size);
+    }
+
+    ret = out.pos;
+
+    if (memcmp(out.dst, fr->srcStart, out.pos) != 0) {
+        return ERROR(corruption_detected);
+    }
+
+cleanup:
+    ZSTD_freeDStream(zd);
+    return ret;
+}
+
+static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS)
+{
+    unsigned fnum;
+
+    clock_t const startClock = clock();
+    clock_t const maxClockSpan = testDurationS * CLOCKS_PER_SEC;
+
+    if (numFiles == 0 && !testDurationS) numFiles = 1;
+
+    DISPLAY("seed: %u\n", seed);
+
+    for (fnum = 0; fnum < numFiles || clockSpan(startClock) < maxClockSpan; fnum++) {
+        frame_t fr;
+
+        if (fnum < numFiles)
+            DISPLAYUPDATE("\r%u/%u        ", fnum, numFiles);
+        else
+            DISPLAYUPDATE("\r%u           ", fnum);
+
+        seed = generateFrame(seed, &fr);
+
+        {   size_t const r = testDecodeSimple(&fr);
+            if (ZSTD_isError(r)) {
+                DISPLAY("Error in simple mode on test seed %u: %s\n", seed + fnum,
+                        ZSTD_getErrorName(r));
+                return 1;
+            }
+        }
+        {   size_t const r = testDecodeStreaming(&fr);
+            if (ZSTD_isError(r)) {
+                DISPLAY("Error in streaming mode on test seed %u: %s\n", seed + fnum,
+                        ZSTD_getErrorName(r));
+                return 1;
+            }
+        }
+    }
+
+    DISPLAY("\r%u tests completed: ", fnum);
+    DISPLAY("OK\n");
+
+    return 0;
+}
+
+/*-*******************************************************
+*  File I/O
+*********************************************************/
+
+static int generateFile(U32 seed, const char* const path,
+                         const char* const origPath)
+{
+    frame_t fr;
+
+    DISPLAY("seed: %u\n", seed);
+
+    generateFrame(seed, &fr);
+
+    outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path);
+    if (origPath) {
+        outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath);
+    }
+    return 0;
+}
+
+static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
+                         const char* const origPath)
+{
+    char outPath[MAX_PATH];
+    unsigned fnum;
+
+    DISPLAY("seed: %u\n", seed);
+
+    for (fnum = 0; fnum < numFiles; fnum++) {
+        frame_t fr;
+
+        DISPLAYUPDATE("\r%u/%u        ", fnum, numFiles);
+
+        seed = generateFrame(seed, &fr);
+
+        if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) {
+            DISPLAY("Error: path too long\n");
+            return 1;
+        }
+        outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath);
+
+        if (origPath) {
+            if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) {
+                DISPLAY("Error: path too long\n");
+                return 1;
+            }
+            outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath);
+        }
+    }
+
+    DISPLAY("\r%u/%u      \n", fnum, numFiles);
+
+    return 0;
+}
+
+
+/*_*******************************************************
+*  Command line
+*********************************************************/
+static U32 makeSeed(void)
+{
+    U32 t = time(NULL);
+    return XXH32(&t, sizeof(t), 0) % 65536;
+}
+
+static unsigned readInt(const char** argument)
+{
+    unsigned val = 0;
+    while ((**argument>='0') && (**argument<='9')) {
+        val *= 10;
+        val += **argument - '0';
+        (*argument)++;
+    }
+    return val;
+}
+
+static void usage(const char* programName)
+{
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [args]\n", programName);
+    DISPLAY( "\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -p<path> : select output path (default:stdout)\n");
+    DISPLAY( "                in multiple files mode this should be a directory\n");
+    DISPLAY( " -o<path> : select path to output original file (default:no output)\n");
+    DISPLAY( "                in multiple files mode this should be a directory\n");
+    DISPLAY( " -s#      : select seed (default:random based on time)\n");
+    DISPLAY( " -n#      : number of files to generate (default:1)\n");
+    DISPLAY( " -t       : activate test mode (test files against libzstd instead of outputting them)\n");
+    DISPLAY( " -T#      : length of time to run tests for\n");
+    DISPLAY( " -v       : increase verbosity level (default:0, max:7)\n");
+    DISPLAY( " -h/H     : display help/long help and exit\n");
+}
+
+static void advancedUsage(const char* programName)
+{
+    usage(programName);
+    DISPLAY( "\n");
+    DISPLAY( "Advanced arguments :\n");
+    DISPLAY( " --content-size    : always include the content size in the frame header\n");
+}
+
+int main(int argc, char** argv)
+{
+    U32 seed = 0;
+    int seedset = 0;
+    unsigned numFiles = 0;
+    unsigned testDuration = 0;
+    int testMode = 0;
+    const char* path = NULL;
+    const char* origPath = NULL;
+
+    int argNb;
+
+    /* Check command line */
+    for (argNb=1; argNb<argc; argNb++) {
+        const char* argument = argv[argNb];
+        if(!argument) continue;   /* Protection if argument empty */
+
+        /* Handle commands. Aggregated commands are allowed */
+        if (argument[0]=='-') {
+            argument++;
+            while (*argument!=0) {
+                switch(*argument)
+                {
+                case 'h':
+                    usage(argv[0]);
+                    return 0;
+                case 'H':
+                    advancedUsage(argv[0]);
+                    return 0;
+                case 'v':
+                    argument++;
+                    g_displayLevel++;
+                    break;
+                case 's':
+                    argument++;
+                    seedset=1;
+                    seed = readInt(&argument);
+                    break;
+                case 'n':
+                    argument++;
+                    numFiles = readInt(&argument);
+                    break;
+                case 'T':
+                    argument++;
+                    testDuration = readInt(&argument);
+                    if (*argument == 'm') {
+                        testDuration *= 60;
+                        argument++;
+                        if (*argument == 'n') argument++;
+                    }
+                    break;
+                case 'o':
+                    argument++;
+                    origPath = argument;
+                    argument += strlen(argument);
+                    break;
+                case 'p':
+                    argument++;
+                    path = argument;
+                    argument += strlen(argument);
+                    break;
+                case 't':
+                    argument++;
+                    testMode = 1;
+                    break;
+                case '-':
+                    argument++;
+                    if (strcmp(argument, "content-size") == 0) {
+                        opts.contentSize = 1;
+                    } else {
+                        advancedUsage(argv[0]);
+                        return 1;
+                    }
+                    argument += strlen(argument);
+                    break;
+                default:
+                    usage(argv[0]);
+                    return 1;
+    }   }   }   }   /* for (argNb=1; argNb<argc; argNb++) */
+
+    if (!seedset) {
+        seed = makeSeed();
+    }
+
+    if (testMode) {
+        return runTestMode(seed, numFiles, testDuration);
+    } else {
+        if (testDuration) {
+            DISPLAY("Error: -T requires test mode (-t)\n\n");
+            usage(argv[0]);
+            return 1;
+        }
+    }
+
+    if (!path) {
+        DISPLAY("Error: path is required in file generation mode\n");
+        usage(argv[0]);
+        return 1;
+    }
+
+    if (numFiles == 0) {
+        return generateFile(seed, path, origPath);
+    } else {
+        return generateCorpus(seed, numFiles, path, origPath);
+    }
+}

tests/fullbench.c

@@ -24,7 +24,7 @@
     #define KB *(1 <<10)
     #define MB *(1 <<20)
     #define GB *(1U<<30)
-    typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; 
+    typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 #endif
 #include "zstd.h"            /* ZSTD_VERSION_STRING */
 #include "datagen.h"
@@ -179,6 +179,23 @@ size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, c
     return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
 }
 
+#define FIRST_BLOCK_SIZE 8
+size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+{
+    BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
+
+    (void)buff2;
+    memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
+    ZSTD_compressBegin(g_zcc, 1);
+
+    {   size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE);
+        if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; }
+        dst = (BYTE*)dst + compressResult;
+        dstCapacity -= compressResult;
+    }
+    return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE);
+}
+
 size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
 {
     size_t regeneratedSize = 0;
@@ -229,6 +246,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
         benchFunction = local_ZSTD_compressContinue; benchName = "ZSTD_compressContinue";
         break;
     case 12:
+        benchFunction = local_ZSTD_compressContinue_extDict; benchName = "ZSTD_compressContinue_extDict";
+        break;
+    case 13:
         benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue";
         break;
 	case 31:
@@ -268,6 +288,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
         if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
         break;
     case 12 :
+        if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
+        break;
+    case 13 :
         if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
         g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
         break;
@@ -336,6 +359,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
     { size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }     /* warming up memory */
 
     {   U32 loopNb;
+        DISPLAY("%2i- %-30.30s : \r", benchNb, benchName);
         for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
             clock_t const timeLoop = TIMELOOP_S * CLOCKS_PER_SEC;
             clock_t clockStart;
@@ -343,8 +367,6 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
             size_t benchResult=0;
             double averageTime;
 
-            DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
-
             clockStart = clock();
             while (clock() == clockStart);
             clockStart = clock();

tests/fuzzer.c

@@ -28,6 +28,7 @@
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "zstd.h"         /* ZSTD_VERSION_STRING */
 #include "zstd_errors.h"  /* ZSTD_getErrorCode */
+#define ZDICT_STATIC_LINKING_ONLY
 #include "zdict.h"        /* ZDICT_trainFromBuffer */
 #include "datagen.h"      /* RDG_genBuffer */
 #include "mem.h"
@@ -65,6 +66,7 @@ static clock_t g_displayClock = 0;
 *  Fuzzer functions
 *********************************************************/
 #define MIN(a,b) ((a)<(b)?(a):(b))
+#define MAX(a,b) ((a)>(b)?(a):(b))
 
 static clock_t FUZ_clockSpan(clock_t cStart)
 {
@@ -107,6 +109,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     void* const CNBuffer = malloc(CNBuffSize);
     void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
     void* const decodedBuffer = malloc(CNBuffSize);
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
     int testResult = 0;
     U32 testNb=0;
     size_t cSize;
@@ -137,7 +140,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
 
     DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
-    {   unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
+    {   unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
         if (rSize != CNBuffSize) goto _output_error;
     }
     DISPLAYLEVEL(4, "OK \n");
@@ -154,6 +157,16 @@ static int basicUnitTests(U32 seed, double compressibility)
     }   }
     DISPLAYLEVEL(4, "OK \n");
 
+    DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++);
+    { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
+      if (r != CNBuffSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
+    DISPLAYLEVEL(4, "test%3i : decompress with null DDict : ", testNb++);
+    { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
+      if (r != CNBuffSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
     DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
     { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
       if (!ZSTD_isError(r)) goto _output_error;
@@ -166,10 +179,49 @@ static int basicUnitTests(U32 seed, double compressibility)
       if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
     DISPLAYLEVEL(4, "OK \n");
 
+    /* Simple API multiframe test */
+    DISPLAYLEVEL(4, "test%3i : compress multiple frames : ", testNb++);
+    {   size_t off = 0;
+        int i;
+        int const segs = 4;
+        /* only use the first half so we don't push against size limit of compressedBuffer */
+        size_t const segSize = (CNBuffSize / 2) / segs;
+        for (i = 0; i < segs; i++) {
+            CHECK_V(r,
+                    ZSTD_compress(
+                            (BYTE *)compressedBuffer + off, CNBuffSize - off,
+                            (BYTE *)CNBuffer + segSize * i,
+                            segSize, 5));
+            off += r;
+            if (i == segs/2) {
+                /* insert skippable frame */
+                const U32 skipLen = 128 KB;
+                MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
+                MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
+                off += skipLen + ZSTD_skippableHeaderSize;
+            }
+        }
+        cSize = off;
+    }
+    DISPLAYLEVEL(4, "OK \n");
+
+    DISPLAYLEVEL(4, "test%3i : get decompressed size of multiple frames : ", testNb++);
+    {   unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
+        if (r != CNBuffSize / 2) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
+    DISPLAYLEVEL(4, "test%3i : decompress multiple frames : ", testNb++);
+    {   CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
+        if (r != CNBuffSize / 2) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
+    DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+    if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
+    DISPLAYLEVEL(4, "OK \n");
+
     /* Dictionary and CCtx Duplication tests */
     {   ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
         ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         static const size_t dictSize = 551;
 
         DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
@@ -179,7 +231,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
         CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
-        CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, CNBuffSize - dictSize) );
+        CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
         DISPLAYLEVEL(4, "OK \n");
 
         DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
@@ -215,6 +267,14 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize - dictSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : decompress with DDict : ", testNb++);
+        {   ZSTD_DDict* const ddict = ZSTD_createDDict_byReference(CNBuffer, dictSize);
+            size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
+            if (r != CNBuffSize - dictSize) goto _output_error;
+            DISPLAYLEVEL(4, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict));
+            ZSTD_freeDDict(ddict);
+        }
+
         DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
         {   size_t const testSize = CNBuffSize / 3;
             {   ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);
@@ -234,12 +294,10 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         ZSTD_freeCCtx(ctxOrig);
         ZSTD_freeCCtx(ctxDuplicated);
-        ZSTD_freeDCtx(dctx);
     }
 
     /* Dictionary and dictBuilder tests */
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         size_t dictSize = 16 KB;
         void* dictBuffer = malloc(dictSize);
         size_t const totalSampleSize = 1 MB;
@@ -311,8 +369,69 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : dictionary containing only header should return error : ", testNb++);
+        {
+          const size_t ret = ZSTD_decompress_usingDict(
+              dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
+              "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
+          if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error;
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
+        ZSTD_freeCCtx(cctx);
+        free(dictBuffer);
+        free(samplesSizes);
+    }
+
+    /* COVER dictionary builder tests */
+    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        size_t dictSize = 16 KB;
+        size_t optDictSize = dictSize;
+        void* dictBuffer = malloc(dictSize);
+        size_t const totalSampleSize = 1 MB;
+        size_t const sampleUnitSize = 8 KB;
+        U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
+        size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
+        COVER_params_t params;
+        U32 dictID;
+
+        if (dictBuffer==NULL || samplesSizes==NULL) {
+            free(dictBuffer);
+            free(samplesSizes);
+            goto _output_error;
+        }
+
+        DISPLAYLEVEL(4, "test%3i : COVER_trainFromBuffer : ", testNb++);
+        { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
+        memset(&params, 0, sizeof(params));
+        params.d = 1 + (FUZ_rand(&seed) % 16);
+        params.k = params.d + (FUZ_rand(&seed) % 256);
+        dictSize = COVER_trainFromBuffer(dictBuffer, dictSize,
+                                         CNBuffer, samplesSizes, nbSamples,
+                                         params);
+        if (ZDICT_isError(dictSize)) goto _output_error;
+        DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
+
+        DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
+        dictID = ZDICT_getDictID(dictBuffer, dictSize);
+        if (dictID==0) goto _output_error;
+        DISPLAYLEVEL(4, "OK : %u \n", dictID);
+
+        DISPLAYLEVEL(4, "test%3i : COVER_optimizeTrainFromBuffer : ", testNb++);
+        memset(&params, 0, sizeof(params));
+        params.steps = 4;
+        optDictSize = COVER_optimizeTrainFromBuffer(dictBuffer, optDictSize,
+                                                    CNBuffer, samplesSizes, nbSamples / 4,
+                                                    &params);
+        if (ZDICT_isError(optDictSize)) goto _output_error;
+        DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize);
+
+        DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
+        dictID = ZDICT_getDictID(dictBuffer, optDictSize);
+        if (dictID==0) goto _output_error;
+        DISPLAYLEVEL(4, "OK : %u \n", dictID);
+
         ZSTD_freeCCtx(cctx);
-        ZSTD_freeDCtx(dctx);
         free(dictBuffer);
         free(samplesSizes);
     }
@@ -332,7 +451,6 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     /* block API tests */
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         static const size_t dictSize = 65 KB;
         static const size_t blockSize = 100 KB;   /* won't cause pb with small dict size */
         size_t cSize2;
@@ -440,6 +558,23 @@ static int basicUnitTests(U32 seed, double compressibility)
       if (r != _3BYTESTESTLENGTH) goto _output_error; }
     DISPLAYLEVEL(4, "OK \n");
 
+    /* findFrameCompressedSize on skippable frames */
+    DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++);
+    {   const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
+        size_t const frameSrcSize = 13;
+        if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* error string tests */
+    DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
+    if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
+    if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
+    if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
+    if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
+    if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
+    if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
+    DISPLAYLEVEL(4, "OK \n");
+
 _end:
     free(CNBuffer);
     free(compressedBuffer);
@@ -583,7 +718,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
         }   }
 
         /* Decompressed size test */
-        {   unsigned long long const rSize = ZSTD_getDecompressedSize(cBuffer, cSize);
+        {   unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
             CHECK(rSize != sampleSize, "decompressed size incorrect");
         }
 
@@ -665,11 +800,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
         /*=====   Streaming compression test, scattered segments and dictionary   =====*/
 
         {   U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
-            int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
+            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+            int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
 
-            dictSize = FUZ_randomLength(&lseed, maxSampleLog);   /* needed also for decompression */
+            dictSize = FUZ_rLogLength(&lseed, dictLog);   /* needed also for decompression */
             dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
 
             if (FUZ_rand(&lseed) & 0xF) {
@@ -690,6 +826,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
                 CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
         }   }
         XXH64_reset(&xxhState, 0);
+        ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1);
         {   U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
             U32 n;
             for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {

tests/gzip/Makefile

@@ -0,0 +1,44 @@
+# ################################################################
+# Copyright (c) 2017-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+PRGDIR = ../../programs
+VOID   = /dev/null
+export PATH := .:$(PATH)
+
+.PHONY: all
+#all: test-gzip-env 
+all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed
+all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid
+all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix
+	@echo Testing completed
+
+.PHONY: zstd
+zstd:
+	$(MAKE) -C $(PRGDIR) zstd
+	ln -sf $(PRGDIR)/zstd gzip
+	@echo PATH=$(PATH)
+	gzip --version
+
+.PHONY: clean
+clean:
+	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	@$(RM) *.trs *.log
+	@echo Cleaning completed
+
+
+#------------------------------------------------------------------------------
+# validated only for Linux, OSX, Hurd and some BSD targets
+#------------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+
+test-%: zstd
+	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
+	# || echo ignoring error
+
+endif

tests/gzip/gzip-env.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+# Test the obsolescent GZIP environment variable.
+
+# Copyright 2015-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+#echo PATH=$PATH
+#gzip --version
+
+echo a >exp || framework_failure_
+gzip <exp >in || framework_failure_
+
+fail=0
+GZIP=-qv gzip -d <in >out 2>err || fail=1
+compare exp out || fail=1
+
+for badopt in -- -c --stdout -d --decompress -f --force -h --help -k --keep \
+  -l --list -L --license -r --recursive -Sxxx --suffix=xxx '--suffix xxx' \
+  -t --test -V --version
+do
+  GZIP=$badopt gzip -d <in >out 2>err && fail=1
+done
+
+for goodopt in -n --no-name -N --name -q --quiet -v --verbose \
+  -1 --fast -2 -3 -4 -5 -6 -7 -8 -9 --best
+do
+  GZIP=$goodopt gzip -d <in >out 2>err || fail=1
+  compare exp out || fail=1
+done
+
+Exit $fail

tests/gzip/helin-segv.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+# Before gzip-1.4, gzip -d would segfault on some inputs.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+# This test case was provided by Aki Helin.
+printf '\037\235\220\0\0\0\304' > helin.gz || framework_failure_
+printf '\0\0' > exp || framework_failure_
+
+fail=0
+
+gzip -dc helin.gz > out || fail=1
+compare exp out || fail=1
+
+Exit $fail

tests/gzip/help-version.sh

@@ -0,0 +1,270 @@
+#! /bin/sh
+# Make sure all these programs work properly
+# when invoked with --help or --version.
+
+# Copyright (C) 2000-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Ensure that $SHELL is set to *some* value and exported.
+# This is required for dircolors, which would fail e.g., when
+# invoked via debuild (which removes SHELL from the environment).
+test "x$SHELL" = x && SHELL=/bin/sh
+export SHELL
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+expected_failure_status_chroot=125
+expected_failure_status_env=125
+expected_failure_status_nice=125
+expected_failure_status_nohup=125
+expected_failure_status_stdbuf=125
+expected_failure_status_su=125
+expected_failure_status_timeout=125
+expected_failure_status_printenv=2
+expected_failure_status_tty=3
+expected_failure_status_sort=2
+expected_failure_status_expr=3
+expected_failure_status_lbracket=2
+expected_failure_status_dir=2
+expected_failure_status_ls=2
+expected_failure_status_vdir=2
+
+expected_failure_status_cmp=2
+expected_failure_status_zcmp=2
+expected_failure_status_sdiff=2
+expected_failure_status_diff3=2
+expected_failure_status_diff=2
+expected_failure_status_zdiff=2
+expected_failure_status_zgrep=2
+expected_failure_status_zegrep=2
+expected_failure_status_zfgrep=2
+
+expected_failure_status_grep=2
+expected_failure_status_egrep=2
+expected_failure_status_fgrep=2
+
+test "$built_programs" \
+  || fail_ "built_programs not specified!?!"
+
+test "$VERSION" \
+  || fail_ "set envvar VERSION; it is required for a PATH sanity-check"
+
+# Extract version from --version output of the first program
+for i in $built_programs; do
+  v=$(env $i --version | sed -n '1s/.* //p;q')
+  break
+done
+
+# Ensure that it matches $VERSION.
+test "x$v" = "x$VERSION" \
+  || fail_ "--version-\$VERSION mismatch"
+
+for lang in C fr da; do
+  for i in $built_programs; do
+
+    # Skip `test'; it doesn't accept --help or --version.
+    test $i = test && continue;
+
+    # false fails even when invoked with --help or --version.
+    if test $i = false; then
+      env LC_MESSAGES=$lang $i --help >/dev/null && fail=1
+      env LC_MESSAGES=$lang $i --version >/dev/null && fail=1
+      continue
+    fi
+
+    args=
+
+    # The just-built install executable is always named `ginstall'.
+    test $i = install && i=ginstall
+
+    # Make sure they exit successfully, under normal conditions.
+    eval "env \$i $args --help    > h-\$i   " || fail=1
+    eval "env \$i $args --version >/dev/null" || fail=1
+
+    # Make sure they mention the bug-reporting address in --help output.
+    grep "$PACKAGE_BUGREPORT" h-$i > /dev/null || fail=1
+    rm -f h-$i
+
+    # Make sure they fail upon `disk full' error.
+    if test -w /dev/full && test -c /dev/full; then
+      eval "env \$i $args --help    >/dev/full 2>/dev/null" && fail=1
+      eval "env \$i $args --version >/dev/full 2>/dev/null" && fail=1
+      status=$?
+      test $i = [ && prog=lbracket || prog=$i
+      eval "expected=\$expected_failure_status_$prog"
+      test x$expected = x && expected=1
+      if test $status = $expected; then
+        : # ok
+      else
+        fail=1
+        echo "*** $i: bad exit status \`$status' (expected $expected)," 1>&2
+        echo "  with --help or --version output redirected to /dev/full" 1>&2
+      fi
+    fi
+  done
+done
+
+bigZ_in=bigZ-in.Z
+zin=zin.gz
+zin2=zin2.gz
+
+tmp=tmp-$$
+tmp_in=in-$$
+tmp_in2=in2-$$
+tmp_dir=dir-$$
+tmp_out=out-$$
+mkdir $tmp || fail=1
+cd $tmp || fail=1
+
+comm_setup () { args="$tmp_in $tmp_in"; }
+csplit_setup () { args="$tmp_in //"; }
+cut_setup () { args='-f 1'; }
+join_setup () { args="$tmp_in $tmp_in"; }
+tr_setup () { args='a a'; }
+
+chmod_setup () { args="a+x $tmp_in"; }
+# Punt on these.
+chgrp_setup () { args=--version; }
+chown_setup () { args=--version; }
+mkfifo_setup () { args=--version; }
+mknod_setup () { args=--version; }
+# Punt on uptime, since it fails (e.g., failing to get boot time)
+# on some systems, and we shouldn't let that stop `make check'.
+uptime_setup () { args=--version; }
+
+# Create a file in the current directory, not in $TMPDIR.
+mktemp_setup () { args=mktemp.XXXX; }
+
+cmp_setup () { args="$tmp_in $tmp_in2"; }
+
+# Tell dd not to print the line with transfer rate and total.
+# The transfer rate would vary between runs.
+dd_setup () { args=status=noxfer; }
+
+zdiff_setup () { args="$args $zin $zin2"; }
+zcmp_setup () { zdiff_setup; }
+zcat_setup () { args="$args $zin"; }
+gunzip_setup () { zcat_setup; }
+zmore_setup () { zcat_setup; }
+zless_setup () { zcat_setup; }
+znew_setup () { args="$args $bigZ_in"; }
+zforce_setup () { zcat_setup; }
+zgrep_setup () { args="$args z $zin"; }
+zegrep_setup () { zgrep_setup; }
+zfgrep_setup () { zgrep_setup; }
+gzexe_setup () { args="$args $tmp_in"; }
+
+# We know that $tmp_in contains a "0"
+grep_setup () { args="0 $tmp_in"; }
+egrep_setup () { args="0 $tmp_in"; }
+fgrep_setup () { args="0 $tmp_in"; }
+
+diff_setup () { args="$tmp_in $tmp_in2"; }
+sdiff_setup () { args="$tmp_in $tmp_in2"; }
+diff3_setup () { args="$tmp_in $tmp_in2 $tmp_in2"; }
+cp_setup () { args="$tmp_in $tmp_in2"; }
+ln_setup () { args="$tmp_in ln-target"; }
+ginstall_setup () { args="$tmp_in $tmp_in2"; }
+mv_setup () { args="$tmp_in $tmp_in2"; }
+mkdir_setup () { args=$tmp_dir/subdir; }
+rmdir_setup () { args=$tmp_dir; }
+rm_setup () { args=$tmp_in; }
+shred_setup () { args=$tmp_in; }
+touch_setup () { args=$tmp_in2; }
+truncate_setup () { args="--reference=$tmp_in $tmp_in2"; }
+
+basename_setup () { args=$tmp_in; }
+dirname_setup () { args=$tmp_in; }
+expr_setup () { args=foo; }
+
+# Punt, in case GNU `id' hasn't been installed yet.
+groups_setup () { args=--version; }
+
+pathchk_setup () { args=$tmp_in; }
+yes_setup () { args=--version; }
+logname_setup () { args=--version; }
+nohup_setup () { args=--version; }
+printf_setup () { args=foo; }
+seq_setup () { args=10; }
+sleep_setup () { args=0; }
+su_setup () { args=--version; }
+stdbuf_setup () { args="-oL true"; }
+timeout_setup () { args=--version; }
+
+# I'd rather not run sync, since it spins up disks that I've
+# deliberately caused to spin down (but not unmounted).
+sync_setup () { args=--version; }
+
+test_setup () { args=foo; }
+
+# This is necessary in the unusual event that there is
+# no valid entry in /etc/mtab.
+df_setup () { args=/; }
+
+# This is necessary in the unusual event that getpwuid (getuid ()) fails.
+id_setup () { args=-u; }
+
+# Use env to avoid invoking built-in sleep of Solaris 11's /bin/sh.
+kill_setup () {
+  env sleep 10m &
+  args=$!
+}
+
+link_setup () { args="$tmp_in link-target"; }
+unlink_setup () { args=$tmp_in; }
+
+readlink_setup () {
+  ln -s . slink
+  args=slink;
+}
+
+stat_setup () { args=$tmp_in; }
+unlink_setup () { args=$tmp_in; }
+lbracket_setup () { args=": ]"; }
+
+# Ensure that each program "works" (exits successfully) when doing
+# something more than --help or --version.
+for i in $built_programs; do
+  # Skip these.
+  case $i in chroot|stty|tty|false|chcon|runcon) continue;; esac
+
+  rm -rf $tmp_in $tmp_in2 $tmp_dir $tmp_out $bigZ_in $zin $zin2
+  echo z |gzip > $zin
+  cp $zin $zin2
+  cp $zin $bigZ_in
+
+  # This is sort of kludgey: use numbers so this is valid input for factor,
+  # and two tokens so it's valid input for tsort.
+  echo 2147483647 0 > $tmp_in
+  # Make $tmp_in2 identical. Then, using $tmp_in and $tmp_in2 as arguments
+  # to the likes of cmp and diff makes them exit successfully.
+  cp $tmp_in $tmp_in2
+  mkdir $tmp_dir
+  # echo ================== $i
+  test $i = [ && prog=lbracket || prog=$i
+  args=
+  if type ${prog}_setup > /dev/null 2>&1; then
+    ${prog}_setup
+  fi
+  if eval "env \$i $args < \$tmp_in > \$tmp_out"; then
+    : # ok
+  else
+    echo FAIL: $i
+    fail=1
+  fi
+  rm -rf $tmp_in $tmp_in2 $tmp_out $tmp_dir
+done
+
+Exit $fail

tests/gzip/hufts.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+# Exercise a bug whereby an invalid input could make gzip -d misbehave.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+printf '\n...: invalid compressed data--format violated\n' > exp \
+  || framework_failure_
+
+fail=0
+gzip -dc "$abs_srcdir/hufts-segv.gz" > out 2> err
+test $? = 1 || fail=1
+
+compare /dev/null out || fail=1
+
+sed 's/.*hufts-segv.gz: /...: /' err > k; mv k err || fail=1
+compare exp err || fail=1
+
+Exit $fail

tests/gzip/init.cfg

@@ -0,0 +1,5 @@
+# This file is sourced by init.sh, *before* its initialization.
+
+# This goes hand in hand with the "exec 9>&2;" in Makefile.am's
+# TESTS_ENVIRONMENT definition.
+stderr_fileno_=9

tests/gzip/init.sh

@@ -0,0 +1,616 @@
+# source this file; set up for tests
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Using this file in a test
+# =========================
+#
+# The typical skeleton of a test looks like this:
+#
+#   #!/bin/sh
+#   . "${srcdir=.}/init.sh"; path_prepend_ .
+#   Execute some commands.
+#   Note that these commands are executed in a subdirectory, therefore you
+#   need to prepend "../" to relative filenames in the build directory.
+#   Note that the "path_prepend_ ." is useful only if the body of your
+#   test invokes programs residing in the initial directory.
+#   For example, if the programs you want to test are in src/, and this test
+#   script is named tests/test-1, then you would use "path_prepend_ ../src",
+#   or perhaps export PATH='$(abs_top_builddir)/src$(PATH_SEPARATOR)'"$$PATH"
+#   to all tests via automake's TESTS_ENVIRONMENT.
+#   Set the exit code 0 for success, 77 for skipped, or 1 or other for failure.
+#   Use the skip_ and fail_ functions to print a diagnostic and then exit
+#   with the corresponding exit code.
+#   Exit $?
+
+# Executing a test that uses this file
+# ====================================
+#
+# Running a single test:
+#   $ make check TESTS=test-foo.sh
+#
+# Running a single test, with verbose output:
+#   $ make check TESTS=test-foo.sh VERBOSE=yes
+#
+# Running a single test, with single-stepping:
+#   1. Go into a sub-shell:
+#   $ bash
+#   2. Set relevant environment variables from TESTS_ENVIRONMENT in the
+#      Makefile:
+#   $ export srcdir=../../tests # this is an example
+#   3. Execute the commands from the test, copy&pasting them one by one:
+#   $ . "$srcdir/init.sh"; path_prepend_ .
+#   ...
+#   4. Finally
+#   $ exit
+
+ME_=`expr "./$0" : '.*/\(.*\)$'`
+
+# We use a trap below for cleanup.  This requires us to go through
+# hoops to get the right exit status transported through the handler.
+# So use 'Exit STATUS' instead of 'exit STATUS' inside of the tests.
+# Turn off errexit here so that we don't trip the bug with OSF1/Tru64
+# sh inside this function.
+Exit () { set +e; (exit $1); exit $1; }
+
+# Print warnings (e.g., about skipped and failed tests) to this file number.
+# Override by defining to say, 9, in init.cfg, and putting say,
+#   export ...ENVVAR_SETTINGS...; $(SHELL) 9>&2
+# in the definition of TESTS_ENVIRONMENT in your tests/Makefile.am file.
+# This is useful when using automake's parallel tests mode, to print
+# the reason for skip/failure to console, rather than to the .log files.
+: ${stderr_fileno_=2}
+
+# Note that correct expansion of "$*" depends on IFS starting with ' '.
+# Always write the full diagnostic to stderr.
+# When stderr_fileno_ is not 2, also emit the first line of the
+# diagnostic to that file descriptor.
+warn_ ()
+{
+  # If IFS does not start with ' ', set it and emit the warning in a subshell.
+  case $IFS in
+    ' '*) printf '%s\n' "$*" >&2
+          test $stderr_fileno_ = 2 \
+            || { printf '%s\n' "$*" | sed 1q >&$stderr_fileno_ ; } ;;
+    *) (IFS=' '; warn_ "$@");;
+  esac
+}
+fail_ () { warn_ "$ME_: failed test: $@"; Exit 1; }
+skip_ () { warn_ "$ME_: skipped test: $@"; Exit 77; }
+fatal_ () { warn_ "$ME_: hard error: $@"; Exit 99; }
+framework_failure_ () { warn_ "$ME_: set-up failure: $@"; Exit 99; }
+
+# This is used to simplify checking of the return value
+# which is useful when ensuring a command fails as desired.
+# I.e., just doing `command ... &&fail=1` will not catch
+# a segfault in command for example.  With this helper you
+# instead check an explicit exit code like
+#   returns_ 1 command ... || fail
+returns_ () {
+  # Disable tracing so it doesn't interfere with stderr of the wrapped command
+  { set +x; } 2>/dev/null
+
+  local exp_exit="$1"
+  shift
+  "$@"
+  test $? -eq $exp_exit && ret_=0 || ret_=1
+
+  if test "$VERBOSE" = yes && test "$gl_set_x_corrupts_stderr_" = false; then
+    set -x
+  fi
+  { return $ret_; } 2>/dev/null
+}
+
+# Sanitize this shell to POSIX mode, if possible.
+DUALCASE=1; export DUALCASE
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  alias -g '${1+"$@"}'='"$@"'
+  setopt NO_GLOB_SUBST
+else
+  case `(set -o) 2>/dev/null` in
+    *posix*) set -o posix ;;
+  esac
+fi
+
+# We require $(...) support unconditionally.
+# We require a few additional shell features only when $EXEEXT is nonempty,
+# in order to support automatic $EXEEXT emulation:
+# - hyphen-containing alias names
+# - we prefer to use ${var#...} substitution, rather than having
+#   to work around lack of support for that feature.
+# The following code attempts to find a shell with support for these features.
+# If the current shell passes the test, we're done.  Otherwise, test other
+# shells until we find one that passes.  If one is found, re-exec it.
+# If no acceptable shell is found, skip the current test.
+#
+# The "...set -x; P=1 true 2>err..." test is to disqualify any shell that
+# emits "P=1" into err, as /bin/sh from SunOS 5.11 and OpenBSD 4.7 do.
+#
+# Use "9" to indicate success (rather than 0), in case some shell acts
+# like Solaris 10's /bin/sh but exits successfully instead of with status 2.
+
+# Eval this code in a subshell to determine a shell's suitability.
+# 10 - passes all tests; ok to use
+#  9 - ok, but enabling "set -x" corrupts app stderr; prefer higher score
+#  ? - not ok
+gl_shell_test_script_='
+test $(echo y) = y || exit 1
+f_local_() { local v=1; }; f_local_ || exit 1
+score_=10
+if test "$VERBOSE" = yes; then
+  test -n "$( (exec 3>&1; set -x; P=1 true 2>&3) 2> /dev/null)" && score_=9
+fi
+test -z "$EXEEXT" && exit $score_
+shopt -s expand_aliases
+alias a-b="echo zoo"
+v=abx
+     test ${v%x} = ab \
+  && test ${v#a} = bx \
+  && test $(a-b) = zoo \
+  && exit $score_
+'
+
+if test "x$1" = "x--no-reexec"; then
+  shift
+else
+  # Assume a working shell.  Export to subshells (setup_ needs this).
+  gl_set_x_corrupts_stderr_=false
+  export gl_set_x_corrupts_stderr_
+
+  # Record the first marginally acceptable shell.
+  marginal_=
+
+  # Search for a shell that meets our requirements.
+  for re_shell_ in __current__ "${CONFIG_SHELL:-no_shell}" \
+      /bin/sh bash dash zsh pdksh fail
+  do
+    test "$re_shell_" = no_shell && continue
+
+    # If we've made it all the way to the sentinel, "fail" without
+    # finding even a marginal shell, skip this test.
+    if test "$re_shell_" = fail; then
+      test -z "$marginal_" && skip_ failed to find an adequate shell
+      re_shell_=$marginal_
+      break
+    fi
+
+    # When testing the current shell, simply "eval" the test code.
+    # Otherwise, run it via $re_shell_ -c ...
+    if test "$re_shell_" = __current__; then
+      # 'eval'ing this code makes Solaris 10's /bin/sh exit with
+      # $? set to 2.  It does not evaluate any of the code after the
+      # "unexpected" first '('.  Thus, we must run it in a subshell.
+      ( eval "$gl_shell_test_script_" ) > /dev/null 2>&1
+    else
+      "$re_shell_" -c "$gl_shell_test_script_" 2>/dev/null
+    fi
+
+    st_=$?
+
+    # $re_shell_ works just fine.  Use it.
+    if test $st_ = 10; then
+      gl_set_x_corrupts_stderr_=false
+      break
+    fi
+
+    # If this is our first marginally acceptable shell, remember it.
+    if test "$st_:$marginal_" = 9: ; then
+      marginal_="$re_shell_"
+      gl_set_x_corrupts_stderr_=true
+    fi
+  done
+
+  if test "$re_shell_" != __current__; then
+    # Found a usable shell.  Preserve -v and -x.
+    case $- in
+      *v*x* | *x*v*) opts_=-vx ;;
+      *v*) opts_=-v ;;
+      *x*) opts_=-x ;;
+      *) opts_= ;;
+    esac
+    re_shell=$re_shell_
+    export re_shell
+    exec "$re_shell_" $opts_ "$0" --no-reexec "$@"
+    echo "$ME_: exec failed" 1>&2
+    exit 127
+  fi
+fi
+
+# If this is bash, turn off all aliases.
+test -n "$BASH_VERSION" && unalias -a
+
+# Note that when supporting $EXEEXT (transparently mapping from PROG_NAME to
+# PROG_NAME.exe), we want to support hyphen-containing names like test-acos.
+# That is part of the shell-selection test above.  Why use aliases rather
+# than functions?  Because support for hyphen-containing aliases is more
+# widespread than that for hyphen-containing function names.
+test -n "$EXEEXT" && shopt -s expand_aliases
+
+# Enable glibc's malloc-perturbing option.
+# This is useful for exposing code that depends on the fact that
+# malloc-related functions often return memory that is mostly zeroed.
+# If you have the time and cycles, use valgrind to do an even better job.
+: ${MALLOC_PERTURB_=87}
+export MALLOC_PERTURB_
+
+# This is a stub function that is run upon trap (upon regular exit and
+# interrupt).  Override it with a per-test function, e.g., to unmount
+# a partition, or to undo any other global state changes.
+cleanup_ () { :; }
+
+# Emit a header similar to that from diff -u;  Print the simulated "diff"
+# command so that the order of arguments is clear.  Don't bother with @@ lines.
+emit_diff_u_header_ ()
+{
+  printf '%s\n' "diff -u $*" \
+    "--- $1	1970-01-01" \
+    "+++ $2	1970-01-01"
+}
+
+# Arrange not to let diff or cmp operate on /dev/null,
+# since on some systems (at least OSF/1 5.1), that doesn't work.
+# When there are not two arguments, or no argument is /dev/null, return 2.
+# When one argument is /dev/null and the other is not empty,
+# cat the nonempty file to stderr and return 1.
+# Otherwise, return 0.
+compare_dev_null_ ()
+{
+  test $# = 2 || return 2
+
+  if test "x$1" = x/dev/null; then
+    test -s "$2" || return 0
+    emit_diff_u_header_ "$@"; sed 's/^/+/' "$2"
+    return 1
+  fi
+
+  if test "x$2" = x/dev/null; then
+    test -s "$1" || return 0
+    emit_diff_u_header_ "$@"; sed 's/^/-/' "$1"
+    return 1
+  fi
+
+  return 2
+}
+
+if diff_out_=`exec 2>/dev/null; diff -u "$0" "$0" < /dev/null` \
+   && diff -u Makefile "$0" 2>/dev/null | grep '^[+]#!' >/dev/null; then
+  # diff accepts the -u option and does not (like AIX 7 'diff') produce an
+  # extra space on column 1 of every content line.
+  if test -z "$diff_out_"; then
+    compare_ () { diff -u "$@"; }
+  else
+    compare_ ()
+    {
+      if diff -u "$@" > diff.out; then
+        # No differences were found, but Solaris 'diff' produces output
+        # "No differences encountered". Hide this output.
+        rm -f diff.out
+        true
+      else
+        cat diff.out
+        rm -f diff.out
+        false
+      fi
+    }
+  fi
+elif
+  for diff_opt_ in -U3 -c '' no; do
+    test "$diff_opt_" = no && break
+    diff_out_=`exec 2>/dev/null; diff $diff_opt_ "$0" "$0" </dev/null` && break
+  done
+  test "$diff_opt_" != no
+then
+  if test -z "$diff_out_"; then
+    compare_ () { diff $diff_opt_ "$@"; }
+  else
+    compare_ ()
+    {
+      if diff $diff_opt_ "$@" > diff.out; then
+        # No differences were found, but AIX and HP-UX 'diff' produce output
+        # "No differences encountered" or "There are no differences between the
+        # files.". Hide this output.
+        rm -f diff.out
+        true
+      else
+        cat diff.out
+        rm -f diff.out
+        false
+      fi
+    }
+  fi
+elif cmp -s /dev/null /dev/null 2>/dev/null; then
+  compare_ () { cmp -s "$@"; }
+else
+  compare_ () { cmp "$@"; }
+fi
+
+# Usage: compare EXPECTED ACTUAL
+#
+# Given compare_dev_null_'s preprocessing, defer to compare_ if 2 or more.
+# Otherwise, propagate $? to caller: any diffs have already been printed.
+compare ()
+{
+  # This looks like it can be factored to use a simple "case $?"
+  # after unchecked compare_dev_null_ invocation, but that would
+  # fail in a "set -e" environment.
+  if compare_dev_null_ "$@"; then
+    return 0
+  else
+    case $? in
+      1) return 1;;
+      *) compare_ "$@";;
+    esac
+  fi
+}
+
+# An arbitrary prefix to help distinguish test directories.
+testdir_prefix_ () { printf gt; }
+
+# Run the user-overridable cleanup_ function, remove the temporary
+# directory and exit with the incoming value of $?.
+remove_tmp_ ()
+{
+  __st=$?
+  cleanup_
+  # cd out of the directory we're about to remove
+  cd "$initial_cwd_" || cd / || cd /tmp
+  chmod -R u+rwx "$test_dir_"
+  # If removal fails and exit status was to be 0, then change it to 1.
+  rm -rf "$test_dir_" || { test $__st = 0 && __st=1; }
+  exit $__st
+}
+
+# Given a directory name, DIR, if every entry in it that matches *.exe
+# contains only the specified bytes (see the case stmt below), then print
+# a space-separated list of those names and return 0.  Otherwise, don't
+# print anything and return 1.  Naming constraints apply also to DIR.
+find_exe_basenames_ ()
+{
+  feb_dir_=$1
+  feb_fail_=0
+  feb_result_=
+  feb_sp_=
+  for feb_file_ in $feb_dir_/*.exe; do
+    # If there was no *.exe file, or there existed a file named "*.exe" that
+    # was deleted between the above glob expansion and the existence test
+    # below, just skip it.
+    test "x$feb_file_" = "x$feb_dir_/*.exe" && test ! -f "$feb_file_" \
+      && continue
+    # Exempt [.exe, since we can't create a function by that name, yet
+    # we can't invoke [ by PATH search anyways due to shell builtins.
+    test "x$feb_file_" = "x$feb_dir_/[.exe" && continue
+    case $feb_file_ in
+      *[!-a-zA-Z/0-9_.+]*) feb_fail_=1; break;;
+      *) # Remove leading file name components as well as the .exe suffix.
+         feb_file_=${feb_file_##*/}
+         feb_file_=${feb_file_%.exe}
+         feb_result_="$feb_result_$feb_sp_$feb_file_";;
+    esac
+    feb_sp_=' '
+  done
+  test $feb_fail_ = 0 && printf %s "$feb_result_"
+  return $feb_fail_
+}
+
+# Consider the files in directory, $1.
+# For each file name of the form PROG.exe, create an alias named
+# PROG that simply invokes PROG.exe, then return 0.  If any selected
+# file name or the directory name, $1, contains an unexpected character,
+# define no alias and return 1.
+create_exe_shims_ ()
+{
+  case $EXEEXT in
+    '') return 0 ;;
+    .exe) ;;
+    *) echo "$0: unexpected \$EXEEXT value: $EXEEXT" 1>&2; return 1 ;;
+  esac
+
+  base_names_=`find_exe_basenames_ $1` \
+    || { echo "$0 (exe_shim): skipping directory: $1" 1>&2; return 0; }
+
+  if test -n "$base_names_"; then
+    for base_ in $base_names_; do
+      alias "$base_"="$base_$EXEEXT"
+    done
+  fi
+
+  return 0
+}
+
+# Use this function to prepend to PATH an absolute name for each
+# specified, possibly-$initial_cwd_-relative, directory.
+path_prepend_ ()
+{
+  while test $# != 0; do
+    path_dir_=$1
+    case $path_dir_ in
+      '') fail_ "invalid path dir: '$1'";;
+      /*) abs_path_dir_=$path_dir_;;
+      *) abs_path_dir_=$initial_cwd_/$path_dir_;;
+    esac
+    case $abs_path_dir_ in
+      *:*) fail_ "invalid path dir: '$abs_path_dir_'";;
+    esac
+    PATH="$abs_path_dir_:$PATH"
+
+    # Create an alias, FOO, for each FOO.exe in this directory.
+    create_exe_shims_ "$abs_path_dir_" \
+      || fail_ "something failed (above): $abs_path_dir_"
+    shift
+  done
+  export PATH
+}
+
+setup_ ()
+{
+  if test "$VERBOSE" = yes; then
+    # Test whether set -x may cause the selected shell to corrupt an
+    # application's stderr.  Many do, including zsh-4.3.10 and the /bin/sh
+    # from SunOS 5.11, OpenBSD 4.7 and Irix 5.x and 6.5.
+    # If enabling verbose output this way would cause trouble, simply
+    # issue a warning and refrain.
+    if $gl_set_x_corrupts_stderr_; then
+      warn_ "using SHELL=$SHELL with 'set -x' corrupts stderr"
+    else
+      set -x
+    fi
+  fi
+
+  initial_cwd_=$PWD
+
+  pfx_=`testdir_prefix_`
+  test_dir_=`mktempd_ "$initial_cwd_" "$pfx_-$ME_.XXXX"` \
+    || fail_ "failed to create temporary directory in $initial_cwd_"
+  cd "$test_dir_" || fail_ "failed to cd to temporary directory"
+
+  # As autoconf-generated configure scripts do, ensure that IFS
+  # is defined initially, so that saving and restoring $IFS works.
+  gl_init_sh_nl_='
+'
+  IFS=" ""	$gl_init_sh_nl_"
+
+  # This trap statement, along with a trap on 0 below, ensure that the
+  # temporary directory, $test_dir_, is removed upon exit as well as
+  # upon receipt of any of the listed signals.
+  for sig_ in 1 2 3 13 15; do
+    eval "trap 'Exit $(expr $sig_ + 128)' $sig_"
+  done
+}
+
+# Create a temporary directory, much like mktemp -d does.
+# Written by Jim Meyering.
+#
+# Usage: mktempd_ /tmp phoey.XXXXXXXXXX
+#
+# First, try to use the mktemp program.
+# Failing that, we'll roll our own mktemp-like function:
+#  - try to get random bytes from /dev/urandom
+#  - failing that, generate output from a combination of quickly-varying
+#      sources and gzip.  Ignore non-varying gzip header, and extract
+#      "random" bits from there.
+#  - given those bits, map to file-name bytes using tr, and try to create
+#      the desired directory.
+#  - make only $MAX_TRIES_ attempts
+
+# Helper function.  Print $N pseudo-random bytes from a-zA-Z0-9.
+rand_bytes_ ()
+{
+  n_=$1
+
+  # Maybe try openssl rand -base64 $n_prime_|tr '+/=\012' abcd first?
+  # But if they have openssl, they probably have mktemp, too.
+
+  chars_=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
+  dev_rand_=/dev/urandom
+  if test -r "$dev_rand_"; then
+    # Note: 256-length($chars_) == 194; 3 copies of $chars_ is 186 + 8 = 194.
+    dd ibs=$n_ count=1 if=$dev_rand_ 2>/dev/null \
+      | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_
+    return
+  fi
+
+  n_plus_50_=`expr $n_ + 50`
+  cmds_='date; date +%N; free; who -a; w; ps auxww; ps ef; netstat -n'
+  data_=` (eval "$cmds_") 2>&1 | gzip `
+
+  # Ensure that $data_ has length at least 50+$n_
+  while :; do
+    len_=`echo "$data_"|wc -c`
+    test $n_plus_50_ -le $len_ && break;
+    data_=` (echo "$data_"; eval "$cmds_") 2>&1 | gzip `
+  done
+
+  echo "$data_" \
+    | dd bs=1 skip=50 count=$n_ 2>/dev/null \
+    | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_
+}
+
+mktempd_ ()
+{
+  case $# in
+  2);;
+  *) fail_ "Usage: mktempd_ DIR TEMPLATE";;
+  esac
+
+  destdir_=$1
+  template_=$2
+
+  MAX_TRIES_=4
+
+  # Disallow any trailing slash on specified destdir:
+  # it would subvert the post-mktemp "case"-based destdir test.
+  case $destdir_ in
+  / | //) destdir_slash_=$destdir;;
+  */) fail_ "invalid destination dir: remove trailing slash(es)";;
+  *) destdir_slash_=$destdir_/;;
+  esac
+
+  case $template_ in
+  *XXXX) ;;
+  *) fail_ \
+       "invalid template: $template_ (must have a suffix of at least 4 X's)";;
+  esac
+
+  # First, try to use mktemp.
+  d=`unset TMPDIR; { mktemp -d -t -p "$destdir_" "$template_"; } 2>/dev/null` &&
+
+  # The resulting name must be in the specified directory.
+  case $d in "$destdir_slash_"*) :;; *) false;; esac &&
+
+  # It must have created the directory.
+  test -d "$d" &&
+
+  # It must have 0700 permissions.  Handle sticky "S" bits.
+  perms=`ls -dgo "$d" 2>/dev/null` &&
+  case $perms in drwx--[-S]---*) :;; *) false;; esac && {
+    echo "$d"
+    return
+  }
+
+  # If we reach this point, we'll have to create a directory manually.
+
+  # Get a copy of the template without its suffix of X's.
+  base_template_=`echo "$template_"|sed 's/XX*$//'`
+
+  # Calculate how many X's we've just removed.
+  template_length_=`echo "$template_" | wc -c`
+  nx_=`echo "$base_template_" | wc -c`
+  nx_=`expr $template_length_ - $nx_`
+
+  err_=
+  i_=1
+  while :; do
+    X_=`rand_bytes_ $nx_`
+    candidate_dir_="$destdir_slash_$base_template_$X_"
+    err_=`mkdir -m 0700 "$candidate_dir_" 2>&1` \
+      && { echo "$candidate_dir_"; return; }
+    test $MAX_TRIES_ -le $i_ && break;
+    i_=`expr $i_ + 1`
+  done
+  fail_ "$err_"
+}
+
+# If you want to override the testdir_prefix_ function,
+# or to add more utility functions, use this file.
+test -f "$srcdir/init.cfg" \
+  && . "$srcdir/init.cfg"
+
+setup_ "$@"
+# This trap is here, rather than in the setup_ function, because some
+# shells run the exit trap at shell function exit, rather than script exit.
+trap remove_tmp_ 0

tests/gzip/keep.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+# Exercise the --keep option.
+
+# Copyright (C) 2013-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+echo fooooooooo > in || framework_failure_
+cp in orig || framework_failure_
+
+fail=0
+
+# Compress and decompress both with and without --keep.
+for k in --keep ''; do
+  # With --keep, the source must be retained, otherwise, it must be removed.
+  case $k in --keep) op='||' ;; *) op='&&' ;; esac
+
+  gzip $k in || fail=1
+  eval "test -f in $op fail=1"
+  test -f in.gz || fail=1
+  rm -f in || fail=1
+
+  gzip -d $k in.gz || fail=1
+  eval "test -f in.gz $op fail=1"
+  test -f in || fail=1
+  compare in orig || fail=1
+  rm -f in.gz || fail=1
+done
+
+cp orig in || framework_failure_
+log=$(gzip -kv in 2>&1) || fail=1
+case $log in
+  *'created in.gz'*) ;;
+  *) fail=1;;
+esac
+
+Exit $fail

tests/gzip/list.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+# Exercise the --list option.
+
+# Copyright 2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+echo zoology zucchini > in || framework_failure_
+cp in orig || framework_failure_
+
+gzip -l in && fail=1
+gzip -9 in || fail=1
+gzip -l in.gz >out1 || fail=1
+gzip -l in.gz | cat >out2 || fail=1
+compare out1 out2 || fail=1
+
+Exit $fail

tests/gzip/memcpy-abuse.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+# Before gzip-1.4, this the use of memcpy in inflate_codes could
+# mistakenly operate on overlapping regions.  Exercise that code.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+# The input must be larger than 32KiB and slightly
+# less uniform than e.g., all zeros.
+printf wxy%032767d 0 | tee in | gzip > in.gz || framework_failure_
+
+fail=0
+
+# Before the fix, this would call memcpy with overlapping regions.
+gzip -dc in.gz > out || fail=1
+
+compare in out || fail=1
+
+Exit $fail

tests/gzip/mixed.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+# Ensure that gzip -cdf handles mixed compressed/not-compressed data
+# Before gzip-1.5, it would produce invalid output.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+printf 'xxx\nyyy\n'      > exp2 || framework_failure_
+printf 'aaa\nbbb\nccc\n' > exp3 || framework_failure_
+
+fail=0
+
+(echo xxx; echo yyy) > in || fail=1
+gzip -cdf < in > out || fail=1
+compare exp2 out || fail=1
+
+# Uncompressed input, followed by compressed data.
+# Currently fails, so skip it.
+# (echo xxx; echo yyy|gzip) > in || fail=1
+# gzip -cdf < in > out || fail=1
+# compare exp2 out || fail=1
+
+# Compressed input, followed by regular (not-compressed) data.
+(echo xxx|gzip; echo yyy) > in || fail=1
+gzip -cdf < in > out || fail=1
+compare exp2 out || fail=1
+
+(echo xxx|gzip; echo yyy|gzip) > in || fail=1
+gzip -cdf < in > out || fail=1
+compare exp2 out || fail=1
+
+in_str=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-+=%
+for i in 0 1 2 3 4 5 6 7 8 9 a; do in_str="$in_str$in_str" ;done
+
+# Start with some small sizes.  $(seq 64)
+sizes=$(i=0; while :; do echo $i; test $i = 64 && break; i=$(expr $i + 1); done)
+
+# gzip's internal buffer size is 32KiB + 64 bytes:
+sizes="$sizes 32831 32832 32833"
+
+# 128KiB, +/- 1
+sizes="$sizes 131071 131072 131073"
+
+# Ensure that "gzip -cdf" acts like cat, for a range of small input files.
+i=0
+for i in $sizes; do
+  echo $i
+  printf %$i.${i}s $in_str > in
+  gzip -cdf < in > out
+  compare in out || fail=1
+done
+
+Exit $fail

tests/gzip/null-suffix-clobber.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Before gzip-1.5, gzip -d -S '' k.gz would delete F.gz and not create "F"
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+printf anything | gzip > F.gz || framework_failure_
+echo y > yes || framework_failure_
+echo "gzip: invalid suffix ''" > expected-err || framework_failure_
+
+fail=0
+
+gzip ---presume-input-tty -d -S '' F.gz < yes > out 2>err && fail=1
+
+compare /dev/null out || fail=1
+compare expected-err err || fail=1
+
+test -f F.gz || fail=1
+
+Exit $fail

tests/gzip/stdin.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+# Ensure that gzip interprets "-" as stdin.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+printf a | gzip > in || framework_failure_
+printf aaa > exp || framework_failure_
+
+fail=0
+gzip -dc in - in < in > out 2>err || fail=1
+
+compare exp out || fail=1
+compare /dev/null err || fail=1
+
+Exit $fail

tests/gzip/test-driver.sh

@@ -0,0 +1,150 @@
+#! /bin/sh
+# test-driver - basic testsuite driver script.
+
+scriptversion=2016-01-11.22; # UTC
+
+# Copyright (C) 2011-2015 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to <bug-automake@gnu.org> or send patches to
+# <automake-patches@gnu.org>.
+
+# Make unconditional expansion of undefined variables an error.  This
+# helps a lot in preventing typo-related bugs.
+set -u
+
+usage_error ()
+{
+  echo "$0: $*" >&2
+  print_usage >&2
+  exit 2
+}
+
+print_usage ()
+{
+  cat <<END
+Usage:
+  test-driver --test-name=NAME --log-file=PATH --trs-file=PATH
+              [--expect-failure={yes|no}] [--color-tests={yes|no}]
+              [--enable-hard-errors={yes|no}] [--]
+              TEST-SCRIPT [TEST-SCRIPT-ARGUMENTS]
+The '--test-name', '--log-file' and '--trs-file' options are mandatory.
+END
+}
+
+test_name= # Used for reporting.
+log_file=  # Where to save the output of the test script.
+trs_file=  # Where to save the metadata of the test run.
+expect_failure=no
+color_tests=no
+enable_hard_errors=yes
+while test $# -gt 0; do
+  case $1 in
+  --help) print_usage; exit $?;;
+  --version) echo "test-driver $scriptversion"; exit $?;;
+  --test-name) test_name=$2; shift;;
+  --log-file) log_file=$2; shift;;
+  --trs-file) trs_file=$2; shift;;
+  --color-tests) color_tests=$2; shift;;
+  --expect-failure) expect_failure=$2; shift;;
+  --enable-hard-errors) enable_hard_errors=$2; shift;;
+  --) shift; break;;
+  -*) usage_error "invalid option: '$1'";;
+   *) break;;
+  esac
+  shift
+done
+
+missing_opts=
+test x"$test_name" = x && missing_opts="$missing_opts --test-name"
+test x"$log_file"  = x && missing_opts="$missing_opts --log-file"
+test x"$trs_file"  = x && missing_opts="$missing_opts --trs-file"
+if test x"$missing_opts" != x; then
+  usage_error "the following mandatory options are missing:$missing_opts"
+fi
+
+if test $# -eq 0; then
+  usage_error "missing argument"
+fi
+
+if test $color_tests = yes; then
+  # Keep this in sync with 'lib/am/check.am:$(am__tty_colors)'.
+  red='' # Red.
+  grn='' # Green.
+  lgn='' # Light green.
+  blu='' # Blue.
+  mgn='' # Magenta.
+  std=''     # No color.
+else
+  red= grn= lgn= blu= mgn= std=
+fi
+
+do_exit='rm -f $log_file $trs_file; (exit $st); exit $st'
+trap "st=129; $do_exit" 1
+trap "st=130; $do_exit" 2
+trap "st=141; $do_exit" 13
+trap "st=143; $do_exit" 15
+
+# Test script is run here.
+"$@" >$log_file 2>&1
+estatus=$?
+
+if test $enable_hard_errors = no && test $estatus -eq 99; then
+  tweaked_estatus=1
+else
+  tweaked_estatus=$estatus
+fi
+
+case $tweaked_estatus:$expect_failure in
+  0:yes) col=$red res=XPASS recheck=yes gcopy=yes;;
+  0:*)   col=$grn res=PASS  recheck=no  gcopy=no;;
+  77:*)  col=$blu res=SKIP  recheck=no  gcopy=yes;;
+  99:*)  col=$mgn res=ERROR recheck=yes gcopy=yes;;
+  *:yes) col=$lgn res=XFAIL recheck=no  gcopy=yes;;
+  *:*)   col=$red res=FAIL  recheck=yes gcopy=yes;;
+esac
+
+# Report the test outcome and exit status in the logs, so that one can
+# know whether the test passed or failed simply by looking at the '.log'
+# file, without the need of also peaking into the corresponding '.trs'
+# file (automake bug#11814).
+echo "$res $test_name (exit status: $estatus)" >>$log_file
+
+# Report outcome to console.
+echo "${col}${res}${std}: $test_name"
+
+# Register the test result, and other relevant metadata.
+echo ":test-result: $res" > $trs_file
+echo ":global-test-result: $res" >> $trs_file
+echo ":recheck: $recheck" >> $trs_file
+echo ":copy-in-global-log: $gcopy" >> $trs_file
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC0"
+# time-stamp-end: "; # UTC"
+# End:
+
+exit $tweaked_estatus

tests/gzip/trailing-nul.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# gzip accepts trailing NUL bytes; don't fail if there is exactly one.
+# Before gzip-1.4, this would fail.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+(echo 0 | gzip; printf '\0') > 0.gz || framework_failure_
+(echo 00 | gzip; printf '\0\0') > 00.gz || framework_failure_
+(echo 1 | gzip; printf '\1') > 1.gz || framework_failure_
+
+fail=0
+
+for i in 0 00 1; do
+  gzip -d $i.gz; ret=$?
+  test $ret -eq $i || fail=1
+  test $ret = 1 && continue
+  echo $i > exp || fail=1
+  compare exp $i || fail=1
+done
+
+Exit $fail

tests/gzip/unpack-invalid.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# gzip should report invalid 'unpack' input when uncompressing.
+# With gzip-1.5, it would output invalid data instead.
+
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+for input in \
+  '\037\036\000\000\037\213\010\000\000\000\000\000\002\003\036\000\000\000\002\003\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\037\000\302\240\037\213\010\000\000\000\000\000\002\003\355\301' \
+  '\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\076\366\017\370\036\016\030\000\000\000\000\000\000\000\000\000\034\010\105\140\104\025\020\047\000\000\037\036\016\030\000\000\000'; do
+
+  printf "$input" >in || framework_failure_
+
+  if gzip -d <in >out 2>err; then
+    fail=1
+  else
+    fail=0
+  fi
+done
+
+Exit $fail

tests/gzip/z-suffix.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# Check that -Sz works.
+
+# Copyright 2014-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+printf anything > F && cp F G || framework_failure_
+gzip -Sz F || fail=1
+test ! -f F || fail=1
+test -f Fz || fail=1
+gzip -dSz F || fail=1
+test ! -f Fz || fail=1
+compare F G || fail\1
+
+Exit $fail

tests/gzip/zdiff.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+# Exercise zdiff with two compressed inputs.
+# Before gzip-1.4, this would fail.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+echo a > a || framework_failure_
+echo b > b || framework_failure_
+gzip a b || framework_failure_
+
+cat <<EOF > exp
+1c1
+< a
+---
+> b
+EOF
+
+fail=0
+zdiff a.gz b.gz > out 2>&1
+test $? = 1 || fail=1
+
+compare exp out || fail=1
+
+rm -f out
+# expect success, for equal files
+zdiff a.gz a.gz > out 2> err || fail=1
+# expect no output
+test -s out && fail=1
+# expect no stderr
+test -s err && fail=1
+
+Exit $fail

tests/gzip/zgrep-context.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+# Ensure that zgrep -15 works.  Before gzip-1.5, it would fail.
+
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+# A limited replacement for seq: handle 1 or 2 args; increment must be 1
+seq()
+{
+  case $# in
+    1) start=1  final=$1;;
+    2) start=$1 final=$2;;
+    *) echo you lose 1>&2; exit 1;;
+  esac
+  awk 'BEGIN{for(i='$start';i<='$final';i++) print i}' < /dev/null
+}
+
+seq 40 > in || framework_failure_
+gzip < in > in.gz || framework_failure_
+seq 2 32 > exp || framework_failure_
+
+: ${GREP=grep}
+$GREP -15 17 - < in > out && compare exp out || {
+  echo >&2 "$0: $GREP does not support context options; skipping this test"
+  exit 77
+}
+
+fail=0
+zgrep -15 17 - < in.gz > out || fail=1
+compare exp out || fail=1
+
+Exit $fail

tests/gzip/zgrep-f.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+# Ensure that zgrep -f - works like grep -f -
+# Before gzip-1.4, it would fail.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+printf 'needle\nn2\n' > n || framework_failure_
+cp n haystack || framework_failure_
+gzip haystack || framework_failure_
+
+fail=0
+zgrep -f - haystack.gz < n > out 2>&1 || fail=1
+
+compare out n || fail=1
+
+if ${BASH_VERSION+:} false; then
+  set +o posix
+  # This failed with gzip 1.6.
+  cat n n >nn || framework_failure_
+  eval 'zgrep -h -f <(cat n) haystack.gz haystack.gz' >out || fail=1
+  compare out nn || fail=1
+fi
+
+# This failed with gzip 1.4.
+echo a-b | zgrep -e - > /dev/null || fail=1
+
+Exit $fail

tests/gzip/zgrep-signal.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+# Check that zgrep is terminated gracefully by signal when
+# its grep/sed pipeline is terminated by a signal.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+echo a | gzip -c > f.gz || framework_failure_
+
+test "x$PERL" = x && PERL=perl
+("$PERL" -e 'use POSIX qw(dup2)') >/dev/null 2>&1 ||
+   skip_ "no suitable perl found"
+
+# Run the arguments as a command, in a process where stdout is a
+# dangling pipe and SIGPIPE has the default signal-handling action.
+# This can't be done portably in the shell, because if SIGPIPE is
+# ignored when the shell is entered, the shell might refuse to trap
+# it.  Fall back on Perl+POSIX, if available.  Take care to close the
+# pipe's read end before running the program; the equivalent of the
+# shell's "command | :" has a race condition in that COMMAND could
+# write before ":" exits.
+write_to_dangling_pipe () {
+  program=${1?}
+  shift
+  args=
+  for arg; do
+    args="$args, '$arg'"
+  done
+  "$PERL" -e '
+     use POSIX qw(dup2);
+     $SIG{PIPE} = "DEFAULT";
+     pipe my ($read_end, $write_end) or die "pipe: $!\n";
+     dup2 fileno $write_end, 1 or die "dup2: $!\n";
+     close $read_end or die "close: $!\n";
+     exec '"'$program'$args"';
+  '
+}
+
+write_to_dangling_pipe cat f.gz f.gz
+signal_status=$?
+test 128 -lt $signal_status ||
+  framework_failure_ 'signal handling busted on this host'
+
+fail=0
+
+write_to_dangling_pipe zgrep a f.gz f.gz
+test $? -eq $signal_status || fail=1
+
+Exit $fail

tests/gzip/znew-k.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+# Check that znew -K works without compress(1).
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+cat <<'EOF' >compress || framework_failure_
+#!/bin/sh
+echo >&2 'compress has been invoked'
+exit 1
+EOF
+chmod +x compress || framework_failure_
+
+# Note that the basename must have a length of 6 or greater.
+# Otherwise, "test -f $name" below would fail.
+name=123456.Z
+
+printf '%1012977s' ' ' | gzip -c > $name || framework_failure_
+
+fail=0
+
+znew -K $name || fail=1
+test -f $name || fail=1
+
+Exit $fail

tests/invalidDictionaries.c

@@ -0,0 +1,51 @@
+#include <stddef.h>
+#include "zstd.h"
+
+static const char invalidRepCode[] = {
+  0x37, 0xa4, 0x30, 0xec, 0x2a, 0x00, 0x00, 0x00, 0x39, 0x10, 0xc0, 0xc2,
+  0xa6, 0x00, 0x0c, 0x30, 0xc0, 0x00, 0x03, 0x0c, 0x30, 0x20, 0x72, 0xf8,
+  0xb4, 0x6d, 0x4b, 0x9f, 0xfc, 0x97, 0x29, 0x49, 0xb2, 0xdf, 0x4b, 0x29,
+  0x7d, 0x4a, 0xfc, 0x83, 0x18, 0x22, 0x75, 0x23, 0x24, 0x44, 0x4d, 0x02,
+  0xb7, 0x97, 0x96, 0xf6, 0xcb, 0xd1, 0xcf, 0xe8, 0x22, 0xea, 0x27, 0x36,
+  0xb7, 0x2c, 0x40, 0x46, 0x01, 0x08, 0x23, 0x01, 0x00, 0x00, 0x06, 0x1e,
+  0x3c, 0x83, 0x81, 0xd6, 0x18, 0xd4, 0x12, 0x3a, 0x04, 0x00, 0x80, 0x03,
+  0x08, 0x0e, 0x12, 0x1c, 0x12, 0x11, 0x0d, 0x0e, 0x0a, 0x0b, 0x0a, 0x09,
+  0x10, 0x0c, 0x09, 0x05, 0x04, 0x03, 0x06, 0x06, 0x06, 0x02, 0x00, 0x03,
+  0x00, 0x00, 0x02, 0x02, 0x00, 0x04, 0x06, 0x03, 0x06, 0x08, 0x24, 0x6b,
+  0x0d, 0x01, 0x10, 0x04, 0x81, 0x07, 0x00, 0x00, 0x04, 0xb9, 0x58, 0x18,
+  0x06, 0x59, 0x92, 0x43, 0xce, 0x28, 0xa5, 0x08, 0x88, 0xc0, 0x80, 0x88,
+  0x8c, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+  0x08, 0x00, 0x00, 0x00
+};
+
+typedef struct dictionary_s {
+  const char *data;
+  size_t size;
+} dictionary;
+
+static const dictionary dictionaries[] = {
+  {invalidRepCode, sizeof(invalidRepCode)},
+  {NULL, 0},
+};
+
+int main(int argc, const char** argv) {
+  const dictionary *dict;
+  for (dict = dictionaries; dict->data != NULL; ++dict) {
+    ZSTD_CDict *cdict;
+    ZSTD_DDict *ddict;
+    cdict = ZSTD_createCDict(dict->data, dict->size, 1);
+    if (cdict) {
+      ZSTD_freeCDict(cdict);
+      return 1;
+    }
+    ddict = ZSTD_createDDict(dict->data, dict->size);
+    if (ddict) {
+      ZSTD_freeDDict(ddict);
+      return 2;
+    }
+  }
+
+  (void)argc;
+  (void)argv;
+  return 0;
+}

tests/legacy.c

@@ -0,0 +1,229 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*
+    This program uses hard-coded data compressed with Zstd legacy versions
+    and tests that the API decompresses them correctly
+*/
+
+/*===========================================
+*   Dependencies
+*==========================================*/
+#include <stddef.h>     /* size_t */
+#include <stdlib.h>     /* malloc, free */
+#include <stdio.h>      /* fprintf */
+#include <string.h>     /* strlen */
+#include "zstd.h"
+#include "zstd_errors.h"
+
+/*===========================================
+*   Macros
+*==========================================*/
+#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
+
+/*===========================================
+*   Precompressed frames
+*==========================================*/
+const char* const COMPRESSED; /* content is at end of file */
+size_t const COMPRESSED_SIZE = 917;
+const char* const EXPECTED; /* content is at end of file */
+
+
+int testSimpleAPI(void)
+{
+    size_t const size = strlen(EXPECTED);
+    char* const output = malloc(size);
+
+    if (!output) {
+        DISPLAY("ERROR: Not enough memory!\n");
+        return 1;
+    }
+
+    {
+        size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE);
+        if (ZSTD_isError(ret)) {
+            if (ret == ZSTD_error_prefix_unknown) {
+                DISPLAY("ERROR: Invalid frame magic number, was this compiled "
+                        "without legacy support?\n");
+            } else {
+                DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
+            }
+            return 1;
+        }
+        if (ret != size) {
+            DISPLAY("ERROR: Wrong decoded size\n");
+        }
+    }
+    if (memcmp(EXPECTED, output, size) != 0) {
+        DISPLAY("ERROR: Wrong decoded output produced\n");
+        return 1;
+    }
+
+    free(output);
+    DISPLAY("Simple API OK\n");
+    return 0;
+}
+
+int testStreamingAPI(void)
+{
+    size_t const outBuffSize = ZSTD_DStreamOutSize();
+    char* const outBuff = malloc(outBuffSize);
+    ZSTD_DStream* const stream = ZSTD_createDStream();
+    ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 };
+    size_t outputPos = 0;
+    int needsInit = 1;
+
+    if (outBuff == NULL) {
+        DISPLAY("ERROR: Could not allocate memory\n");
+        return 1;
+    }
+    if (stream == NULL) {
+        DISPLAY("ERROR: Could not create dstream\n");
+        return 1;
+    }
+
+    while (1) {
+        ZSTD_outBuffer output = {outBuff, outBuffSize, 0};
+        if (needsInit) {
+            size_t const ret = ZSTD_initDStream(stream);
+            if (ZSTD_isError(ret)) {
+                DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
+                return 1;
+            }
+        }
+        {
+            size_t const ret = ZSTD_decompressStream(stream, &output, &input);
+            if (ZSTD_isError(ret)) {
+                DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
+                return 1;
+            }
+
+            if (ret == 0) {
+                needsInit = 1;
+            }
+        }
+
+        if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) {
+            DISPLAY("ERROR: Wrong decoded output produced\n");
+            return 1;
+        }
+        outputPos += output.pos;
+        if (input.pos == input.size && output.pos < output.size) {
+            break;
+        }
+    }
+
+    free(outBuff);
+    ZSTD_freeDStream(stream);
+    DISPLAY("Streaming API OK\n");
+    return 0;
+}
+
+int main(void)
+{
+    int ret;
+
+    ret = testSimpleAPI();
+    if (ret) return ret;
+    ret = testStreamingAPI();
+    if (ret) return ret;
+
+    DISPLAY("OK\n");
+
+    return 0;
+}
+
+/* Consists of the "EXPECTED" string compressed with default settings on
+    - v0.4.3
+    - v0.5.0
+    - v0.6.0
+    - v0.7.0
+    - v0.8.0
+*/
+const char* const COMPRESSED =
+    "\x24\xB5\x2F\xFD\x00\x00\x00\xBB\xB0\x02\xC0\x10\x00\x1E\xB0\x01"
+    "\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF\xE9\xF3\xEF\x53"
+    "\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89\x17\x00\x18\x00"
+    "\x18\x00\x3F\xE6\xE2\xE3\x74\xD6\xEC\xC9\x4A\xE0\x71\x71\x42\x3E"
+    "\x64\x4F\x6A\x45\x4E\x78\xEC\x49\x03\x3F\xC6\x80\xAB\x8F\x75\x5E"
+    "\x6F\x2E\x3E\x7E\xC6\xDC\x45\x69\x6C\xC5\xFD\xC7\x40\xB8\x84\x8A"
+    "\x01\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B\x67"
+    "\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\x19\x03\x01\x50\x67\x56\xF5\x9F"
+    "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF"
+    "\xC6\xBA\x01\x0E\x00\x54\x00\x00\x19\x00\x00\x54\x14\x00\x24\x24"
+    "\x04\xFE\x04\x84\x4E\x41\x00\x27\xE2\x02\xC4\xB1\x00\xD2\x51\x00"
+    "\x79\x58\x41\x28\x00\xE0\x0C\x01\x68\x65\x00\x04\x13\x0C\xDA\x0C"
+    "\x80\x22\x06\xC0\x00\x00\x25\xB5\x2F\xFD\x00\x00\x00\xAD\x12\xB0"
+    "\x7D\x1E\xB0\x01\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF"
+    "\xE9\xF3\xEF\x53\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89"
+    "\x03\x01\x50\x67\x56\xF5\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC"
+    "\xAB\xAB\xE0\xE2\x81\xFA\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C"
+    "\x64\xF8\xEB\x53\xE6\x18\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A"
+    "\xF9\x63\x0C\xB8\xFA\x58\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6"
+    "\x56\xDC\x7F\x0C\x84\x4B\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC"
+    "\x04\x1E\x17\x27\xE4\x43\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00"
+    "\x00\x32\x40\x80\xA8\x00\x01\x49\x81\xE0\x3C\x01\x29\x1D\x00\x87"
+    "\xCE\x80\x75\x08\x80\x72\x24\x00\x7B\x52\x00\x94\x00\x20\xCC\x01"
+    "\x86\xD2\x00\x81\x09\x83\xC1\x34\xA0\x88\x01\xC0\x00\x00\x26\xB5"
+    "\x2F\xFD\x42\xEF\x00\x00\xA6\x12\xB0\x7D\x1E\xB0\x01\x02\x00\x00"
+    "\x54\xA0\xBA\x24\x8D\xC4\x25\xF2\x77\xFA\xFC\xFB\x94\x7A\xC7\xC9"
+    "\x13\x03\x11\x24\x43\x63\x3C\x6D\x22\x03\x01\x50\x67\x56\xF5\x9F"
+    "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF"
+    "\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B"
+    "\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA\x58\xE7"
+    "\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84\x4B\xA8"
+    "\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4\x43\xF6"
+    "\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x35\x0B\x71\xB5\xC0\x2A\x5C"
+    "\x26\x94\x22\x20\x8B\x4C\x8D\x13\x47\x58\x67\x15\x6C\xF1\x1C\x4B"
+    "\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0\x00\x00"
+    "\x27\xB5\x2F\xFD\x20\xEF\x00\x00\xA6\x12\xE4\x84\x1F\xB0\x01\x10"
+    "\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF\x9F\xEF"
+    "\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21\x80\x32"
+    "\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20\x5F\x45"
+    "\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB\x44\x14"
+    "\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB\x89\x51"
+    "\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F\x59\xDB"
+    "\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E\xDF\x78"
+    "\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55\x4A\xD4"
+    "\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39\x43\xF1"
+    "\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0"
+    "\x00\x00\x28\xB5\x2F\xFD\x24\xEF\x35\x05\x00\x92\x0B\x21\x1F\xB0"
+    "\x01\x10\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF"
+    "\x9F\xEF\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21"
+    "\x80\x32\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20"
+    "\x5F\x45\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB"
+    "\x44\x14\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB"
+    "\x89\x51\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F"
+    "\x59\xDB\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E"
+    "\xDF\x78\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55"
+    "\x4A\xD4\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39"
+    "\x43\xF1\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D"
+    "\x01\xD2\x2F\x21\x80";
+
+const char* const EXPECTED =
+    "snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
+    "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
+    "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
+
+    "snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
+    "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
+    "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
+
+    "snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
+    "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
+    "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
+
+    "snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
+    "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
+    "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
+
+    "snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
+    "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
+    "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n";
+

tests/playTests.sh

@@ -7,17 +7,17 @@ die() {
 
 roundTripTest() {
     if [ -n "$3" ]; then
-        local c="$3"
-        local p="$2"
+        local_c="$3"
+        local_p="$2"
     else
-        local c="$2"
+        local_c="$2"
     fi
 
     rm -f tmp1 tmp2
-    $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
-    ./datagen $1 $p | $MD5SUM > tmp1
-    ./datagen $1 $p | $ZSTD --ultra -v$c | $ZSTD -d  | $MD5SUM > tmp2
-    diff -q tmp1 tmp2
+    $ECHO "roundTripTest: ./datagen $1 $local_p | $ZSTD -v$local_c | $ZSTD -d"
+    ./datagen $1 $local_p | $MD5SUM > tmp1
+    ./datagen $1 $local_p | $ZSTD --ultra -v$local_c | $ZSTD -d  | $MD5SUM > tmp2
+    $DIFF -q tmp1 tmp2
 }
 
 isWindows=false
@@ -27,7 +27,6 @@ case "$OS" in
   Windows*)
     isWindows=true
     ECHO="echo -e"
-    INTOVOID="nul"
     ;;
 esac
 
@@ -38,6 +37,12 @@ case "$UNAME" in
   *) MD5SUM="md5sum" ;;
 esac
 
+DIFF="diff"
+case "$UNAME" in
+  SunOS) DIFF="gdiff" ;;
+esac
+
+
 $ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
 
 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
@@ -96,11 +101,25 @@ $ZSTD -f --rm tmp
 ls tmp && die "tmp should no longer be present"
 $ZSTD -f -d --rm tmp.zst
 ls tmp.zst && die "tmp.zst should no longer be present"
+$ECHO "test : --rm on stdin"
+$ECHO a | $ZSTD --rm > $INTOVOID   # --rm should remain silent
 rm tmp
 $ZSTD -f tmp && die "tmp not present : should have failed"
 ls tmp.zst && die "tmp.zst should not be created"
 
 
+$ECHO "\n**** Advanced compression parameters **** "
+$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,      - -o tmp.zst && die "wrong parameters not detected!"
+$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21        - -o tmp.zst && die "wrong parameters not detected!"
+$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog  - -o tmp.zst && die "wrong parameters not detected!"
+ls tmp.zst && die "tmp.zst should not be created"
+roundTripTest -g512K
+roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
+roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
+roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
+roundTripTest -g512K 19
+
+
 $ECHO "\n**** Pass-Through mode **** "
 $ECHO "Hello world 1!" | $ZSTD -df
 $ECHO "Hello world 2!" | $ZSTD -dcf
@@ -118,19 +137,19 @@ $ZSTD -c world.tmp > world.zstd
 cat hello.zstd world.zstd > helloworld.zstd
 $ZSTD -dc helloworld.zstd > result.tmp
 cat result.tmp
-sdiff helloworld.tmp result.tmp
+$DIFF helloworld.tmp result.tmp
 $ECHO "frame concatenation without checksum"
 $ZSTD -c hello.tmp > hello.zstd --no-check
 $ZSTD -c world.tmp > world.zstd --no-check
 cat hello.zstd world.zstd > helloworld.zstd
 $ZSTD -dc helloworld.zstd > result.tmp
 cat result.tmp
-sdiff helloworld.tmp result.tmp
+$DIFF helloworld.tmp result.tmp
 rm ./*.tmp ./*.zstd
 $ECHO "frame concatenation tests completed"
 
 
-if [ "$isWindows" = false ] ; then
+if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then
 $ECHO "\n**** flush write error test **** "
 
 $ECHO "$ECHO foo | $ZSTD > /dev/full"
@@ -144,14 +163,14 @@ $ECHO "\n**** test sparse file support **** "
 
 ./datagen -g5M  -P100 > tmpSparse
 $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
-diff -s tmpSparse tmpSparseRegen
+$DIFF -s tmpSparse tmpSparseRegen
 $ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse
-diff -s tmpSparse tmpOutSparse
+$DIFF -s tmpSparse tmpOutSparse
 $ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
-diff -s tmpSparse tmpOutNoSparse
+$DIFF -s tmpSparse tmpOutNoSparse
 ls -ls tmpSparse*
 ./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd   # Odd size file (to not finish on an exact nb of blocks)
-./datagen -s1 -g1200007 -P100 | diff -s - tmpSparseOdd
+./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
 ls -ls tmpSparseOdd
 $ECHO "\n Sparse Compatibility with Console :"
 $ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c
@@ -163,7 +182,7 @@ $ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
 $ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
 $ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
 ls -ls tmpSparse*
-diff tmpSparse2M tmpSparseRegenerated
+$DIFF tmpSparse2M tmpSparseRegenerated
 rm tmpSparse*
 
 
@@ -192,18 +211,19 @@ $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
 
 $ECHO "\n**** dictionary tests **** "
 
-TESTFILE=../programs/zstdcli.c
+$ECHO "- test with raw dict (content only) "
 ./datagen > tmpDict
 ./datagen -g1M | $MD5SUM > tmp1
 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
-diff -q tmp1 tmp2
-$ECHO "- Create first dictionary"
+$DIFF -q tmp1 tmp2
+$ECHO "- Create first dictionary "
+TESTFILE=../programs/zstdcli.c
 $ZSTD --train *.c ../programs/*.c -o tmpDict
 cp $TESTFILE tmp
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -fo result
-diff $TESTFILE result
-$ECHO "- Create second (different) dictionary"
+$DIFF $TESTFILE result
+$ECHO "- Create second (different) dictionary "
 $ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
 $ECHO "- Create dictionary with short dictID"
@@ -218,7 +238,7 @@ $ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong orde
 $ECHO "- Compress without dictID"
 $ZSTD -f tmp -D tmpDict1 --no-dictID
 $ZSTD -d tmp.zst -D tmpDict -fo result
-diff $TESTFILE result
+$DIFF $TESTFILE result
 $ECHO "- Compress with wrong argument order (must fail)"
 $ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name "
 $ECHO "- Compress multiple files with dictionary"
@@ -238,6 +258,27 @@ rm -rf dirTestDict
 rm tmp*
 
 
+$ECHO "\n**** cover dictionary tests **** "
+
+TESTFILE=../programs/zstdcli.c
+./datagen > tmpDict
+$ECHO "- Create first dictionary"
+$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
+cp $TESTFILE tmp
+$ZSTD -f tmp -D tmpDict
+$ZSTD -d tmp.zst -D tmpDict -fo result
+$DIFF $TESTFILE result
+$ECHO "- Create second (different) dictionary"
+$ZSTD --train --cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
+$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
+$ECHO "- Create dictionary with short dictID"
+$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c --dictID 1 -o tmpDict1
+cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
+$ECHO "- Create dictionary with size limit"
+$ZSTD --train --optimize-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K
+rm tmp*
+
+
 $ECHO "\n**** integrity tests **** "
 
 $ECHO "test one file (tmp1.zst) "
@@ -263,11 +304,91 @@ $ECHO "\n**** benchmark mode tests **** "
 
 $ECHO "bench one file"
 ./datagen > tmp1
-$ZSTD -bi1 tmp1
+$ZSTD -bi0 tmp1
 $ECHO "bench multiple levels"
-$ZSTD -i1b1e3 tmp1
+$ZSTD -i0b0e3 tmp1
 $ECHO "with recursive and quiet modes"
-$ZSTD -rqi1b1e3 tmp1
+$ZSTD -rqi1b1e2 tmp1
+
+
+$ECHO "\n**** gzip compatibility tests **** "
+
+GZIPMODE=1
+$ZSTD --format=gzip -V || GZIPMODE=0
+if [ $GZIPMODE -eq 1 ]; then
+    $ECHO "gzip support detected"
+    GZIPEXE=1
+    gzip -V || GZIPEXE=0
+    if [ $GZIPEXE -eq 1 ]; then
+        ./datagen > tmp
+        $ZSTD --format=gzip -f tmp
+        gzip -t -v tmp.gz
+        gzip -f tmp
+        $ZSTD -d -f -v tmp.gz
+        rm tmp*
+    else
+        $ECHO "gzip binary not detected"
+    fi
+else
+    $ECHO "gzip mode not supported"
+fi
+
+
+$ECHO "\n**** gzip frame tests **** "
+
+if [ $GZIPMODE -eq 1 ]; then
+    ./datagen > tmp
+    $ZSTD -f --format=gzip tmp
+    $ZSTD -f tmp
+    cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
+    head -c -1 tmp.gz | $ZSTD -t && die "incomplete frame not detected !"
+    rm tmp*
+else
+    $ECHO "gzip mode not supported"
+fi
+
+
+$ECHO "\n**** xz compatibility tests **** "
+
+LZMAMODE=1
+$ZSTD --format=xz -V || LZMAMODE=0
+if [ $LZMAMODE -eq 1 ]; then
+    $ECHO "xz support detected"
+    XZEXE=1
+    xz -V && lzma -V || XZEXE=0
+    if [ $XZEXE -eq 1 ]; then
+        ./datagen > tmp
+        $ZSTD --format=lzma -f tmp
+        $ZSTD --format=xz -f tmp
+        xz -t -v tmp.xz
+        xz -t -v tmp.lzma
+        xz -f -k tmp
+        lzma -f -k --lzma1 tmp
+        $ZSTD -d -f -v tmp.xz
+        $ZSTD -d -f -v tmp.lzma
+        rm tmp*
+    else
+        $ECHO "xz binary not detected"
+    fi
+else
+    $ECHO "xz mode not supported"
+fi
+
+
+$ECHO "\n**** xz frame tests **** "
+
+if [ $LZMAMODE -eq 1 ]; then
+    ./datagen > tmp
+    $ZSTD -f --format=xz tmp
+    $ZSTD -f --format=lzma tmp
+    $ZSTD -f tmp
+    cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp
+    head -c -1 tmp.xz | $ZSTD -t && die "incomplete frame not detected !"
+    head -c -1 tmp.lzma | $ZSTD -t && die "incomplete frame not detected !"
+    rm tmp*
+else
+    $ECHO "xz mode not supported"
+fi
 
 
 $ECHO "\n**** zstd round-trip tests **** "
@@ -276,10 +397,10 @@ roundTripTest
 roundTripTest -g15K       # TableID==3
 roundTripTest -g127K      # TableID==2
 roundTripTest -g255K      # TableID==1
-roundTripTest -g513K      # TableID==0
-roundTripTest -g512K 6    # greedy, hash chain
-roundTripTest -g512K 16   # btlazy2
-roundTripTest -g512K 19   # btopt
+roundTripTest -g522K      # TableID==0
+roundTripTest -g519K 6    # greedy, hash chain
+roundTripTest -g517K 16   # btlazy2
+roundTripTest -g516K 19   # btopt
 
 rm tmp*
 

tests/pool.c

@@ -0,0 +1,70 @@
+#include "pool.h"
+#include "threading.h"
+#include <stddef.h>
+#include <stdio.h>
+
+#define ASSERT_TRUE(p)                                                         \
+  do {                                                                         \
+    if (!(p)) {                                                                \
+      return 1;                                                                \
+    }                                                                          \
+  } while (0)
+#define ASSERT_FALSE(p) ASSERT_TRUE(!(p))
+#define ASSERT_EQ(lhs, rhs) ASSERT_TRUE((lhs) == (rhs))
+
+struct data {
+  pthread_mutex_t mutex;
+  unsigned data[16];
+  size_t i;
+};
+
+void fn(void *opaque) {
+  struct data *data = (struct data *)opaque;
+  pthread_mutex_lock(&data->mutex);
+  data->data[data->i] = data->i;
+  ++data->i;
+  pthread_mutex_unlock(&data->mutex);
+}
+
+int testOrder(size_t numThreads, size_t queueSize) {
+  struct data data;
+  POOL_ctx *ctx = POOL_create(numThreads, queueSize);
+  ASSERT_TRUE(ctx);
+  data.i = 0;
+  pthread_mutex_init(&data.mutex, NULL);
+  {
+    size_t i;
+    for (i = 0; i < 16; ++i) {
+      POOL_add(ctx, &fn, &data);
+    }
+  }
+  POOL_free(ctx);
+  ASSERT_EQ(16, data.i);
+  {
+    size_t i;
+    for (i = 0; i < data.i; ++i) {
+      ASSERT_EQ(i, data.data[i]);
+    }
+  }
+  pthread_mutex_destroy(&data.mutex);
+  return 0;
+}
+
+int main(int argc, const char **argv) {
+  size_t numThreads;
+  for (numThreads = 1; numThreads <= 4; ++numThreads) {
+    size_t queueSize;
+    for (queueSize = 1; queueSize <= 2; ++queueSize) {
+      if (testOrder(numThreads, queueSize)) {
+        printf("FAIL: testOrder\n");
+        return 1;
+      }
+    }
+  }
+  printf("PASS: testOrder\n");
+  (void)argc;
+  (void)argv;
+  return (POOL_create(0, 1) || POOL_create(1, 0)) ? printf("FAIL: testInvalid\n"), 1
+                                                  : printf("PASS: testInvalid\n"), 0;
+  return 0;
+}

tests/symbols.c

@@ -0,0 +1,148 @@
+#include <stdio.h>
+#include "zstd_errors.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#define ZBUFF_DISABLE_DEPRECATE_WARNINGS
+#define ZBUFF_STATIC_LINKING_ONLY
+#include "zbuff.h"
+#define ZDICT_DISABLE_DEPRECATE_WARNINGS
+#define ZDICT_STATIC_LINKING_ONLY
+#include "zdict.h"
+
+static const void *symbols[] = {
+/* zstd.h */
+  &ZSTD_versionNumber,
+  &ZSTD_compress,
+  &ZSTD_decompress,
+  &ZSTD_getDecompressedSize,
+  &ZSTD_findDecompressedSize,
+  &ZSTD_findFrameCompressedSize,
+  &ZSTD_getFrameContentSize,
+  &ZSTD_maxCLevel,
+  &ZSTD_compressBound,
+  &ZSTD_isError,
+  &ZSTD_getErrorName,
+  &ZSTD_createCCtx,
+  &ZSTD_freeCCtx,
+  &ZSTD_compressCCtx,
+  &ZSTD_createDCtx,
+  &ZSTD_freeDCtx,
+  &ZSTD_decompressDCtx,
+  &ZSTD_compress_usingDict,
+  &ZSTD_decompress_usingDict,
+  &ZSTD_createCDict,
+  &ZSTD_freeCDict,
+  &ZSTD_compress_usingCDict,
+  &ZSTD_createDDict,
+  &ZSTD_freeDDict,
+  &ZSTD_decompress_usingDDict,
+  &ZSTD_createCStream,
+  &ZSTD_freeCStream,
+  &ZSTD_initCStream,
+  &ZSTD_compressStream,
+  &ZSTD_flushStream,
+  &ZSTD_endStream,
+  &ZSTD_CStreamInSize,
+  &ZSTD_CStreamOutSize,
+  &ZSTD_createDStream,
+  &ZSTD_freeDStream,
+  &ZSTD_initDStream,
+  &ZSTD_decompressStream,
+  &ZSTD_DStreamInSize,
+  &ZSTD_DStreamOutSize,
+/* zstd.h: advanced functions */
+  &ZSTD_estimateCCtxSize,
+  &ZSTD_createCCtx_advanced,
+  &ZSTD_sizeof_CCtx,
+  &ZSTD_createCDict_advanced,
+  &ZSTD_sizeof_CDict,
+  &ZSTD_getCParams,
+  &ZSTD_getParams,
+  &ZSTD_checkCParams,
+  &ZSTD_adjustCParams,
+  &ZSTD_compress_advanced,
+  &ZSTD_isFrame,
+  &ZSTD_estimateDCtxSize,
+  &ZSTD_createDCtx_advanced,
+  &ZSTD_sizeof_DCtx,
+  &ZSTD_sizeof_DDict,
+  &ZSTD_getDictID_fromDict,
+  &ZSTD_getDictID_fromDDict,
+  &ZSTD_getDictID_fromFrame,
+  &ZSTD_createCStream_advanced,
+  &ZSTD_initCStream_srcSize,
+  &ZSTD_initCStream_usingDict,
+  &ZSTD_initCStream_advanced,
+  &ZSTD_initCStream_usingCDict,
+  &ZSTD_resetCStream,
+  &ZSTD_sizeof_CStream,
+  &ZSTD_createDStream_advanced,
+  &ZSTD_initDStream_usingDict,
+  &ZSTD_setDStreamParameter,
+  &ZSTD_initDStream_usingDDict,
+  &ZSTD_resetDStream,
+  &ZSTD_sizeof_DStream,
+  &ZSTD_compressBegin,
+  &ZSTD_compressBegin_usingDict,
+  &ZSTD_compressBegin_advanced,
+  &ZSTD_copyCCtx,
+  &ZSTD_compressContinue,
+  &ZSTD_compressEnd,
+  &ZSTD_getFrameParams,
+  &ZSTD_decompressBegin,
+  &ZSTD_decompressBegin_usingDict,
+  &ZSTD_copyDCtx,
+  &ZSTD_nextSrcSizeToDecompress,
+  &ZSTD_decompressContinue,
+  &ZSTD_nextInputType,
+  &ZSTD_getBlockSizeMax,
+  &ZSTD_compressBlock,
+  &ZSTD_decompressBlock,
+  &ZSTD_insertBlock,
+/* zstd_errors.h */
+  &ZSTD_getErrorCode,
+  &ZSTD_getErrorString,
+/* zbuff.h */
+  &ZBUFF_createCCtx,
+  &ZBUFF_freeCCtx,
+  &ZBUFF_compressInit,
+  &ZBUFF_compressInitDictionary,
+  &ZBUFF_compressContinue,
+  &ZBUFF_compressFlush,
+  &ZBUFF_compressEnd,
+  &ZBUFF_createDCtx,
+  &ZBUFF_freeDCtx,
+  &ZBUFF_decompressInit,
+  &ZBUFF_decompressInitDictionary,
+  &ZBUFF_decompressContinue,
+  &ZBUFF_isError,
+  &ZBUFF_getErrorName,
+  &ZBUFF_recommendedCInSize,
+  &ZBUFF_recommendedCOutSize,
+  &ZBUFF_recommendedDInSize,
+  &ZBUFF_recommendedDOutSize,
+/* zbuff.h: advanced functions */
+  &ZBUFF_createCCtx_advanced,
+  &ZBUFF_createDCtx_advanced,
+  &ZBUFF_compressInit_advanced,
+/* zdict.h */
+  &ZDICT_trainFromBuffer,
+  &ZDICT_getDictID,
+  &ZDICT_isError,
+  &ZDICT_getErrorName,
+/* zdict.h: advanced functions */
+  &ZDICT_trainFromBuffer_advanced,
+  &ZDICT_addEntropyTablesFromBuffer,
+  NULL,
+};
+
+int main(int argc, const char** argv) {
+  const void **symbol;
+  (void)argc;
+  (void)argv;
+
+  for (symbol = symbols; *symbol != NULL; ++symbol) {
+    printf("%p\n", *symbol);
+  }
+  return 0;
+}

tests/zstreamtest.c

@@ -26,9 +26,11 @@
 #include <time.h>         /* clock_t, clock() */
 #include <string.h>       /* strcmp */
 #include "mem.h"
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel, ZSTD_customMem */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
 #include "zstd.h"         /* ZSTD_compressBound */
 #include "zstd_errors.h"  /* ZSTD_error_srcSize_wrong */
+#include "zstdmt_compress.h"
+#include "zdict.h"        /* ZDICT_trainFromBuffer */
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"       /* XXH64_* */
@@ -44,8 +46,7 @@
 static const U32 nbTestsDefault = 10000;
 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
 #define FUZ_COMPRESSIBILITY_DEFAULT 50
-static const U32 prime1 = 2654435761U;
-static const U32 prime2 = 2246822519U;
+static const U32 prime32 = 2654435761U;
 
 
 /*-************************************
@@ -81,8 +82,9 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart)
 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
 unsigned int FUZ_rand(unsigned int* seedPtr)
 {
+    static const U32 prime2 = 2246822519U;
     U32 rand32 = *seedPtr;
-    rand32 *= prime1;
+    rand32 *= prime32;
     rand32 += prime2;
     rand32  = FUZ_rotl32(rand32, 13);
     *seedPtr = rand32;
@@ -107,6 +109,41 @@ static void freeFunction(void* opaque, void* address)
 *   Basic Unit tests
 ======================================================*/
 
+typedef struct {
+    void* start;
+    size_t size;
+    size_t filled;
+} buffer_t;
+
+static const buffer_t g_nullBuffer = { NULL, 0 , 0 };
+
+static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
+{
+    buffer_t dict = { NULL, 0, 0 };
+    size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
+    size_t* const blockSizes = (size_t*) malloc(nbBlocks * sizeof(size_t));
+    if (!blockSizes) return dict;
+    dict.start = malloc(requestedDictSize);
+    if (!dict.start) { free(blockSizes); return dict; }
+    {   size_t nb;
+        for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
+        blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
+    }
+    {   size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
+        free(blockSizes);
+        if (ZDICT_isError(dictSize)) { free(dict.start); return (buffer_t){ NULL, 0, 0 }; }
+        dict.size = requestedDictSize;
+        dict.filled = dictSize;
+        return dict;   /* how to return dictSize ? */
+    }
+}
+
+static void FUZ_freeDictionary(buffer_t dict)
+{
+    free(dict.start);
+}
+
+
 static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
 {
     size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
@@ -123,21 +160,32 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
     ZSTD_inBuffer  inBuff, inBuff2;
     ZSTD_outBuffer outBuff;
+    buffer_t dictionary = g_nullBuffer;
+    unsigned dictID = 0;
 
     /* Create compressible test buffer */
     if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
-        DISPLAY("Not enough memory, aborting\n");
+        DISPLAY("Not enough memory, aborting \n");
         goto _output_error;
     }
     RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
 
+    /* Create dictionary */
+    MEM_STATIC_ASSERT(COMPRESSIBLE_NOISE_LENGTH >= 4 MB);
+    dictionary = FUZ_createDictionary(CNBuffer, 4 MB, 4 KB, 40 KB);
+    if (!dictionary.start) {
+        DISPLAY("Error creating dictionary, aborting \n");
+        goto _output_error;
+    }
+    dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
+
     /* generate skippable frame */
     MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
     MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
     cSize = skippableFrameSize + 8;
 
     /* Basic compression test */
-    DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
     ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
     outBuff.dst = (char*)(compressedBuffer)+cSize;
     outBuff.size = compressedBufferSize;
@@ -151,16 +199,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     { size_t const r = ZSTD_endStream(zc, &outBuff);
       if (r != 0) goto _output_error; }  /* error, or some data not flushed */
     cSize += outBuff.pos;
-    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+    DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
 
-    DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
     { size_t const s = ZSTD_sizeof_CStream(zc);
       if (ZSTD_isError(s)) goto _output_error;
-      DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
+      DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }
 
     /* skippable frame test */
-    DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
     ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
     inBuff.src = compressedBuffer;
     inBuff.size = cSize;
@@ -170,46 +218,46 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     outBuff.pos = 0;
     { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
       if (r != 0) goto _output_error; }
-    if (outBuff.pos != 0) goto _output_error;   /* skippable frame len is 0 */
-    DISPLAYLEVEL(4, "OK \n");
+    if (outBuff.pos != 0) goto _output_error;   /* skippable frame output len is 0 */
+    DISPLAYLEVEL(3, "OK \n");
 
     /* Basic decompression test */
     inBuff2 = inBuff;
-    DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
     ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
-    { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000);  /* large limit */
+    { size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000);  /* large limit */
       if (ZSTD_isError(r)) goto _output_error; }
     { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
       if (remaining != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
     if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
     if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
-    DISPLAYLEVEL(4, "OK \n");
+    DISPLAYLEVEL(3, "OK \n");
 
     /* Re-use without init */
-    DISPLAYLEVEL(4, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
     outBuff.pos = 0;
     { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
       if (remaining != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
     if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
     if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
-    DISPLAYLEVEL(4, "OK \n");
+    DISPLAYLEVEL(3, "OK \n");
 
     /* check regenerated data is byte exact */
-    DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
     {   size_t i;
         for (i=0; i<CNBufferSize; i++) {
             if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
     }   }
-    DISPLAYLEVEL(4, "OK \n");
+    DISPLAYLEVEL(3, "OK \n");
 
-    DISPLAYLEVEL(4, "test%3i : check DStream size : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : check DStream size : ", testNb++);
     { size_t const s = ZSTD_sizeof_DStream(zd);
       if (ZSTD_isError(s)) goto _output_error;
-      DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
+      DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }
 
     /* Byte-by-byte decompression test */
-    DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
     {   /* skippable frame */
         size_t r = 1;
         ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
@@ -235,20 +283,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }
     if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
     if (inBuff.pos != cSize) goto _output_error;   /* should have read the entire frame */
-    DISPLAYLEVEL(4, "OK \n");
+    DISPLAYLEVEL(3, "OK \n");
 
     /* check regenerated data is byte exact */
-    DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
     {   size_t i;
         for (i=0; i<CNBufferSize; i++) {
             if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
     }   }
-    DISPLAYLEVEL(4, "OK \n");
+    DISPLAYLEVEL(3, "OK \n");
 
     /* _srcSize compression test */
-    DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
     ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
-    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    outBuff.dst = (char*)(compressedBuffer);
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
     inBuff.src = CNBuffer;
@@ -259,12 +307,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
     { size_t const r = ZSTD_endStream(zc, &outBuff);
       if (r != 0) goto _output_error; }  /* error, or some data not flushed */
-    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+    { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
+      if ((size_t)origSize != CNBufferSize) goto _output_error; }  /* exact original size must be present */
+    DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
 
     /* wrong _srcSize compression test */
-    DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
+    DISPLAYLEVEL(3, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
     ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
-    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    outBuff.dst = (char*)(compressedBuffer);
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
     inBuff.src = CNBuffer;
@@ -275,10 +325,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
     { size_t const r = ZSTD_endStream(zc, &outBuff);
       if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error;    /* must fail : wrong srcSize */
-      DISPLAYLEVEL(4, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
+      DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
 
     /* Complex context re-use scenario */
-    DISPLAYLEVEL(4, "test%3i : context re-use : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
     ZSTD_freeCStream(zc);
     zc = ZSTD_createCStream_advanced(customMem);
     if (zc==NULL) goto _output_error;   /* memory allocation issue */
@@ -312,11 +362,11 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         { size_t const r = ZSTD_endStream(zc, &outBuff);
             if (r != 0) goto _output_error; }  /* error, or some data not flushed */
     }
-    DISPLAYLEVEL(4, "OK \n");
+    DISPLAYLEVEL(3, "OK \n");
 
     /* CDict scenario */
-    DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++);
-    {   ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1);
+    DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
+    {   ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1);
         size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
         if (ZSTD_isError(initError)) goto _output_error;
         cSize = 0;
@@ -333,18 +383,24 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
           if (r != 0) goto _output_error; }  /* error, or some data not flushed */
         cSize = outBuff.pos;
         ZSTD_freeCDict(cdict);
-        DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+        DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
     }
 
-    DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
     { size_t const s = ZSTD_sizeof_CStream(zc);
       if (ZSTD_isError(s)) goto _output_error;
-      DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
+      DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
+    }
+
+    DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
+    { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
+      if (dID != dictID) goto _output_error;
+      DISPLAYLEVEL(4, "OK (%u) \n", dID);
     }
 
     /* DDict scenario */
-    DISPLAYLEVEL(4, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize);
-    {   ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, 128 KB);
+    DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize);
+    {   ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
         size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
         if (ZSTD_isError(initError)) goto _output_error;
         inBuff.src = compressedBuffer;
@@ -358,19 +414,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
         if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
         ZSTD_freeDDict(ddict);
-        DISPLAYLEVEL(4, "OK \n");
+        DISPLAYLEVEL(3, "OK \n");
     }
 
     /* test ZSTD_setDStreamParameter() resilience */
-    DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
     { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1);  /* large limit */
       if (!ZSTD_isError(r)) goto _output_error; }
-    DISPLAYLEVEL(4, "OK \n");
+    DISPLAYLEVEL(3, "OK \n");
 
     /* Memory restriction */
-    DISPLAYLEVEL(4, "test%3i : maxWindowSize < frame requirement : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
     ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
-    { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000);  /* too small limit */
+    { size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000);  /* too small limit */
       if (ZSTD_isError(r)) goto _output_error; }
     inBuff.src = compressedBuffer;
     inBuff.size = cSize;
@@ -380,10 +436,73 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     outBuff.pos = 0;
     { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
       if (!ZSTD_isError(r)) goto _output_error;  /* must fail : frame requires > 100 bytes */
-      DISPLAYLEVEL(4, "OK (%s)\n", ZSTD_getErrorName(r)); }
+      DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
 
+    /* Unknown srcSize */
+    DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
+    {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
+        params.fParams.contentSizeFlag = 1;
+        ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */
+    inBuff.src = CNBuffer;
+    inBuff.size = 0;
+    inBuff.pos = 0;
+    outBuff.dst = compressedBuffer;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
+    if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
+    cSize = outBuff.pos;
+    if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
+
+    ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
+    inBuff.src = CNBuffer;
+    inBuff.size = 0;
+    inBuff.pos = 0;
+    outBuff.dst = compressedBuffer;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
+    if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
+    cSize = outBuff.pos;
+    if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
+    DISPLAYLEVEL(3, "OK \n");
+
+    /* Overlen overwriting window data bug */
+    DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
+    {   /* This test has a window size of 1024 bytes and consists of 3 blocks:
+            1. 'a' repeated 517 times
+            2. 'b' repeated 516 times
+            3. a compressed block with no literals and 3 sequence commands:
+                litlength = 0, offset = 24, match length = 24
+                litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
+                litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
+
+        const char* testCase =
+            "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
+            "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
+            "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
+        ZSTD_DStream* zds = ZSTD_createDStream();
+
+        ZSTD_initDStream(zds);
+        inBuff.src = testCase;
+        inBuff.size = 47;
+        inBuff.pos = 0;
+        outBuff.dst = decodedBuffer;
+        outBuff.size = CNBufferSize;
+        outBuff.pos = 0;
+
+        while (inBuff.pos < inBuff.size) {
+            size_t const r = ZSTD_decompressStream(zds, &outBuff, &inBuff);
+            /* Bug will cause checksum to fail */
+            if (ZSTD_isError(r)) goto _output_error;
+        }
+
+        ZSTD_freeDStream(zds);
+    }
+    DISPLAYLEVEL(3, "OK \n");
 
 _end:
+    FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);
     ZSTD_freeDStream(zd);
     free(CNBuffer);
@@ -408,6 +527,11 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
     for (u=0; u<max; u++) {
         if (b1[u] != b2[u]) break;
     }
+    DISPLAY("Error at position %u / %u \n", (U32)u, (U32)max);
+    DISPLAY(" %02X %02X %02X  :%02X:  %02X %02X %02X %02X %02X \n",
+            b1[u-3], b1[u-2], b1[u-1], b1[u-0], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
+    DISPLAY(" %02X %02X %02X  :%02X:  %02X %02X %02X %02X %02X \n",
+            b2[u-3], b2[u-2], b2[u-1], b2[u-0], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
     return u;
 }
 
@@ -488,7 +612,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
         else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
         FUZ_rand(&coreSeed);
-        lseed = coreSeed ^ prime1;
+        lseed = coreSeed ^ prime32;
 
         /* states full reset (deliberately not synchronized) */
         /* some issues can only happen when reusing states */
@@ -521,11 +645,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
             }
         } else {
             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
-            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
+            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             oldTestLog = testLog;
             /* random dictionary selection */
-            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
+            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
             }
@@ -629,10 +754,13 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
                 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
+                size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
                 outBuff.size = outBuff.pos + adjustedDstSize;
-                inBuff.size  = inBuff.pos + randomCSrcSize;
+                inBuff.size  = inBuff.pos + adjustedCSrcSize;
                 {   size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                     if (ZSTD_isError(decompressError)) break;   /* error correctly detected */
+                    /* No forward progress possible */
+                    if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
     }   }   }   }
     DISPLAY("\r%u fuzzer tests completed   \n", testNb);
 
@@ -656,6 +784,254 @@ _output_error:
 }
 
 
+/* Multi-threading version of fuzzer Tests */
+static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
+{
+    static const U32 maxSrcLog = 24;
+    static const U32 maxSampleLog = 19;
+    size_t const srcBufferSize = (size_t)1<<maxSrcLog;
+    BYTE* cNoiseBuffer[5];
+    size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
+    BYTE*  const copyBuffer = (BYTE*)malloc (copyBufferSize);
+    size_t const cBufferSize   = ZSTD_compressBound(srcBufferSize);
+    BYTE*  const cBuffer = (BYTE*)malloc (cBufferSize);
+    size_t const dstBufferSize = srcBufferSize;
+    BYTE*  const dstBuffer = (BYTE*)malloc (dstBufferSize);
+    U32 result = 0;
+    U32 testNb = 0;
+    U32 coreSeed = seed;
+    ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(2);   /* will be reset sometimes */
+    ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
+    ZSTD_DStream* const zd_noise = ZSTD_createDStream();
+    clock_t const startClock = clock();
+    const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
+    size_t dictSize = 0;
+    U32 oldTestLog = 0;
+
+    /* allocations */
+    cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
+    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
+           !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
+           "Not enough memory, fuzzer tests cancelled");
+
+    /* Create initial samples */
+    RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
+    RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
+    RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
+    RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
+    RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
+    memset(copyBuffer, 0x65, copyBufferSize);                             /* make copyBuffer considered initialized */
+    ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
+
+    /* catch up testNb */
+    for (testNb=1; testNb < startTest; testNb++)
+        FUZ_rand(&coreSeed);
+
+    /* test loop */
+    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+        U32 lseed;
+        const BYTE* srcBuffer;
+        size_t totalTestSize, totalGenSize, cSize;
+        XXH64_state_t xxhState;
+        U64 crcOrig;
+        U32 resetAllowed = 1;
+        size_t maxTestSize;
+
+        /* init */
+        if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
+        else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
+        FUZ_rand(&coreSeed);
+        lseed = coreSeed ^ prime32;
+
+        /* states full reset (deliberately not synchronized) */
+        /* some issues can only happen when reusing states */
+        if ((FUZ_rand(&lseed) & 0xFF) == 131) {
+            U32 const nbThreads = (FUZ_rand(&lseed) % 6) + 1;
+            ZSTDMT_freeCCtx(zc);
+            zc = ZSTDMT_createCCtx(nbThreads);
+            resetAllowed=0;
+        }
+        if ((FUZ_rand(&lseed) & 0xFF) == 132) {
+            ZSTD_freeDStream(zd);
+            zd = ZSTD_createDStream();
+            ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
+        }
+
+        /* srcBuffer selection [0-4] */
+        {   U32 buffNb = FUZ_rand(&lseed) & 0x7F;
+            if (buffNb & 7) buffNb=2;   /* most common : compressible (P) */
+            else {
+                buffNb >>= 3;
+                if (buffNb & 7) {
+                    const U32 tnb[2] = { 1, 3 };   /* barely/highly compressible */
+                    buffNb = tnb[buffNb >> 3];
+                } else {
+                    const U32 tnb[2] = { 0, 4 };   /* not compressible / sparse */
+                    buffNb = tnb[buffNb >> 3];
+            }   }
+            srcBuffer = cNoiseBuffer[buffNb];
+        }
+
+        /* compression init */
+        if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
+            && oldTestLog /* at least one test happened */ && resetAllowed) {
+            maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
+            if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
+            {   int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
+                size_t const resetError = ZSTDMT_initCStream(zc, compressionLevel);
+                CHECK(ZSTD_isError(resetError), "ZSTDMT_initCStream error : %s", ZSTD_getErrorName(resetError));
+            }
+        } else {
+            U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
+            maxTestSize = FUZ_rLogLength(&lseed, testLog);
+            oldTestLog = testLog;
+            /* random dictionary selection */
+            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
+            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
+                dict = srcBuffer + dictStart;
+            }
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+                ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
+                DISPLAYLEVEL(5, "Init with windowLog = %u \n", params.cParams.windowLog);
+                params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
+                params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+                { size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
+                  CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); }
+                ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12);
+                ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1));
+        }   }
+
+        /* multi-segments compression test */
+        XXH64_reset(&xxhState, 0);
+        {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
+            U32 n;
+            for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
+                /* compress random chunks into randomly sized dst buffers */
+                {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
+                    size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+                    ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
+                    outBuff.size = outBuff.pos + dstBuffSize;
+
+                    DISPLAYLEVEL(5, "Sending %u bytes to compress \n", (U32)srcSize);
+                    { size_t const compressionError = ZSTDMT_compressStream(zc, &outBuff, &inBuff);
+                      CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
+                    DISPLAYLEVEL(5, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
+
+                    XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
+                    memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
+                    totalTestSize += inBuff.pos;
+                }
+
+                /* random flush operation, to mess around */
+                if ((FUZ_rand(&lseed) & 15) == 0) {
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
+                    outBuff.size = outBuff.pos + adjustedDstSize;
+                    DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize);
+                    {   size_t const flushError = ZSTDMT_flushStream(zc, &outBuff);
+                        CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
+            }   }   }
+
+            /* final frame epilogue */
+            {   size_t remainingToFlush = (size_t)(-1);
+                while (remainingToFlush) {
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
+                    outBuff.size = outBuff.pos + adjustedDstSize;
+                    DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (U32)adjustedDstSize);
+                    remainingToFlush = ZSTDMT_endStream(zc, &outBuff);
+                    CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
+                    DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (U32)remainingToFlush);
+            }   }
+            DISPLAYLEVEL(5, "Frame completed \n");
+            crcOrig = XXH64_digest(&xxhState);
+            cSize = outBuff.pos;
+        }
+
+        /* multi - fragments decompression test */
+        if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
+            CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+        } else {
+            ZSTD_initDStream_usingDict(zd, dict, dictSize);
+        }
+        {   size_t decompressionResult = 1;
+            ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
+            ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
+            for (totalGenSize = 0 ; decompressionResult ; ) {
+                size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
+                inBuff.size = inBuff.pos + readCSrcSize;
+                outBuff.size = inBuff.pos + dstBuffSize;
+                decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+                CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
+            }
+            CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
+            CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
+            {   U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
+                if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
+                CHECK (crcDest!=crcOrig, "decompressed data corrupted");
+        }   }
+
+        /*=====   noisy/erroneous src decompression test   =====*/
+
+        /* add some noise */
+        {   U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
+            U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
+                size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const noiseSize  = MIN((cSize/3) , randomNoiseSize);
+                size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
+                size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
+                memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
+        }   }
+
+        /* try decompression on noisy data */
+        ZSTD_initDStream(zd_noise);   /* note : no dictionary */
+        {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
+            ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
+            while (outBuff.pos < dstBufferSize) {
+                size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
+                size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
+                outBuff.size = outBuff.pos + adjustedDstSize;
+                inBuff.size  = inBuff.pos + adjustedCSrcSize;
+                {   size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+                    if (ZSTD_isError(decompressError)) break;   /* error correctly detected */
+                    /* No forward progress possible */
+                    if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
+    }   }   }   }
+    DISPLAY("\r%u fuzzer tests completed   \n", testNb);
+
+_cleanup:
+    ZSTDMT_freeCCtx(zc);
+    ZSTD_freeDStream(zd);
+    ZSTD_freeDStream(zd_noise);
+    free(cNoiseBuffer[0]);
+    free(cNoiseBuffer[1]);
+    free(cNoiseBuffer[2]);
+    free(cNoiseBuffer[3]);
+    free(cNoiseBuffer[4]);
+    free(copyBuffer);
+    free(cBuffer);
+    free(dstBuffer);
+    return result;
+
+_output_error:
+    result = 1;
+    goto _cleanup;
+}
+
+
 /*-*******************************************************
 *  Command line
 *********************************************************/
@@ -685,10 +1061,11 @@ int main(int argc, const char** argv)
     int testNb = 0;
     int proba = FUZ_COMPRESSIBILITY_DEFAULT;
     int result=0;
-    U32 mainPause = 0;
-    const char* programName = argv[0];
-    ZSTD_customMem customMem = { allocFunction, freeFunction, NULL };
-    ZSTD_customMem customNULL = { NULL, NULL, NULL };
+    int mainPause = 0;
+    int mtOnly = 0;
+    const char* const programName = argv[0];
+    ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
+    ZSTD_customMem const customNULL = { NULL, NULL, NULL };
 
     /* Check command line */
     for(argNb=1; argNb<argc; argNb++) {
@@ -697,27 +1074,32 @@ int main(int argc, const char** argv)
 
         /* Parsing commands. Aggregated commands are allowed */
         if (argument[0]=='-') {
-            argument++;
 
+            if (!strcmp(argument, "--mt")) { mtOnly=1; continue; }
+
+            argument++;
             while (*argument!=0) {
                 switch(*argument)
                 {
                 case 'h':
                     return FUZ_usage(programName);
+
                 case 'v':
                     argument++;
-                    g_displayLevel=4;
+                    g_displayLevel++;
                     break;
+
                 case 'q':
                     argument++;
                     g_displayLevel--;
                     break;
+
                 case 'p': /* pause at the end */
                     argument++;
                     mainPause = 1;
                     break;
 
-                case 'i':
+                case 'i':   /* limit tests by nb of iterations (default) */
                     argument++;
                     nbTests=0; g_clockTime=0;
                     while ((*argument>='0') && (*argument<='9')) {
@@ -727,7 +1109,7 @@ int main(int argc, const char** argv)
                     }
                     break;
 
-                case 'T':
+                case 'T':   /* limit tests by time */
                     argument++;
                     nbTests=0; g_clockTime=0;
                     while ((*argument>='0') && (*argument<='9')) {
@@ -740,7 +1122,7 @@ int main(int argc, const char** argv)
                     g_clockTime *= CLOCKS_PER_SEC;
                     break;
 
-                case 's':
+                case 's':   /* manually select seed */
                     argument++;
                     seed=0;
                     seedset=1;
@@ -751,7 +1133,7 @@ int main(int argc, const char** argv)
                     }
                     break;
 
-                case 't':
+                case 't':   /* select starting test number */
                     argument++;
                     testNb=0;
                     while ((*argument>='0') && (*argument<='9')) {
@@ -795,12 +1177,12 @@ int main(int argc, const char** argv)
     if (testNb==0) {
         result = basicUnitTests(0, ((double)proba) / 100, customNULL);  /* constant seed for predictability */
         if (!result) {
-            DISPLAYLEVEL(4, "Unit tests using customMem :\n")
+            DISPLAYLEVEL(3, "Unit tests using customMem :\n")
             result = basicUnitTests(0, ((double)proba) / 100, customMem);  /* use custom memory allocation functions */
     }   }
 
-    if (!result)
-        result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
+    if (!result && !mtOnly) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
+    if (!result) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100);
 
     if (mainPause) {
         int unused;

zlibWrapper/.gitignore

@@ -20,3 +20,6 @@ zwrapbench
 *.bat
 *.zip
 *.txt
+
+# Directories
+minizip/

zlibWrapper/BUCK

@@ -0,0 +1,22 @@
+cxx_library(
+    name='zlib_wrapper',
+    visibility=['PUBLIC'],
+    exported_linker_flags=['-lz'],
+    header_namespace='',
+    exported_headers=['zstd_zlibwrapper.h'],
+    headers=[
+        'gzcompatibility.h',
+        'gzguts.h',
+    ],
+    srcs=glob(['*.c']),
+    deps=[
+        '//lib:zstd',
+        '//lib:zstd_common',
+    ]
+)
+
+cxx_binary(
+    name='minigzip',
+    srcs=['examples/minigzip.c'],
+    deps=[':zlib_wrapper'],
+)

zlibWrapper/examples/fitblk.c

@@ -82,7 +82,7 @@ local int partcompress(FILE *in, z_streamp def)
 
     flush = Z_SYNC_FLUSH;
     do {
-        def->avail_in = fread(raw, 1, RAWLEN, in);
+        def->avail_in = (uInt)fread(raw, 1, RAWLEN, in);
         if (ferror(in))
             return Z_ERRNO;
         def->next_in = raw;
@@ -148,7 +148,7 @@ int main(int argc, char **argv)
     /* get requested output size */
     if (argc != 2)
         quit("need one argument: size of output block");
-    ret = strtol(argv[1], argv + 1, 10);
+    ret = (int)strtol(argv[1], argv + 1, 10);
     if (argv[1][0] != 0)
         quit("argument must be a number");
     if (ret < 8)            /* 8 is minimum zlib stream size */

zlibWrapper/examples/zwrapbench.c

@@ -160,7 +160,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
     ZSTD_CCtx* const ctx = ZSTD_createCCtx();
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
     U32 nbBlocks;
-    UTIL_time_t ticksPerSecond;
+    UTIL_freq_t ticksPerSecond;
 
     /* checks */
     if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
@@ -234,7 +234,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                 if (compressor == BMK_ZSTD) {
                     ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
                     ZSTD_customMem const cmem = { NULL, NULL, NULL };
-                    ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
+                    ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
                     if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
 
                     do {
@@ -315,10 +315,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                                 if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
                             }
                             def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
-                            def.avail_in = blockTable[blockNb].srcSize;
+                            def.avail_in = (uInt)blockTable[blockNb].srcSize;
                             def.total_in = 0;
                             def.next_out = (void*) blockTable[blockNb].cPtr;
-                            def.avail_out = blockTable[blockNb].cRoom;
+                            def.avail_out = (uInt)blockTable[blockNb].cRoom;
                             def.total_out = 0;
                             ret = deflate(&def, Z_FINISH);
                             if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure ret=%d srcSize=%d" , ret, (int)blockTable[blockNb].srcSize);
@@ -346,10 +346,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                                 if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
                             }
                             def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
-                            def.avail_in = blockTable[blockNb].srcSize;
+                            def.avail_in = (uInt)blockTable[blockNb].srcSize;
                             def.total_in = 0;
                             def.next_out = (void*) blockTable[blockNb].cPtr;
-                            def.avail_out = blockTable[blockNb].cRoom;
+                            def.avail_out = (uInt)blockTable[blockNb].cRoom;
                             def.total_out = 0;
                             ret = deflate(&def, Z_FINISH);
                             if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure");
@@ -451,10 +451,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                                 ret = inflateReset(&inf);
                             if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
                             inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
-                            inf.avail_in = blockTable[blockNb].cSize;
+                            inf.avail_in = (uInt)blockTable[blockNb].cSize;
                             inf.total_in = 0;
                             inf.next_out = (void*) blockTable[blockNb].resPtr;
-                            inf.avail_out = blockTable[blockNb].srcSize;
+                            inf.avail_out = (uInt)blockTable[blockNb].srcSize;
                             inf.total_out = 0;
                             ret = inflate(&inf, Z_FINISH);
                             if (ret == Z_NEED_DICT) {
@@ -483,10 +483,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                             ret = inflateInit(&inf);
                             if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
                             inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
-                            inf.avail_in = blockTable[blockNb].cSize;
+                            inf.avail_in = (uInt)blockTable[blockNb].cSize;
                             inf.total_in = 0;
                             inf.next_out = (void*) blockTable[blockNb].resPtr;
-                            inf.avail_out = blockTable[blockNb].srcSize;
+                            inf.avail_out = (uInt)blockTable[blockNb].srcSize;
                             inf.total_out = 0;
                             ret = inflate(&inf, Z_FINISH);
                             if (ret == Z_NEED_DICT) {
@@ -591,7 +591,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     if (!pch) pch = strrchr(displayName, '/'); /* Linux */
     if (pch) displayName = pch+1;
 
-    SET_HIGH_PRIORITY;
+    SET_REALTIME_PRIORITY;
 
     if (g_displayLevel == 1 && !g_additionalParam)
         DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));

zlibWrapper/gzcompatibility.h

@@ -43,3 +43,25 @@ ZEXTERN gzFile         ZEXPORT gzopen_w OF((const wchar_t *path,
                                             const char *mode));
 #endif
 #endif
+
+
+#if ZLIB_VERNUM < 0x12B0
+#ifdef Z_SOLO
+   typedef unsigned long z_size_t;
+#else
+#  define z_longlong long long
+#  if defined(NO_SIZE_T)
+     typedef unsigned NO_SIZE_T z_size_t;
+#  elif defined(STDC)
+#    include <stddef.h>
+     typedef size_t z_size_t;
+#  else
+     typedef unsigned long z_size_t;
+#  endif
+#  undef z_longlong
+#endif
+ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems,
+                                     gzFile file));
+ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size,
+                                      z_size_t nitems, gzFile file));
+#endif

zlibWrapper/gzguts.h

@@ -3,7 +3,7 @@
  * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */
 
 /* gzguts.h -- zlib internal header definitions for gz* operations
- * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
+ * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
  * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
@@ -30,6 +30,10 @@
 #  include <stdlib.h>
 #  include <limits.h>
 #endif
+
+#ifndef _POSIX_SOURCE
+#  define _POSIX_SOURCE
+#endif
 #include <fcntl.h>
 
 #ifdef _WIN32
@@ -40,6 +44,10 @@
 #  include <io.h>
 #endif
 
+#if defined(_WIN32) || defined(__CYGWIN__)
+#  define WIDECHAR
+#endif
+
 #ifdef WINAPI_FAMILY
 #  define open _open
 #  define read _read
@@ -100,18 +108,19 @@
 #  endif
 #endif
 
-/* unlike snprintf (which is required in C99, yet still not supported by
-   Microsoft more than a decade later!), _snprintf does not guarantee null
-   termination of the result -- however this is only used in gzlib.c where
+/* unlike snprintf (which is required in C99), _snprintf does not guarantee
+   null termination of the result -- however this is only used in gzlib.c where
    the result is assured to fit in the space provided */
-#ifdef _MSC_VER
+#if defined(_MSC_VER) && _MSC_VER < 1900
 #  define snprintf _snprintf
 #endif
 
 #ifndef local
 #  define local static
 #endif
-/* compile with -Dlocal if your debugger can't find static symbols */
+/* since "static" is used to mean two completely different things in C, we
+   define "local" for the non-static meaning of "static", for readability
+   (compile with -Dlocal if your debugger can't find static symbols) */
 
 /* gz* functions always use library allocation functions */
 #ifndef STDC
@@ -175,7 +184,7 @@ typedef struct {
     char *path;             /* path or fd for error messages */
     unsigned size;          /* buffer size, zero if not allocated yet */
     unsigned want;          /* requested buffer size, default is GZBUFSIZE */
-    unsigned char *in;      /* input buffer */
+    unsigned char *in;      /* input buffer (double-sized when writing) */
     unsigned char *out;     /* output buffer (double-sized when reading) */
     int direct;             /* 0 if processing gzip, 1 if transparent */
         /* just for reading */

zlibWrapper/gzlib.c

@@ -1,14 +1,14 @@
 /* gzlib.c contains minimal changes required to be compiled with zlibWrapper:
- * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */
+ * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */ 
 
 /* gzlib.c -- zlib functions common to reading and writing gzip files
- * Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler
+ * Copyright (C) 2004-2017 Mark Adler
  * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
 
-#if defined(_WIN32) && !defined(__BORLANDC__)
+#if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__)
 #  define LSEEK _lseeki64
 #else
 #if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0
@@ -97,7 +97,7 @@ local gzFile gz_open(path, fd, mode)
     const char *mode;
 {
     gz_statep state;
-    size_t len;
+    z_size_t len;
     int oflag;
 #ifdef O_CLOEXEC
     int cloexec = 0;
@@ -191,10 +191,10 @@ local gzFile gz_open(path, fd, mode)
     }
 
     /* save the path name for error messages */
-#ifdef _WIN32
+#ifdef WIDECHAR
     if (fd == -2) {
         len = wcstombs(NULL, path, 0);
-        if (len == (size_t)-1)
+        if (len == (z_size_t)-1)
             len = 0;
     }
     else
@@ -205,7 +205,7 @@ local gzFile gz_open(path, fd, mode)
         free(state.state);
         return NULL;
     }
-#ifdef _WIN32
+#ifdef WIDECHAR
     if (fd == -2)
         if (len)
             wcstombs(state.state->path, path, len + 1);
@@ -214,7 +214,7 @@ local gzFile gz_open(path, fd, mode)
     else
 #endif
 #if !defined(NO_snprintf) && !defined(NO_vsnprintf)
-        snprintf(state.state->path, len + 1, "%s", (const char *)path);
+        (void)snprintf(state.state->path, len + 1, "%s", (const char *)path);
 #else
         strcpy(state.state->path, path);
 #endif
@@ -242,7 +242,7 @@ local gzFile gz_open(path, fd, mode)
 
     /* open the file with the appropriate flags (or just use fd) */
     state.state->fd = fd > -1 ? fd : (
-#ifdef _WIN32
+#ifdef WIDECHAR
         fd == -2 ? _wopen(path, oflag, 0666) :
 #endif
         open((const char *)path, oflag, 0666));
@@ -251,8 +251,10 @@ local gzFile gz_open(path, fd, mode)
         free(state.state);
         return NULL;
     }
-    if (state.state->mode == GZ_APPEND)
+    if (state.state->mode == GZ_APPEND) {
+        LSEEK(state.state->fd, 0, SEEK_END);  /* so gzoffset() is correct */
         state.state->mode = GZ_WRITE;         /* simplify later checks */
+    }
 
     /* save the current position for rewinding (only if reading) */
     if (state.state->mode == GZ_READ) {
@@ -294,7 +296,7 @@ gzFile ZEXPORT gzdopen(fd, mode)
     if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL)
         return NULL;
 #if !defined(NO_snprintf) && !defined(NO_vsnprintf)
-    snprintf(path, 7 + 3 * sizeof(int), "<fd:%d>", fd); /* for debugging */
+    (void)snprintf(path, 7 + 3 * sizeof(int), "<fd:%d>", fd);
 #else
     sprintf(path, "<fd:%d>", fd);   /* for debugging */
 #endif
@@ -304,7 +306,7 @@ gzFile ZEXPORT gzdopen(fd, mode)
 }
 
 /* -- see zlib.h -- */
-#ifdef _WIN32
+#ifdef WIDECHAR
 gzFile ZEXPORT gzopen_w(path, mode)
     const wchar_t *path;
     const char *mode;
@@ -332,6 +334,8 @@ int ZEXPORT gzbuffer(file, size)
         return -1;
 
     /* check and set requested size */
+    if ((size << 1) < size)
+        return -1;              /* need to be able to double it */
     if (size < 2)
         size = 2;               /* need two bytes to check magic header */
     state.state->want = size;
@@ -569,8 +573,8 @@ void ZEXPORT gzclearerr(file)
     gz_error(state, Z_OK, NULL);
 }
 
-/* Create an error message in allocated memory and set state->err and
-   state->msg accordingly.  Free any previous error message already there.  Do
+/* Create an error message in allocated memory and set state.state->err and
+   state.state->msg accordingly.  Free any previous error message already there.  Do
    not try to free or allocate space if the error is Z_MEM_ERROR (out of
    memory).  Simply save the error message as a static string.  If there is an
    allocation failure constructing the error message, then convert the error to
@@ -587,7 +591,7 @@ void ZLIB_INTERNAL gz_error(state, err, msg)
         state.state->msg = NULL;
     }
 
-    /* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */
+    /* if fatal, set state.state->x.have to 0 so that the gzgetc() macro fails */
     if (err != Z_OK && err != Z_BUF_ERROR)
         state.state->x.have = 0;
 
@@ -607,14 +611,13 @@ void ZLIB_INTERNAL gz_error(state, err, msg)
         return;
     }
 #if !defined(NO_snprintf) && !defined(NO_vsnprintf)
-    snprintf(state.state->msg, strlen(state.state->path) + strlen(msg) + 3,
-             "%s%s%s", state.state->path, ": ", msg);
+    (void)snprintf(state.state->msg, strlen(state.state->path) + strlen(msg) + 3,
+                   "%s%s%s", state.state->path, ": ", msg);
 #else
     strcpy(state.state->msg, state.state->path);
     strcat(state.state->msg, ": ");
     strcat(state.state->msg, msg);
 #endif
-    return;
 }
 
 #ifndef INT_MAX

zlibWrapper/gzread.c

@@ -1,8 +1,8 @@
 /* gzread.c contains minimal changes required to be compiled with zlibWrapper:
- * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */
-
-/* gzread.c -- zlib functions for reading gzip files
- * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
+ * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */ 
+ 
+ /* gzread.c -- zlib functions for reading gzip files
+ * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
  * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
@@ -15,9 +15,10 @@ local int gz_look OF((gz_statep));
 local int gz_decomp OF((gz_statep));
 local int gz_fetch OF((gz_statep));
 local int gz_skip OF((gz_statep, z_off64_t));
+local z_size_t gz_read OF((gz_statep, voidp, z_size_t));
 
 /* Use read() to load a buffer -- return -1 on error, otherwise 0.  Read from
-   state->fd, and update state->eof, state->err, and state->msg as appropriate.
+   state.state->fd, and update state.state->eof, state.state->err, and state.state->msg as appropriate.
    This function needs to loop on read(), since read() is not guaranteed to
    read the number of bytes requested, depending on the type of descriptor. */
 local int gz_load(state, buf, len, have)
@@ -26,14 +27,18 @@ local int gz_load(state, buf, len, have)
     unsigned len;
     unsigned *have;
 {
-    int ret;
+    ssize_t ret;
+    unsigned get, max = ((unsigned)-1 >> 2) + 1;
 
     *have = 0;
     do {
-        ret = read(state.state->fd, buf + *have, len - *have);
+        get = len - *have;
+        if (get > max)
+            get = max;
+        ret = read(state.state->fd, buf + *have, get);
         if (ret <= 0)
             break;
-        *have += ret;
+        *have += (unsigned)ret;
     } while (*have < len);
     if (ret < 0) {
         gz_error(state, Z_ERRNO, zstrerror());
@@ -77,8 +82,8 @@ local int gz_avail(state)
     return 0;
 }
 
-/* Look for gzip header, set up for inflate or copy.  state->x.have must be 0.
-   If this is the first time in, allocate required memory.  state->how will be
+/* Look for gzip header, set up for inflate or copy.  state.state->x.have must be 0.
+   If this is the first time in, allocate required memory.  state.state->how will be
    left unchanged if there is no more input data available, will be set to COPY
    if there is no gzip header and direct copying will be performed, or it will
    be set to GZIP for decompression.  If direct copying, then leftover input
@@ -97,10 +102,8 @@ local int gz_look(state)
         state.state->in = (unsigned char *)malloc(state.state->want);
         state.state->out = (unsigned char *)malloc(state.state->want << 1);
         if (state.state->in == NULL || state.state->out == NULL) {
-            if (state.state->out != NULL)
-                free(state.state->out);
-            if (state.state->in != NULL)
-                free(state.state->in);
+            free(state.state->out);
+            free(state.state->in);
             gz_error(state, Z_MEM_ERROR, "out of memory");
             return -1;
         }
@@ -136,7 +139,6 @@ local int gz_look(state)
        file -- for here we assume that if a gzip file is being written, then
        the header will be written in a single operation, so that reading a
        single byte is sufficient indication that it is not a gzip file) */
-    //printf("strm->next_in[0]=%d strm->next_in[1]=%d\n", strm->next_in[0], strm->next_in[1]);
     if (strm->avail_in > 1 &&
             ((strm->next_in[0] == 31 && strm->next_in[1] == 139) /* gz header */
             || (strm->next_in[0] == 40 && strm->next_in[1] == 181))) { /* zstd header */
@@ -170,9 +172,9 @@ local int gz_look(state)
 }
 
 /* Decompress from input to the provided next_out and avail_out in the state.
-   On return, state->x.have and state->x.next point to the just decompressed
-   data.  If the gzip stream completes, state->how is reset to LOOK to look for
-   the next gzip stream or raw data, once state->x.have is depleted.  Returns 0
+   On return, state.state->x.have and state.state->x.next point to the just decompressed
+   data.  If the gzip stream completes, state.state->how is reset to LOOK to look for
+   the next gzip stream or raw data, once state.state->x.have is depleted.  Returns 0
    on success, -1 on failure. */
 local int gz_decomp(state)
     gz_statep state;
@@ -222,11 +224,11 @@ local int gz_decomp(state)
     return 0;
 }
 
-/* Fetch data and put it in the output buffer.  Assumes state->x.have is 0.
+/* Fetch data and put it in the output buffer.  Assumes state.state->x.have is 0.
    Data is either copied from the input file or decompressed from the input
-   file depending on state->how.  If state->how is LOOK, then a gzip header is
+   file depending on state.state->how.  If state.state->how is LOOK, then a gzip header is
    looked for to determine whether to copy or decompress.  Returns -1 on error,
-   otherwise 0.  gz_fetch() will leave state->how as COPY or GZIP unless the
+   otherwise 0.  gz_fetch() will leave state.state->how as COPY or GZIP unless the
    end of the input file has been reached and all data has been processed.  */
 local int gz_fetch(state)
     gz_statep state;
@@ -289,33 +291,17 @@ local int gz_skip(state, len)
     return 0;
 }
 
-/* -- see zlib.h -- */
-int ZEXPORT gzread(file, buf, len)
-    gzFile file;
-    voidp buf;
-    unsigned len;
-{
-    unsigned got, n;
+/* Read len bytes into buf from file, or less than len up to the end of the
+   input.  Return the number of bytes read.  If zero is returned, either the
+   end of file was reached, or there was an error.  state.state->err must be
+   consulted in that case to determine which. */
+local z_size_t gz_read(state, buf, len)
     gz_statep state;
-    z_streamp strm;
-
-    /* get internal structure */
-    if (file == NULL)
-        return -1;
-    state = (gz_statep)file;
-    strm = &(state.state->strm);
-
-    /* check that we're reading and that there's no (serious) error */
-    if (state.state->mode != GZ_READ ||
-            (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR))
-        return -1;
-
-    /* since an int is returned, make sure len fits in one, otherwise return
-       with an error (this avoids the flaw in the interface) */
-    if ((int)len < 0) {
-        gz_error(state, Z_DATA_ERROR, "requested length does not fit in int");
-        return -1;
-    }
+    voidp buf;
+    z_size_t len;
+{
+    z_size_t got;
+    unsigned n;
 
     /* if len is zero, avoid unnecessary operations */
     if (len == 0)
@@ -325,32 +311,38 @@ int ZEXPORT gzread(file, buf, len)
     if (state.state->seek) {
         state.state->seek = 0;
         if (gz_skip(state, state.state->skip) == -1)
-            return -1;
+            return 0;
     }
 
     /* get len bytes to buf, or less than len if at the end */
     got = 0;
     do {
+        /* set n to the maximum amount of len that fits in an unsigned int */
+        n = -1;
+        if (n > len)
+            n = (unsigned)len;
+
         /* first just try copying data from the output buffer */
         if (state.state->x.have) {
-            n = state.state->x.have > len ? len : state.state->x.have;
+            if (state.state->x.have < n)
+                n = state.state->x.have;
             memcpy(buf, state.state->x.next, n);
             state.state->x.next += n;
             state.state->x.have -= n;
         }
 
         /* output buffer empty -- return if we're at the end of the input */
-        else if (state.state->eof && strm->avail_in == 0) {
+        else if (state.state->eof && state.state->strm.avail_in == 0) {
             state.state->past = 1;        /* tried to read past end */
             break;
         }
 
         /* need output data -- for small len or new stream load up our output
            buffer */
-        else if (state.state->how == LOOK || len < (state.state->size << 1)) {
+        else if (state.state->how == LOOK || n < (state.state->size << 1)) {
             /* get more output, looking for header if required */
             if (gz_fetch(state) == -1)
-                return -1;
+                return 0;
             continue;       /* no progress yet -- go back to copy above */
             /* the copy above assures that we will leave with space in the
                output buffer, allowing at least one gzungetc() to succeed */
@@ -358,16 +350,16 @@ int ZEXPORT gzread(file, buf, len)
 
         /* large len -- read directly into user buffer */
         else if (state.state->how == COPY) {      /* read directly */
-            if (gz_load(state, (unsigned char *)buf, len, &n) == -1)
-                return -1;
+            if (gz_load(state, (unsigned char *)buf, n, &n) == -1)
+                return 0;
         }
 
         /* large len -- decompress directly into user buffer */
-        else {  /* state->how == GZIP */
-            strm->avail_out = len;
-            strm->next_out = (unsigned char *)buf;
+        else {  /* state.state->how == GZIP */
+            state.state->strm.avail_out = n;
+            state.state->strm.next_out = (unsigned char *)buf;
             if (gz_decomp(state) == -1)
-                return -1;
+                return 0;
             n = state.state->x.have;
             state.state->x.have = 0;
         }
@@ -379,8 +371,75 @@ int ZEXPORT gzread(file, buf, len)
         state.state->x.pos += n;
     } while (len);
 
-    /* return number of bytes read into user buffer (will fit in int) */
-    return (int)got;
+    /* return number of bytes read into user buffer */
+    return got;
+}
+
+/* -- see zlib.h -- */
+int ZEXPORT gzread(file, buf, len)
+    gzFile file;
+    voidp buf;
+    unsigned len;
+{
+    gz_statep state;
+
+    /* get internal structure */
+    if (file == NULL)
+        return -1;
+    state = (gz_statep)file;
+
+    /* check that we're reading and that there's no (serious) error */
+    if (state.state->mode != GZ_READ ||
+            (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR))
+        return -1;
+
+    /* since an int is returned, make sure len fits in one, otherwise return
+       with an error (this avoids a flaw in the interface) */
+    if ((int)len < 0) {
+        gz_error(state, Z_STREAM_ERROR, "request does not fit in an int");
+        return -1;
+    }
+
+    /* read len or fewer bytes to buf */
+    len = (unsigned)gz_read(state, buf, len);
+
+    /* check for an error */
+    if (len == 0 && state.state->err != Z_OK && state.state->err != Z_BUF_ERROR)
+        return -1;
+
+    /* return the number of bytes read (this is assured to fit in an int) */
+    return (int)len;
+}
+
+/* -- see zlib.h -- */
+z_size_t ZEXPORT gzfread(buf, size, nitems, file)
+    voidp buf;
+    z_size_t size;
+    z_size_t nitems;
+    gzFile file;
+{
+    z_size_t len;
+    gz_statep state;
+
+    /* get internal structure */
+    if (file == NULL)
+        return 0;
+    state = (gz_statep)file;
+
+    /* check that we're reading and that there's no (serious) error */
+    if (state.state->mode != GZ_READ ||
+            (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR))
+        return 0;
+
+    /* compute bytes to read -- error on overflow */
+    len = nitems * size;
+    if (size && len / size != nitems) {
+        gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t");
+        return 0;
+    }
+
+    /* read len or fewer bytes to buf, return the number of full items read */
+    return len ? gz_read(state, buf, len) / size : 0;
 }
 
 /* -- see zlib.h -- */
@@ -401,7 +460,6 @@ ZEXTERN int ZEXPORT gzgetc OF((gzFile file));
 ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));
 #endif
 
-
 int ZEXPORT gzgetc(file)
     gzFile file;
 {
@@ -426,8 +484,8 @@ int ZEXPORT gzgetc(file)
         return *(state.state->x.next)++;
     }
 
-    /* nothing there -- try gzread() */
-    ret = gzread(file, buf, 1);
+    /* nothing there -- try gz_read() */
+    ret = (unsigned)gz_read(state, buf, 1);
     return ret < 1 ? -1 : buf[0];
 }
 
@@ -469,7 +527,7 @@ int ZEXPORT gzungetc(c, file)
     if (state.state->x.have == 0) {
         state.state->x.have = 1;
         state.state->x.next = state.state->out + (state.state->size << 1) - 1;
-        state.state->x.next[0] = c;
+        state.state->x.next[0] = (unsigned char)c;
         state.state->x.pos--;
         state.state->past = 0;
         return c;
@@ -491,7 +549,7 @@ int ZEXPORT gzungetc(c, file)
     }
     state.state->x.have++;
     state.state->x.next--;
-    state.state->x.next[0] = c;
+    state.state->x.next[0] = (unsigned char)c;
     state.state->x.pos--;
     state.state->past = 0;
     return c;

zlibWrapper/gzwrite.c

@@ -1,8 +1,8 @@
 /* gzwrite.c contains minimal changes required to be compiled with zlibWrapper:
- * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */
-
-/* gzwrite.c -- zlib functions for writing gzip files
- * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
+ * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */ 
+ 
+ /* gzwrite.c -- zlib functions for writing gzip files
+ * Copyright (C) 2004-2017 Mark Adler
  * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
@@ -12,17 +12,19 @@
 local int gz_init OF((gz_statep));
 local int gz_comp OF((gz_statep, int));
 local int gz_zero OF((gz_statep, z_off64_t));
+local z_size_t gz_write OF((gz_statep, voidpc, z_size_t));
 
 /* Initialize state for writing a gzip file.  Mark initialization by setting
-   state->size to non-zero.  Return -1 on failure or 0 on success. */
+   state.state->size to non-zero.  Return -1 on a memory allocation failure, or 0 on
+   success. */
 local int gz_init(state)
     gz_statep state;
 {
     int ret;
     z_streamp strm = &(state.state->strm);
 
-    /* allocate input buffer */
-    state.state->in = (unsigned char *)malloc(state.state->want);
+    /* allocate input buffer (double size for gzprintf) */
+    state.state->in = (unsigned char *)malloc(state.state->want << 1);
     if (state.state->in == NULL) {
         gz_error(state, Z_MEM_ERROR, "out of memory");
         return -1;
@@ -50,6 +52,7 @@ local int gz_init(state)
             gz_error(state, Z_MEM_ERROR, "out of memory");
             return -1;
         }
+        strm->next_in = NULL;
     }
 
     /* mark state as initialized */
@@ -65,17 +68,17 @@ local int gz_init(state)
 }
 
 /* Compress whatever is at avail_in and next_in and write to the output file.
-   Return -1 if there is an error writing to the output file, otherwise 0.
-   flush is assumed to be a valid deflate() flush value.  If flush is Z_FINISH,
-   then the deflate() state is reset to start a new gzip stream.  If gz->direct
-   is true, then simply write to the output file without compressing, and
-   ignore flush. */
+   Return -1 if there is an error writing to the output file or if gz_init()
+   fails to allocate memory, otherwise 0.  flush is assumed to be a valid
+   deflate() flush value.  If flush is Z_FINISH, then the deflate() state is
+   reset to start a new gzip stream.  If gz->direct is true, then simply write
+   to the output file without compressing, and ignore flush. */
 local int gz_comp(state, flush)
     gz_statep state;
     int flush;
 {
-    int ret, got;
-    unsigned have;
+    int ret, writ;
+    unsigned have, put, max = ((unsigned)-1 >> 2) + 1;
     z_streamp strm = &(state.state->strm);
 
     /* allocate memory if this is the first time through */
@@ -84,12 +87,16 @@ local int gz_comp(state, flush)
 
     /* write directly if requested */
     if (state.state->direct) {
-        got = write(state.state->fd, strm->next_in, strm->avail_in);
-        if (got < 0 || (unsigned)got != strm->avail_in) {
-            gz_error(state, Z_ERRNO, zstrerror());
-            return -1;
+        while (strm->avail_in) {
+            put = strm->avail_in > max ? max : strm->avail_in;
+            writ = (int)write(state.state->fd, strm->next_in, put);
+            if (writ < 0) {
+                gz_error(state, Z_ERRNO, zstrerror());
+                return -1;
+            }
+            strm->avail_in -= (unsigned)writ;
+            strm->next_in += writ;
         }
-        strm->avail_in = 0;
         return 0;
     }
 
@@ -100,17 +107,21 @@ local int gz_comp(state, flush)
            doing Z_FINISH then don't write until we get to Z_STREAM_END */
         if (strm->avail_out == 0 || (flush != Z_NO_FLUSH &&
             (flush != Z_FINISH || ret == Z_STREAM_END))) {
-            have = (unsigned)(strm->next_out - state.state->x.next);
-            if (have && ((got = write(state.state->fd, state.state->x.next, have)) < 0 ||
-                         (unsigned)got != have)) {
-                gz_error(state, Z_ERRNO, zstrerror());
-                return -1;
+            while (strm->next_out > state.state->x.next) {
+                put = strm->next_out - state.state->x.next > (int)max ? max :
+                      (unsigned)(strm->next_out - state.state->x.next);
+                writ = (int)write(state.state->fd, state.state->x.next, put);
+                if (writ < 0) {
+                    gz_error(state, Z_ERRNO, zstrerror());
+                    return -1;
+                }
+                state.state->x.next += writ;
             }
             if (strm->avail_out == 0) {
                 strm->avail_out = state.state->size;
                 strm->next_out = state.state->out;
+                state.state->x.next = state.state->out;
             }
-            state.state->x.next = strm->next_out;
         }
 
         /* compress */
@@ -132,7 +143,8 @@ local int gz_comp(state, flush)
     return 0;
 }
 
-/* Compress len zeros to output.  Return -1 on error, 0 on success. */
+/* Compress len zeros to output.  Return -1 on a write error or memory
+   allocation failure by gz_comp(), or 0 on success. */
 local int gz_zero(state, len)
     gz_statep state;
     z_off64_t len;
@@ -164,32 +176,14 @@ local int gz_zero(state, len)
     return 0;
 }
 
-/* -- see zlib.h -- */
-int ZEXPORT gzwrite(file, buf, len)
-    gzFile file;
-    voidpc buf;
-    unsigned len;
-{
-    unsigned put = len;
+/* Write len bytes from buf to file.  Return the number of bytes written.  If
+   the returned value is less than len, then there was an error. */
+local z_size_t gz_write(state, buf, len)
     gz_statep state;
-    z_streamp strm;
-
-    /* get internal structure */
-    if (file == NULL)
-        return 0;
-    state = (gz_statep)file;
-    strm = &(state.state->strm);
-
-    /* check that we're writing and that there's no error */
-    if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
-        return 0;
-
-    /* since an int is returned, make sure len fits in one, otherwise return
-       with an error (this avoids the flaw in the interface) */
-    if ((int)len < 0) {
-        gz_error(state, Z_DATA_ERROR, "requested length does not fit in int");
-        return 0;
-    }
+    voidpc buf;
+    z_size_t len;
+{
+    z_size_t put = len;
 
     /* if len is zero, avoid unnecessary operations */
     if (len == 0)
@@ -210,16 +204,17 @@ int ZEXPORT gzwrite(file, buf, len)
     if (len < state.state->size) {
         /* copy to input buffer, compress when full */
         do {
-            unsigned have, copy;
+            z_size_t have, copy;
 
-            if (strm->avail_in == 0)
-                strm->next_in = state.state->in;
-            have = (unsigned)((strm->next_in + strm->avail_in) - state.state->in);
+            if (state.state->strm.avail_in == 0)
+                state.state->strm.next_in = state.state->in;
+            have = (unsigned)((state.state->strm.next_in + state.state->strm.avail_in) -
+                              state.state->in);
             copy = state.state->size - have;
             if (copy > len)
                 copy = len;
             memcpy(state.state->in + have, buf, copy);
-            strm->avail_in += copy;
+            state.state->strm.avail_in += copy;
             state.state->x.pos += copy;
             buf = (const char *)buf + copy;
             len -= copy;
@@ -229,19 +224,83 @@ int ZEXPORT gzwrite(file, buf, len)
     }
     else {
         /* consume whatever's left in the input buffer */
-        if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
+        if (state.state->strm.avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
             return 0;
 
         /* directly compress user buffer to file */
-        strm->avail_in = len;
-        strm->next_in = (z_const Bytef *)buf;
-        state.state->x.pos += len;
-        if (gz_comp(state, Z_NO_FLUSH) == -1)
-            return 0;
+        state.state->strm.next_in = (z_const Bytef *)buf;
+        do {
+            z_size_t n = (unsigned)-1;
+            if (n > len)
+                n = len;
+            state.state->strm.avail_in = (z_uInt)n;
+            state.state->x.pos += n;
+            if (gz_comp(state, Z_NO_FLUSH) == -1)
+                return 0;
+            len -= n;
+        } while (len);
     }
 
-    /* input was all buffered or compressed (put will fit in int) */
-    return (int)put;
+    /* input was all buffered or compressed */
+    return put;
+}
+
+/* -- see zlib.h -- */
+int ZEXPORT gzwrite(file, buf, len)
+    gzFile file;
+    voidpc buf;
+    unsigned len;
+{
+    gz_statep state;
+
+    /* get internal structure */
+    if (file == NULL)
+        return 0;
+    state = (gz_statep)file;
+
+    /* check that we're writing and that there's no error */
+    if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
+        return 0;
+
+    /* since an int is returned, make sure len fits in one, otherwise return
+       with an error (this avoids a flaw in the interface) */
+    if ((int)len < 0) {
+        gz_error(state, Z_DATA_ERROR, "requested length does not fit in int");
+        return 0;
+    }
+
+    /* write len bytes from buf (the return value will fit in an int) */
+    return (int)gz_write(state, buf, len);
+}
+
+/* -- see zlib.h -- */
+z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
+    voidpc buf;
+    z_size_t size;
+    z_size_t nitems;
+    gzFile file;
+{
+    z_size_t len;
+    gz_statep state;
+
+    /* get internal structure */
+    if (file == NULL)
+        return 0;
+    state = (gz_statep)file;
+
+    /* check that we're writing and that there's no error */
+    if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
+        return 0;
+
+    /* compute bytes to read -- error on overflow */
+    len = nitems * size;
+    if (size && len / size != nitems) {
+        gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t");
+        return 0;
+    }
+
+    /* write len bytes to buf, return the number of full items written */
+    return len ? gz_write(state, buf, len) / size : 0;
 }
 
 /* -- see zlib.h -- */
@@ -271,14 +330,14 @@ int ZEXPORT gzputc(file, c)
             return -1;
     }
 
-    /* try writing to input buffer for speed (state->size == 0 if buffer not
+    /* try writing to input buffer for speed (state.state->size == 0 if buffer not
        initialized) */
     if (state.state->size) {
         if (strm->avail_in == 0)
             strm->next_in = state.state->in;
         have = (unsigned)((strm->next_in + strm->avail_in) - state.state->in);
         if (have < state.state->size) {
-            state.state->in[have] = c;
+            state.state->in[have] = (unsigned char)c;
             strm->avail_in++;
             state.state->x.pos++;
             return c & 0xff;
@@ -286,8 +345,8 @@ int ZEXPORT gzputc(file, c)
     }
 
     /* no room in buffer or not initialized, use gz_write() */
-    buf[0] = c;
-    if (gzwrite(file, buf, 1) != 1)
+    buf[0] = (unsigned char)c;
+    if (gz_write(state, buf, 1) != 1)
         return -1;
     return c & 0xff;
 }
@@ -298,11 +357,21 @@ int ZEXPORT gzputs(file, str)
     const char *str;
 {
     int ret;
-    unsigned len;
+    z_size_t len;
+    gz_statep state;
+
+    /* get internal structure */
+    if (file == NULL)
+        return -1;
+    state = (gz_statep)file;
+
+    /* check that we're writing and that there's no error */
+    if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
+        return -1;
 
     /* write string */
-    len = (unsigned)strlen(str);
-    ret = gzwrite(file, str, len);
+    len = strlen(str);
+    ret = (int)gz_write(state, str, len);
     return ret == 0 && len != 0 ? -1 : ret;
 }
 
@@ -312,63 +381,73 @@ int ZEXPORT gzputs(file, str)
 /* -- see zlib.h -- */
 int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va)
 {
-    int size, len;
+    int len;
+    unsigned left;
+    char *next;
     gz_statep state;
     z_streamp strm;
 
     /* get internal structure */
     if (file == NULL)
-        return -1;
+        return Z_STREAM_ERROR;
     state = (gz_statep)file;
     strm = &(state.state->strm);
 
     /* check that we're writing and that there's no error */
     if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
-        return 0;
+        return Z_STREAM_ERROR;
 
     /* make sure we have some buffer space */
     if (state.state->size == 0 && gz_init(state) == -1)
-        return 0;
+        return state.state->err;
 
     /* check for seek request */
     if (state.state->seek) {
         state.state->seek = 0;
         if (gz_zero(state, state.state->skip) == -1)
-            return 0;
+            return state.state->err;
     }
 
-    /* consume whatever's left in the input buffer */
-    if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
-        return 0;
-
-    /* do the printf() into the input buffer, put length in len */
-    size = (int)(state.state->size);
-    state.state->in[size - 1] = 0;
+    /* do the printf() into the input buffer, put length in len -- the input
+       buffer is double-sized just for this function, so there is guaranteed to
+       be state.state->size bytes available after the current contents */
+    if (strm->avail_in == 0)
+        strm->next_in = state.state->in;
+    next = (char *)(state.state->in + (strm->next_in - state.state->in) + strm->avail_in);
+    next[state.state->size - 1] = 0;
 #ifdef NO_vsnprintf
 #  ifdef HAS_vsprintf_void
-    (void)vsprintf((char *)(state.state->in), format, va);
-    for (len = 0; len < size; len++)
-        if (state.state->in[len] == 0) break;
+    (void)vsprintf(next, format, va);
+    for (len = 0; len < state.state->size; len++)
+        if (next[len] == 0) break;
 #  else
-    len = vsprintf((char *)(state.state->in), format, va);
+    len = vsprintf(next, format, va);
 #  endif
 #else
 #  ifdef HAS_vsnprintf_void
-    (void)vsnprintf((char *)(state.state->in), size, format, va);
-    len = strlen((char *)(state.state->in));
+    (void)vsnprintf(next, state.state->size, format, va);
+    len = strlen(next);
 #  else
-    len = vsnprintf((char *)(state.state->in), size, format, va);
+    len = vsnprintf(next, state.state->size, format, va);
 #  endif
 #endif
 
     /* check that printf() results fit in buffer */
-    if (len <= 0 || len >= (int)size || state.state->in[size - 1] != 0)
+    if (len == 0 || (unsigned)len >= state.state->size || next[state.state->size - 1] != 0)
         return 0;
 
-    /* update buffer and position, defer compression until needed */
-    strm->avail_in = (unsigned)len;
-    strm->next_in = state.state->in;
+    /* update buffer and position, compress first half if past that */
+    strm->avail_in += (unsigned)len;
     state.state->x.pos += len;
+    if (strm->avail_in >= state.state->size) {
+        left = strm->avail_in - state.state->size;
+        strm->avail_in = state.state->size;
+        if (gz_comp(state, Z_NO_FLUSH) == -1)
+            return state.state->err;
+        memcpy(state.state->in, state.state->in + state.state->size, left);
+        strm->next_in = state.state->in;
+        strm->avail_in = left;
+    }
     return len;
 }
 
@@ -393,73 +472,82 @@ int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
     int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
         a11, a12, a13, a14, a15, a16, a17, a18, a19, a20;
 {
-    int size, len;
+    unsigned len, left;
+    char *next;
     gz_statep state;
     z_streamp strm;
 
     /* get internal structure */
     if (file == NULL)
-        return -1;
+        return Z_STREAM_ERROR;
     state = (gz_statep)file;
     strm = &(state.state->strm);
 
     /* check that can really pass pointer in ints */
     if (sizeof(int) != sizeof(void *))
-        return 0;
+        return Z_STREAM_ERROR;
 
     /* check that we're writing and that there's no error */
     if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
-        return 0;
+        return Z_STREAM_ERROR;
 
     /* make sure we have some buffer space */
     if (state.state->size == 0 && gz_init(state) == -1)
-        return 0;
+        return state.state->error;
 
     /* check for seek request */
     if (state.state->seek) {
         state.state->seek = 0;
         if (gz_zero(state, state.state->skip) == -1)
-            return 0;
+            return state.state->error;
     }
 
-    /* consume whatever's left in the input buffer */
-    if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
-        return 0;
-
-    /* do the printf() into the input buffer, put length in len */
-    size = (int)(state.state->size);
-    state.state->in[size - 1] = 0;
+    /* do the printf() into the input buffer, put length in len -- the input
+       buffer is double-sized just for this function, so there is guaranteed to
+       be state.state->size bytes available after the current contents */
+    if (strm->avail_in == 0)
+        strm->next_in = state.state->in;
+    next = (char *)(strm->next_in + strm->avail_in);
+    next[state.state->size - 1] = 0;
 #ifdef NO_snprintf
 #  ifdef HAS_sprintf_void
-    sprintf((char *)(state.state->in), format, a1, a2, a3, a4, a5, a6, a7, a8,
-            a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
+    sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12,
+            a13, a14, a15, a16, a17, a18, a19, a20);
     for (len = 0; len < size; len++)
-        if (state.state->in[len] == 0) break;
+        if (next[len] == 0)
+            break;
 #  else
-    len = sprintf((char *)(state.state->in), format, a1, a2, a3, a4, a5, a6, a7, a8,
-                  a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
+    len = sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11,
+                  a12, a13, a14, a15, a16, a17, a18, a19, a20);
 #  endif
 #else
 #  ifdef HAS_snprintf_void
-    snprintf((char *)(state.state->in), size, format, a1, a2, a3, a4, a5, a6, a7, a8,
-             a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
-    len = strlen((char *)(state.state->in));
+    snprintf(next, state.state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9,
+             a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
+    len = strlen(next);
 #  else
-    len = snprintf((char *)(state.state->in), size, format, a1, a2, a3, a4, a5, a6,
-                   a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18,
-                   a19, a20);
+    len = snprintf(next, state.state->size, format, a1, a2, a3, a4, a5, a6, a7, a8,
+                   a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
 #  endif
 #endif
 
     /* check that printf() results fit in buffer */
-    if (len <= 0 || len >= (int)size || state.state->in[size - 1] != 0)
+    if (len == 0 || len >= state.state->size || next[state.state->size - 1] != 0)
         return 0;
 
-    /* update buffer and position, defer compression until needed */
-    strm->avail_in = (unsigned)len;
-    strm->next_in = state.state->in;
+    /* update buffer and position, compress first half if past that */
+    strm->avail_in += len;
     state.state->x.pos += len;
-    return len;
+    if (strm->avail_in >= state.state->size) {
+        left = strm->avail_in - state.state->size;
+        strm->avail_in = state.state->size;
+        if (gz_comp(state, Z_NO_FLUSH) == -1)
+            return state.state->err;
+        memcpy(state.state->in, state.state->in + state.state->size, left);
+        strm->next_in = state.state->in;
+        strm->avail_in = left;
+    }
+    return (int)len;
 }
 
 #endif
@@ -473,7 +561,7 @@ int ZEXPORT gzflush(file, flush)
 
     /* get internal structure */
     if (file == NULL)
-        return -1;
+        return Z_STREAM_ERROR;
     state = (gz_statep)file;
 
     /* check that we're writing and that there's no error */
@@ -488,11 +576,11 @@ int ZEXPORT gzflush(file, flush)
     if (state.state->seek) {
         state.state->seek = 0;
         if (gz_zero(state, state.state->skip) == -1)
-            return -1;
+            return state.state->err;
     }
 
     /* compress remaining data with requested flush */
-    gz_comp(state, flush);
+    (void)gz_comp(state, flush);
     return state.state->err;
 }
 
@@ -523,13 +611,13 @@ int ZEXPORT gzsetparams(file, level, strategy)
     if (state.state->seek) {
         state.state->seek = 0;
         if (gz_zero(state, state.state->skip) == -1)
-            return -1;
+            return state.state->err;
     }
 
     /* change compression parameters for subsequent input */
     if (state.state->size) {
         /* flush previous input with previous parameters before changing */
-        if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1)
+        if (strm->avail_in && gz_comp(state, Z_BLOCK) == -1)
             return state.state->err;
         deflateParams(strm, level, strategy);
     }

zlibWrapper/zstd_zlibwrapper.c

@@ -61,7 +61,7 @@ ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion();  }
 static void* ZWRAP_allocFunction(void* opaque, size_t size)
 {
     z_streamp strm = (z_streamp) opaque;
-    void* address = strm->zalloc(strm->opaque, 1, size);
+    void* address = strm->zalloc(strm->opaque, 1, (uInt)size);
   /*  printf("ZWRAP alloc %p, %d \n", address, (int)size); */
     return address;
 }
@@ -81,7 +81,8 @@ typedef enum { ZWRAP_useInit, ZWRAP_useReset, ZWRAP_streamEnd } ZWRAP_state_t;
 typedef struct {
     ZSTD_CStream* zbc;
     int compressionLevel;
-    int streamEnd;
+    int streamEnd; /* a flag to signal the end of a stream */
+    unsigned long long totalInBytes; /* we need it as strm->total_in can be reset by user */
     ZSTD_customMem customMem;
     z_stream allocFunc; /* copy of zalloc, zfree, opaque */
     ZSTD_inBuffer inBuffer;
@@ -130,7 +131,7 @@ int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize,
 {
     LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc);
     if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR;
-    
+
     if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize;
     { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
       size_t errorCode;
@@ -156,7 +157,7 @@ int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message)
     ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
     strm->msg = message;
     if (zwc == NULL) return Z_STREAM_ERROR;
-    
+
     return ZWRAPC_finishWithError(zwc, strm, 0);
 }
 
@@ -165,7 +166,7 @@ int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize)
 {
     ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
     if (zwc == NULL) return Z_STREAM_ERROR;
-    
+
     zwc->pledgedSrcSize = pledgedSrcSize;
     zwc->comprState = ZWRAP_useInit;
     return Z_OK;
@@ -189,6 +190,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
         level = ZWRAP_DEFAULT_CLEVEL;
 
     zwc->streamEnd = 0;
+    zwc->totalInBytes = 0;
     zwc->compressionLevel = level;
     strm->state = (struct internal_state*) zwc; /* use state which in not used by user */
     strm->total_in = 0;
@@ -217,7 +219,10 @@ int ZWRAP_deflateReset_keepDict(z_streamp strm)
         return deflateReset(strm);
 
     { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
-      if (zwc) zwc->streamEnd = 0;
+      if (zwc) { 
+          zwc->streamEnd = 0;
+          zwc->totalInBytes = 0;
+      }
     }
 
     strm->total_in = 0;
@@ -232,7 +237,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
     LOG_WRAPPERC("- deflateReset\n");
     if (!g_ZWRAP_useZSTDcompression)
         return deflateReset(strm);
-    
+
     ZWRAP_deflateReset_keepDict(strm);
 
     { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
@@ -284,12 +289,12 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
     if (zwc->zbc == NULL) {
         int res;
         zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
-        if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); 
+        if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
         res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
         if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
         if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
     } else {
-        if (strm->total_in == 0) {
+        if (zwc->totalInBytes == 0) {
             if (zwc->comprState == ZWRAP_useReset) {
                 size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
                 if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); }
@@ -317,13 +322,14 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         strm->total_out += zwc->outBuffer.pos;
         strm->avail_out -= zwc->outBuffer.pos;
         strm->total_in += zwc->inBuffer.pos;
+        zwc->totalInBytes += zwc->inBuffer.pos;
         strm->next_in += zwc->inBuffer.pos;
         strm->avail_in -= zwc->inBuffer.pos;
     }
 
-    if (flush == Z_FULL_FLUSH 
+    if (flush == Z_FULL_FLUSH
 #if ZLIB_VERNUM >= 0x1240
-        || flush == Z_TREES 
+        || flush == Z_TREES
 #endif
         || flush == Z_BLOCK)
         return ZWRAPC_finishWithErrorMsg(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!");
@@ -411,6 +417,7 @@ typedef struct {
     ZSTD_DStream* zbd;
     char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */
     int errorCount;
+    unsigned long long totalInBytes; /* we need it as strm->total_in can be reset by user */
     ZWRAP_state_t decompState;
     ZSTD_inBuffer inBuffer;
     ZSTD_outBuffer outBuffer;
@@ -424,7 +431,7 @@ typedef struct {
 } ZWRAP_DCtx;
 
 
-int ZWRAP_isUsingZSTDdecompression(z_streamp strm) 
+int ZWRAP_isUsingZSTDdecompression(z_streamp strm)
 {
     if (strm == NULL) return 0;
     return (strm->reserved == ZWRAP_ZSTD_STREAM);
@@ -458,7 +465,7 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
         memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
     }
 
-    MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_frameHeaderSize_min);   /* if compilation fails here, assertion is false */
+    MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN);   /* if compilation fails here, assertion is false */
     ZWRAP_initDCtx(zwd);
     return zwd;
 }
@@ -488,7 +495,7 @@ int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
     ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
     strm->msg = message;
     if (zwd == NULL) return Z_STREAM_ERROR;
-    
+
     return ZWRAPD_finishWithError(zwd, strm, 0);
 }
 
@@ -511,6 +518,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
     strcpy(zwd->version, version);
 
     zwd->stream_size = stream_size;
+    zwd->totalInBytes = 0;
     strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
     strm->total_in = 0;
     strm->total_out = 0;
@@ -528,7 +536,7 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int  windowBits,
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
         return inflateInit2_(strm, windowBits, version, stream_size);
     }
-    
+
     {
     int ret = z_inflateInit_ (strm, version, stream_size);
     LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits);
@@ -551,8 +559,9 @@ int ZWRAP_inflateReset_keepDict(z_streamp strm)
         if (zwd == NULL) return Z_STREAM_ERROR;
         ZWRAP_initDCtx(zwd);
         zwd->decompState = ZWRAP_useReset;
+        zwd->totalInBytes = 0;
     }
-    
+
     strm->total_in = 0;
     strm->total_out = 0;
     return Z_OK;
@@ -569,7 +578,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
       if (ret != Z_OK) return ret; }
 
     { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
-      if (zwd == NULL) return Z_STREAM_ERROR;    
+      if (zwd == NULL) return Z_STREAM_ERROR;
       zwd->decompState = ZWRAP_useInit; }
 
     return Z_OK;
@@ -608,11 +617,11 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
         if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
         errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
         if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
-        zwd->decompState = ZWRAP_useReset; 
+        zwd->decompState = ZWRAP_useReset;
 
-        if (strm->total_in == ZSTD_HEADERSIZE) {
+        if (zwd->totalInBytes == ZSTD_HEADERSIZE) {
             zwd->inBuffer.src = zwd->headerBuf;
-            zwd->inBuffer.size = strm->total_in;
+            zwd->inBuffer.size = zwd->totalInBytes;
             zwd->inBuffer.pos = 0;
             zwd->outBuffer.dst = strm->next_out;
             zwd->outBuffer.size = 0;
@@ -650,8 +659,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
         if (zwd == NULL) return Z_STREAM_ERROR;
         if (zwd->decompState == ZWRAP_streamEnd) return Z_STREAM_END;
 
-        if (strm->total_in < ZLIB_HEADERSIZE) {
-            if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
+        if (zwd->totalInBytes < ZLIB_HEADERSIZE) {
+            if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
                 if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
                     if (zwd->windowBits)
                         errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
@@ -668,12 +677,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                     return res;
                 }
             } else {
-                srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
-                memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
+                srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - zwd->totalInBytes);
+                memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
                 strm->total_in += srcSize;
+                zwd->totalInBytes += srcSize;
                 strm->next_in += srcSize;
                 strm->avail_in -= srcSize;
-                if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
+                if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK;
 
                 if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
                     z_stream strm2;
@@ -725,9 +735,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
             zwd->decompState = ZWRAP_useInit;
         }
 
-        if (strm->total_in < ZSTD_HEADERSIZE)
+        if (zwd->totalInBytes < ZSTD_HEADERSIZE)
         {
-            if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
+            if (zwd->totalInBytes == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
                 if (zwd->decompState == ZWRAP_useInit) {
                     errorCode = ZSTD_initDStream(zwd->zbd);
                     if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
@@ -736,12 +746,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                     if (ZSTD_isError(errorCode)) goto error;
                 }
             } else {
-                srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
-                memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
+                srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - zwd->totalInBytes);
+                memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
                 strm->total_in += srcSize;
+                zwd->totalInBytes += srcSize;
                 strm->next_in += srcSize;
                 strm->avail_in -= srcSize;
-                if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
+                if (zwd->totalInBytes < ZSTD_HEADERSIZE) return Z_OK;
 
                 if (zwd->decompState == ZWRAP_useInit) {
                     errorCode = ZSTD_initDStream(zwd->zbd);
@@ -785,11 +796,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
         strm->total_out += zwd->outBuffer.pos;
         strm->avail_out -= zwd->outBuffer.pos;
         strm->total_in += zwd->inBuffer.pos;
+        zwd->totalInBytes += zwd->inBuffer.pos;
         strm->next_in += zwd->inBuffer.pos;
         strm->avail_in -= zwd->inBuffer.pos;
-        if (errorCode == 0) { 
-            LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); 
-            zwd->decompState = ZWRAP_streamEnd; 
+        if (errorCode == 0) {
+            LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+            zwd->decompState = ZWRAP_streamEnd;
             return Z_STREAM_END;
         }
     }
@@ -1042,3 +1054,24 @@ ZEXTERN uLong ZEXPORT z_crc32   OF((uLong crc, const Bytef *buf, uInt len))
 {
     return crc32(crc, buf, len);
 }
+
+
+#if ZLIB_VERNUM >= 0x12B0
+ZEXTERN uLong ZEXPORT z_adler32_z OF((uLong adler, const Bytef *buf, z_size_t len))
+{
+    return adler32_z(adler, buf, len);
+}
+
+ZEXTERN uLong ZEXPORT z_crc32_z OF((uLong crc, const Bytef *buf, z_size_t len))
+{
+    return crc32_z(crc, buf, len);
+}
+#endif
+
+
+#if ZLIB_VERNUM >= 0x1270
+ZEXTERN const z_crc_t FAR * ZEXPORT z_get_crc_table    OF((void))
+{
+    return get_crc_table();
+}
+#endif

zlibWrapper/zstd_zlibwrapper.h

@@ -30,7 +30,13 @@ const char * zstdVersion(void);
 
 
 /*** COMPRESSION ***/
-/* enables/disables zstd compression during runtime */
+/* ZWRAP_useZSTDcompression() enables/disables zstd compression during runtime.
+   By default zstd compression is disabled. To enable zstd compression please use one of the methods:
+   - compilation with the additional option -DZWRAP_USE_ZSTD=1 
+   - using '#define ZWRAP_USE_ZSTD 1' in source code before '#include "zstd_zlibwrapper.h"'
+   - calling ZWRAP_useZSTDcompression(1)
+   All above-mentioned methods will enable zstd compression for all threads.
+   Be aware that ZWRAP_useZSTDcompression() is not thread-safe and may lead to a race condition. */
 void ZWRAP_useZSTDcompression(int turn_on);
 
 /* checks if zstd compression is turned on */
@@ -54,7 +60,11 @@ int ZWRAP_deflateReset_keepDict(z_streamp strm);
 /*** DECOMPRESSION ***/
 typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_AUTO } ZWRAP_decompress_type;
 
-/* enables/disables automatic recognition of zstd/zlib compressed data during runtime */
+/* ZWRAP_setDecompressionType() enables/disables automatic recognition of zstd/zlib compressed data during runtime.
+   By default auto-detection of zstd and zlib streams in enabled (ZWRAP_AUTO).
+   Forcing zlib decompression with ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB) slightly improves
+   decompression speed of zlib-encoded streams.
+   Be aware that ZWRAP_setDecompressionType() is not thread-safe and may lead to a race condition. */
 void ZWRAP_setDecompressionType(ZWRAP_decompress_type type);
 
 /* checks zstd decompression type */