Merge pull request #954 from facebook/dev

Release v1.3.3
Merge pull request #959 from shawnl/dev
2017-12-21 10:16:40 +01:00 · 2017-12-20 11:20:07 +01:00 · 2017-12-19 22:02:03 -08:00 · 2017-12-20 00:12:50 +01:00 · 2017-12-19 21:49:04 +01:00 · 2017-12-19 21:20:46 +01:00
94 changed files with 4967 additions and 3688 deletions
@@ -19,3 +19,6 @@
 # Windows
 *.bat text eol=crlf
 *.cmd text eol=crlf
+
+# .travis.yml merging
+.travis.yml merge=ours
@@ -1,4 +1,4 @@
-# Medium Tests: Run on all commits/PRs to dev branch
+# Long tests: run on commits to master branch/cron builds

 language: c
 sudo: required
@@ -6,21 +6,37 @@ dist: trusty
 matrix:
  include:
    # Ubuntu 14.04
-    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test-zstd'
-    - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32'
-    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
+    - env: Cmd="make cxxtest && make clean && make gnu90build && make clean && make cmakebuild && make clean && make travis-install"

-    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
-    - env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest'
-    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
+    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test'
+    - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test32'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test'
    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
-
    - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'

-    - env: Cmd='make arminstall && make armfuzz'
-    - env: Cmd='make arminstall && make aarch64fuzz'
-    - env: Cmd='make ppcinstall && make ppcfuzz'
-    - env: Cmd='make ppcinstall && make ppc64fuzz'
+    - env: Cmd='make arminstall && make armtest'
+    - env: Cmd='make arminstall && make aarch64test'
+    - env: Cmd='make ppcinstall && make ppctest'
+    - env: Cmd='make ppcinstall && make ppc64test'
+
+    - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
+    - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zstd-nolegacy'
+    - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zbuff'
+
+    - env: Cmd='make -j uasanregressiontest'
+    - env: Cmd='make -j msanregressiontest'
+
+    - env: Cmd='make -C tests versionsTest && make lz4install && make clean && make -C tests test-lz4'
+
+    # OS X Mavericks
+    - env: Cmd="make test"
+      os: osx
+
+before_install:
+  - if [ `uname` = "Darwin" ]; then brew update; fi
+
+install:
+  - if [ `uname` = "Darwin" ]; then brew install xz; fi

 git:
  depth: 1
@@ -33,7 +49,13 @@ branches:
 script:
  - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
  - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-  - export FUZZERTEST=-T2mn;
-    export ZSTREAM_TESTTIME=-T2mn;
-    export DECODECORPUS_TESTTIME=-T1mn;
+  - export FUZZERTEST=-T3mn;
+    export ZSTREAM_TESTTIME=-T80s;
+    export DECODECORPUS_TESTTIME=-T40s;
+    if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
+        date;
+        git fetch origin dev;
+        git checkout -f FETCH_HEAD;
+        date;
+    fi;
    sh -c "$Cmd" || travis_terminate 1;
@@ -12,6 +12,7 @@ ZSTDDIR  = lib
 BUILDIR  = build
 ZWRAPDIR = zlibWrapper
 TESTDIR  = tests
+FUZZDIR  = $(TESTDIR)/fuzz

 # Define nul output
 VOID = /dev/null
@@ -71,9 +72,12 @@ zstdmt:
 zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) test

+.PHONY: check
+check: shortest
+
 .PHONY: test shortest
 test shortest:
-	$(MAKE) -C $(PRGDIR) allVariants
+	$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS="-g -DZSTD_DEBUG=1"
 	$(MAKE) -C $(TESTDIR) $@

 .PHONY: examples
@@ -97,6 +101,7 @@ clean:
 	@$(MAKE) -C examples/ $@ > $(VOID)
 	@$(MAKE) -C contrib/gen_html $@ > $(VOID)
 	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
+	@$(RM) -r lz4
 	@echo Cleaning completed

 #------------------------------------------------------------------------------
@@ -111,7 +116,7 @@ CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_B
 list:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs

-.PHONY: install clangtest gpptest armtest usan asan uasan
+.PHONY: install clangtest armtest usan asan uasan
 install:
 	@$(MAKE) -C $(ZSTDDIR) $@
 	@$(MAKE) -C $(PRGDIR) $@
@@ -125,11 +130,6 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir

-.PHONY: gppbuild
-gppbuild: clean
-	g++ -v
-	CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-
 .PHONY: gcc5build
 gcc5build: clean
 	gcc-5 -v
@@ -161,7 +161,7 @@ aarch64build: clean
 	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd

 ppcbuild: clean
-	CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd

 ppc64build: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
@@ -178,8 +178,10 @@ ppcfuzz: clean
 ppc64fuzz: clean
 	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest

-gpptest: clean
-	CC=$(CXX) $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+.PHONY: cxxtest
+cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+cxxtest: clean
+	$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)"   # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly

 gcc5test: clean
 	gcc-5 -v
@@ -215,6 +217,15 @@ arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"

+regressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest
+
+uasanregressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
+
+msanregressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
+
 # run UBsan with -fsanitize-recover=signed-integer-overflow
 # due to a bug in UBsan when doing pointer subtraction
 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
@@ -274,6 +285,10 @@ gpp6install: apt-add-repo
 clang38install:
 	APT_PACKAGES="clang-3.8" $(MAKE) apt-install

+# Ubuntu 14.04 ships a too-old lz4
+lz4install:
+	[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
+
 endif


@@ -284,7 +299,7 @@ endif


 #------------------------------------------------------------------------
-#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
+# target specific tests
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 cmakebuild:
@@ -294,38 +309,38 @@ cmakebuild:
 	cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall

 c90build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`

 gnu90build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-std=gnu90" $(MAKE) allmost

 c99build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-std=c99" $(MAKE) allmost

 gnu99build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-std=gnu99" $(MAKE) allmost

 c11build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-std=c11" $(MAKE) allmost

 bmix64build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test

 bmix32build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test

 bmi32build: clean
-	gcc -v
+	$(CC) -v
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test

 staticAnalyze: clean
-	gcc -v
+	$(CC) -v
 	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
 endif
@@ -1,14 +1,37 @@
+v1.3.3
+perf: faster zstd_opt strategy (levels 17-19)
+fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
+cli : fix : content size written in header by default
+cli : fix : improved LZ4 format support, by @felixhandte
+cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
+api : fix : support large skippable frames, by @terrelln
+api : fix : streaming interface was adding a useless 3-bytes null block to small frames
+api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
+build: fix : compilation under rhel6 and centos6, reported by @pixelb
+build: added `check` target
+
 v1.3.2
 new : long range mode, using --long command, by Stella Lau (@stellamplau)
-license : changed /examples license to BSD + GPLv2
-license : fix a few header files to reflect new license (#825)
+new : ability to generate and decode magicless frames (#591)
+changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
 fix : multi-threading compression works with custom allocators
-fix : a rare compression bug when compression generates very large distances (only possible at --ultra -22)
+fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
+fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
 fix : 32-bits build can now decode large offsets (levels 21+)
+cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
+cli : improved --list output
 cli : new : can split input file for dictionary training, using command -B#
+cli : new : clean operation artefact on Ctrl-C interruption
 cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
+cli : fix : write file size in header in multiple-files mode
+api : added macro ZSTD_COMPRESSBOUND() for static allocation
+api : experimental : new advanced decompression API
+api : fix : sizeof_CCtx() used to over-estimate
 build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
 build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
+example : added streaming_memory_usage
+license : changed /examples license to BSD + GPLv2
+license : fix a few header files to reflect new license (#825)

 v1.3.1
 New license : BSD + GPLv2
@@ -1,14 +1,15 @@
- __Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
- targeting real-time compression scenarios at zlib-level and better compression ratios.
+<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/readme/doc/images/zstd_logo86.png" alt="Zstandard"></p>

-It is provided as an open-source BSD-licensed **C** library,
-and a command line utility producing and decoding `.zst` and `.gz` files.
-For other programming languages,
-you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
+__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
+targeting real-time compression scenarios at zlib-level and better compression ratios.
+It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).

-| dev branch status |
-|-------------------|
-| [![Build Status][travisDevBadge]][travisLink]   [![Build status][AppveyorDevBadge]][AppveyorLink]   [![Build status][CircleDevBadge]][CircleLink]
+The project is provided as an open-source BSD-licensed **C** library,
+and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
+Should your project require another programming language,
+a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
+
+Development branch status : [![Build Status][travisDevBadge]][travisLink]   [![Build status][AppveyorDevBadge]][AppveyorLink]   [![Build status][CircleDevBadge]][CircleLink]

 [travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
 [travisLink]: https://travis-ci.org/facebook/zstd
@@ -17,8 +18,9 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
 [CircleLink]: https://circleci.com/gh/facebook/zstd

+### Benchmarks

-As a reference, several fast compression algorithms were tested and compared
+For reference, several fast compression algorithms were tested and compared
 on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
 with a Core i7-6700K CPU @ 4.0GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
@@ -43,7 +45,9 @@ on the [Silesia compression corpus].
 [LZ4]: http://www.lz4.org/

 Zstd can also offer stronger compression ratios at the cost of compression speed.
-Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
+Speed vs Compression trade-off is configurable by small increments.
+Decompression speed is preserved and remains roughly the same at all settings,
+a property shared by most LZ compression algorithms, such as [zlib] or lzma.

 The following tests were run
 on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
@@ -56,8 +60,8 @@ Compression Speed vs Ratio | Decompression Speed
 ---------------------------|--------------------
 ![Compression Speed vs Ratio](doc/images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/Dspeed4.png "Decompression Speed")

-Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph.
-For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) .
+A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.
+For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).


 ### The case for Small Data compression
@@ -84,7 +88,7 @@ Training works if there is some correlation in a family of small data samples. T
 Hence, deploying one dictionary per type of data will provide the greatest benefits.
 Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.

-#### Dictionary compression How To :
+#### Dictionary compression How To:

 1) Create the dictionary

@@ -99,19 +103,16 @@ Dictionary gains are mostly effective in the first few KB. Then, the compression
 `zstd -D dictionaryName --decompress FILE.zst`


-### Build
-
-Once you have the repository cloned, there are multiple ways provided to build Zstandard.
+### Build instructions

 #### Makefile

-If your system is compatible with a standard `make` (or `gmake`) binary generator,
-you can simply run it at the root directory.
-It will generate `zstd` within root directory.
+If your system is compatible with standard `make` (or `gmake`),
+invoking `make` in root directory will generate `zstd` cli in root directory.

-Other available options include :
- `make install` : create and install zstd binary, library and man page
- `make test` : create and run `zstd` and test tools on local platform
+Other available options include:
+- `make install` : create and install zstd cli, library and man pages
+- `make check` : create and run `zstd`, tests its behavior on local platform

 #### cmake

@@ -125,9 +126,9 @@ A Meson project is provided within `contrib/meson`.

 #### Visual Studio (Windows)

-Going into `build` directory, you will find additional possibilities :
- Projects for Visual Studio 2005, 2008 and 2010
-  + VS2010 project is compatible with VS2012, VS2013 and VS2015
+Going into `build` directory, you will find additional possibilities:
+- Projects for Visual Studio 2005, 2008 and 2010.
+  + VS2010 project is compatible with VS2012, VS2013 and VS2015.
 - Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`,
  which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.

@@ -143,11 +144,7 @@ Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).

 ### Contributing

-The "dev" branch is the one where all contributions will be merged before reaching "master".
-If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
+The "dev" branch is the one where all contributions are merged before reaching "master".
+If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
 Direct commit to "master" are not permitted.
 For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
-
-### Miscellaneous
-
-Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy).
@@ -63,7 +63,7 @@ SET(Headers
        ${LIBRARY_DIR}/common/huf.h
        ${LIBRARY_DIR}/common/mem.h
        ${LIBRARY_DIR}/common/zstd_internal.h
-        ${LIBRARY_DIR}/compress/zstd_compress.h
+        ${LIBRARY_DIR}/compress/zstd_compress_internal.h
        ${LIBRARY_DIR}/compress/zstd_fast.h
        ${LIBRARY_DIR}/compress/zstd_double_fast.h
        ${LIBRARY_DIR}/compress/zstd_lazy.h
@@ -3,13 +3,11 @@ dependencies:
    - sudo dpkg --add-architecture i386
    - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
    - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    - sudo apt-get -y install libstdc++-7-dev clang gcc g++ gcc-5 gcc-6 gcc-7 zlib1g-dev liblzma-dev
-    - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386

 test:
  override:
    - ? |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; make all   && make clean && make -C lib libzstd-nomt && make clean; fi &&
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi &&
        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build   && make clean; fi
      :
        parallel: true
@@ -20,22 +18,7 @@ test:
        parallel: true
    - ? |
        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c11build     && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild   && make clean; fi
-      :
-        parallel: true
-    - ? |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gppbuild     && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build    && make clean; fi
-      :
-        parallel: true
-    - ? |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gcc6build    && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild   && make clean; fi
-      :
-        parallel: true
-    - ? |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make m32build     && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build   && make clean; fi
      :
        parallel: true
    - ? |
@@ -44,8 +27,8 @@ test:
      :
        parallel: true
    - ? |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make ppc64build   && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build    && make clean; fi #could add another test here
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make -j regressiontest && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
      :
        parallel: true
    - ? |
@@ -53,6 +36,11 @@ test:
        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
      :
        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make cxxtest      && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi
+      :
+        parallel: true

  post:
    - echo Circle CI tests finished
@@ -141,7 +141,7 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
  return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
 }

-static unsigned ZSTD_NbCommonBytes (register size_t val) {
+static unsigned ZSTD_NbCommonBytes (size_t val) {
  if (MEM_isLittleEndian()) {
    if (MEM_64bits()) {
 #    if defined(_MSC_VER) && defined(_WIN64)
@@ -854,4 +854,3 @@ void LDM_outputConfiguration(void) {
  printf("HASH_CHAR_OFFSET: %d\n", HASH_CHAR_OFFSET);
  printf("=====================\n");
 }
-
@@ -2,14 +2,39 @@ project('zstd', 'c', license: 'BSD')

 libm = meson.get_compiler('c').find_library('m', required: true)

-lib_dir = join_paths(meson.source_root(), '..', '..', 'lib')
+lib_dir = join_paths('..', '..', 'lib')
 common_dir = join_paths(lib_dir, 'common')
 compress_dir = join_paths(lib_dir, 'compress')
 decompress_dir = join_paths(lib_dir, 'decompress')
 dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
 deprecated_dir = join_paths(lib_dir, 'deprecated')

-libzstd_srcs = [join_paths(common_dir, 'entropy_common.c'), join_paths(common_dir, 'fse_decompress.c'), join_paths(common_dir, 'threading.c'), join_paths(common_dir, 'pool.c'), join_paths(common_dir, 'zstd_common.c'), join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstdmt_compress.c'), join_paths(decompress_dir, 'huf_decompress.c'), join_paths(decompress_dir, 'zstd_decompress.c'), join_paths(dictbuilder_dir, 'cover.c'), join_paths(dictbuilder_dir, 'divsufsort.c'), join_paths(dictbuilder_dir, 'zdict.c'), join_paths(deprecated_dir, 'zbuff_common.c'), join_paths(deprecated_dir, 'zbuff_compress.c'), join_paths(deprecated_dir, 'zbuff_decompress.c')]
+libzstd_srcs = [
+    join_paths(common_dir, 'entropy_common.c'),
+    join_paths(common_dir, 'fse_decompress.c'),
+    join_paths(common_dir, 'threading.c'),
+    join_paths(common_dir, 'pool.c'),
+    join_paths(common_dir, 'zstd_common.c'),
+    join_paths(common_dir, 'error_private.c'),
+    join_paths(common_dir, 'xxhash.c'),
+    join_paths(compress_dir, 'fse_compress.c'),
+    join_paths(compress_dir, 'huf_compress.c'),
+    join_paths(compress_dir, 'zstd_compress.c'),
+    join_paths(compress_dir, 'zstd_fast.c'),
+    join_paths(compress_dir, 'zstd_double_fast.c'),
+    join_paths(compress_dir, 'zstd_lazy.c'),
+    join_paths(compress_dir, 'zstd_opt.c'),
+    join_paths(compress_dir, 'zstd_ldm.c'),
+    join_paths(compress_dir, 'zstdmt_compress.c'),
+    join_paths(decompress_dir, 'huf_decompress.c'),
+    join_paths(decompress_dir, 'zstd_decompress.c'),
+    join_paths(dictbuilder_dir, 'cover.c'),
+    join_paths(dictbuilder_dir, 'divsufsort.c'),
+    join_paths(dictbuilder_dir, 'zdict.c'),
+    join_paths(deprecated_dir, 'zbuff_common.c'),
+    join_paths(deprecated_dir, 'zbuff_compress.c'),
+    join_paths(deprecated_dir, 'zbuff_decompress.c')
+]

 libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]

@@ -19,7 +44,15 @@ if get_option('legacy_support')

    legacy_dir = join_paths(lib_dir, 'legacy')
    libzstd_includes += [include_directories(legacy_dir)]
-    libzstd_srcs += [join_paths(legacy_dir, 'zstd_v01.c'), join_paths(legacy_dir, 'zstd_v02.c'), join_paths(legacy_dir, 'zstd_v03.c'), join_paths(legacy_dir, 'zstd_v04.c'), join_paths(legacy_dir, 'zstd_v05.c'), join_paths(legacy_dir, 'zstd_v06.c'), join_paths(legacy_dir, 'zstd_v07.c')]
+    libzstd_srcs += [
+        join_paths(legacy_dir, 'zstd_v01.c'),
+        join_paths(legacy_dir, 'zstd_v02.c'),
+        join_paths(legacy_dir, 'zstd_v03.c'),
+        join_paths(legacy_dir, 'zstd_v04.c'),
+        join_paths(legacy_dir, 'zstd_v05.c'),
+        join_paths(legacy_dir, 'zstd_v06.c'),
+        join_paths(legacy_dir, 'zstd_v07.c')
+    ]
 else
    libzstd_cflags = []
 endif
@@ -39,16 +72,20 @@ libzstd = library('zstd',
                  dependencies: libzstd_deps,
                  install: true)

-programs_dir = join_paths(meson.source_root(), '..', '..', 'programs')
+programs_dir = join_paths('..', '..', 'programs')

 zstd = executable('zstd',
-                  join_paths(programs_dir, 'bench.c'), join_paths(programs_dir, 'datagen.c'), join_paths(programs_dir, 'dibio.c'), join_paths(programs_dir, 'fileio.c'), join_paths(programs_dir, 'zstdcli.c'),
+                  join_paths(programs_dir, 'bench.c'),
+                  join_paths(programs_dir, 'datagen.c'),
+                  join_paths(programs_dir, 'dibio.c'),
+                  join_paths(programs_dir, 'fileio.c'),
+                  join_paths(programs_dir, 'zstdcli.c'),
                  include_directories: libzstd_includes,
                  c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
                  link_with: libzstd,
                  install: true)

-tests_dir = join_paths(meson.source_root(), '..', '..', 'tests')
+tests_dir = join_paths('..', '..', 'tests')
 datagen_c = join_paths(programs_dir, 'datagen.c')
 test_includes = libzstd_includes + [include_directories(programs_dir)]

@@ -2,9 +2,10 @@
 * Copyright (c) 2017-present, Facebook, Inc.
 * All rights reserved.
 *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
 */

 /* *********************************************************
@@ -91,10 +91,10 @@ Overview

 Frames
 ------
-Zstandard compressed data is made of up one or more __frames__.
+Zstandard compressed data is made of one or more __frames__.
 Each frame is independent and can be decompressed indepedently of other frames.
 The decompressed content of multiple concatenated frames is the concatenation of
-each frames decompressed content.
+each frame decompressed content.

 There are two frame formats defined by Zstandard:
  Zstandard frames and Skippable frames.
@@ -182,7 +182,7 @@ data must be regenerated within a single continuous memory segment.
 In this case, `Window_Descriptor` byte is skipped,
 but `Frame_Content_Size` is necessarily present.
 As a consequence, the decoder must allocate a memory segment
-of size equal or bigger than `Frame_Content_Size`.
+of size equal or larger than `Frame_Content_Size`.

 In order to preserve the decoder from unreasonable memory requirements,
 a decoder is allowed to reject a compressed frame
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.3.1 Manual</title>
+<title>zstd 1.3.3 Manual</title>
 </head>
 <body>
-<h1>zstd 1.3.1 Manual</h1>
+<h1>zstd 1.3.3 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -19,16 +19,17 @@
 <li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
 <li><a href="#Chapter10">START OF ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
 <li><a href="#Chapter11">Advanced types</a></li>
-<li><a href="#Chapter12">Frame size functions</a></li>
-<li><a href="#Chapter13">Context memory usage</a></li>
-<li><a href="#Chapter14">Advanced compression functions</a></li>
-<li><a href="#Chapter15">Advanced decompression functions</a></li>
-<li><a href="#Chapter16">Advanced streaming functions</a></li>
-<li><a href="#Chapter17">Buffer-less and synchronous inner streaming functions</a></li>
-<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li>
-<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li>
-<li><a href="#Chapter20">ZSTD_CCtx_params</a></li>
-<li><a href="#Chapter21">Block functions</a></li>
+<li><a href="#Chapter12">Custom memory allocation functions</a></li>
+<li><a href="#Chapter13">Frame size functions</a></li>
+<li><a href="#Chapter14">Context memory usage</a></li>
+<li><a href="#Chapter15">Advanced compression functions</a></li>
+<li><a href="#Chapter16">Advanced decompression functions</a></li>
+<li><a href="#Chapter17">Advanced streaming functions</a></li>
+<li><a href="#Chapter18">Buffer-less and synchronous inner streaming functions</a></li>
+<li><a href="#Chapter19">Buffer-less streaming compression (synchronous mode)</a></li>
+<li><a href="#Chapter20">Buffer-less streaming decompression (synchronous mode)</a></li>
+<li><a href="#Chapter21">New advanced API (experimental)</a></li>
+<li><a href="#Chapter22">Block level API</a></li>
 </ol>
 <hr>
 <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -111,10 +112,11 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
  @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. 
 </p></pre><BR>

-<h3>Helper functions</h3><pre></pre><b><pre>int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
+<h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
 size_t      ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
 unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
 const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
+int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
 </pre></b><BR>
 <a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>

@@ -345,17 +347,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
    ZSTD_frameParameters fParams;
 } ZSTD_parameters;
 </b></pre><BR>
-<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
-typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
-typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-</b>/* use this constant to defer to stdlib's functions */<b>
-static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
-</pre></b><BR>
-<a name="Chapter12"></a><h2>Frame size functions</h2><pre></pre>
+<a name="Chapter12"></a><h2>Custom memory allocation functions</h2><pre></pre>
+
+<pre><b>typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+</b></pre><BR>
+<a name="Chapter13"></a><h2>Frame size functions</h2><pre></pre>

 <pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
 </b><p>  `src` should point to the start of a ZSTD encoded frame or skippable frame
-  `srcSize` must be at least as large as the frame
+  `srcSize` must be >= first frame size
  @return : the compressed size of the first frame starting at `src`,
            suitable to pass to `ZSTD_decompress` or similar,
            or an error code if input is invalid 
@@ -390,7 +390,7 @@ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
   @return : size of the Frame Header 
 </p></pre><BR>

-<a name="Chapter13"></a><h2>Context memory usage</h2><pre></pre>
+<a name="Chapter14"></a><h2>Context memory usage</h2><pre></pre>

 <pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
@@ -399,33 +399,33 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 </b><p>  These functions give the current memory usage of selected object.
-  Object memory usage can evolve if it's re-used multiple times. 
+  Object memory usage can evolve when re-used multiple times. 
 </p></pre><BR>

 <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
-size_t ZSTD_estimateCCtxSize_advanced_usingCParams(ZSTD_compressionParameters cParams);
-size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params);
+size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 size_t ZSTD_estimateDCtxSize(void);
 </b><p>  These functions make it possible to estimate memory usage
  of a future {D,C}Ctx, before its creation.
  ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
  It will also consider src size to be arbitrarily "large", which is worst case.
-  If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced_usingCParams() can provide a tighter estimation.
-  ZSTD_estimateCCtxSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
-  ZSTD_estimateCCtxSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1.
+  If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
+  ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1.
  Note : CCtx estimation is only correct for single-threaded compression 
 </p></pre><BR>

 <pre><b>size_t ZSTD_estimateCStreamSize(int compressionLevel);
-size_t ZSTD_estimateCStreamSize_advanced_usingCParams(ZSTD_compressionParameters cParams);
-size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params);
+size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 size_t ZSTD_estimateDStreamSize(size_t windowSize);
 size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
 </b><p>  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
  It will also consider src size to be arbitrarily "large", which is worst case.
-  If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced_usingCParams() can provide a tighter estimation.
-  ZSTD_estimateCStreamSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
-  ZSTD_estimateCStreamSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1.
+  If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
+  ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1.
  Note : CStream estimation is only correct for single-threaded compression.
  ZSTD_DStream memory budget depends on window Size.
  This information can be passed manually, using ZSTD_estimateDStreamSize,
@@ -436,8 +436,8 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
 </p></pre><BR>

 <pre><b>typedef enum {
-    ZSTD_dlm_byCopy = 0,      </b>/* Copy dictionary content internally. */<b>
-    ZSTD_dlm_byRef,           </b>/* Reference dictionary content -- the dictionary buffer must outlives its users. */<b>
+    ZSTD_dlm_byCopy = 0,     </b>/**< Copy dictionary content internally */<b>
+    ZSTD_dlm_byRef,          </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b>
 } ZSTD_dictLoadMethod_e;
 </b></pre><BR>
 <pre><b>size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
@@ -449,7 +449,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
 
 </p></pre><BR>

-<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
+<a name="Chapter15"></a><h2>Advanced compression functions</h2><pre></pre>

 <pre><b>ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 </b><p>  Create a ZSTD compression context using external alloc and free functions 
@@ -461,7 +461,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
             It must outlive context usage.
  workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
                 to determine how large workspace must be to support scenario.
- @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
+ @return : pointer to ZSTD_CCtx* (same address as workspace, but different type),
+           or NULL if error (typically size too small)
  Note : zstd will never resize nor malloc() when using a static cctx.
         If it needs more memory than available, it will simply error out.
  Note 2 : there is no corresponding "free" function.
@@ -504,7 +505,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
                 to determine how large workspace must be.
  cParams : use ZSTD_getCParams() to transform a compression level
            into its relevants cParams.
- @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ @return : pointer to ZSTD_CDict* (same address as workspace, but different type),
+           or NULL if error (typically, size too small).
  Note : there is no corresponding "free" function.
         Since workspace was allocated externally, it must be freed externally.
 
@@ -517,7 +519,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet

 <pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 </b><p>   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
-   All fields of `ZSTD_frameParameters` are set to default (0) 
+   All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 
 </p></pre><BR>

 <pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
@@ -544,7 +546,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
 </b><p>   Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters 
 </p></pre><BR>

-<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre>
+<a name="Chapter16"></a><h2>Advanced decompression functions</h2><pre></pre>

 <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
 </b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -563,7 +565,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
             It must outlive context usage.
  workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
                 to determine how large workspace must be to support scenario.
- @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
+ @return : pointer to ZSTD_DCtx* (same address as workspace, but different type),
+           or NULL if error (typically size too small)
  Note : zstd will never resize nor malloc() when using a static dctx.
         If it needs more memory than available, it will simply error out.
  Note 2 : static dctx is incompatible with legacy support
@@ -626,42 +629,44 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
 </p></pre><BR>

-<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
+<a name="Chapter17"></a><h2>Advanced streaming functions</h2><pre></pre>

 <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticCCtx() */<b>
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */<b>
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b>
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b>
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b>
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b>
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b>
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */<b>
 </pre></b><BR>
 <pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 </b><p>  start a new compression job, using same parameters from previous job.
  This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
  Note that zcs must be init at least once before using ZSTD_resetCStream().
-  pledgedSrcSize==0 means "srcSize unknown".
+  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
-  @return : 0, or an error code (which can be tested using ZSTD_isError()) 
+  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+  but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
+ @return : 0, or an error code (which can be tested using ZSTD_isError()) 
 </p></pre><BR>

-<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
-ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticDCtx() */<b>
-size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
-size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
+typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
+size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);   </b>/* obsolete : this API will be removed in a future version */<b>
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */<b>
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  </b>/**< note : ddict is referenced, it must outlive decompression session */<b>
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
 </pre></b><BR>
-<a name="Chapter17"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
+<a name="Chapter18"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
  This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
-  But it's also a complex one, with many restrictions (documented below).
-  Prefer using normal streaming API for an easier experience
+  But it's also a complex one, with several restrictions, documented below.
+  Prefer normal streaming API for an easier experience.
 
 <BR></pre>

-<a name="Chapter18"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
+<a name="Chapter19"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
  A ZSTD_CCtx object is required to track streaming operations.
  Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
  ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -673,8 +678,8 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds);  </b>/**< re-use decompression para

  Then, consume your input using ZSTD_compressContinue().
  There are some important considerations to keep in mind when using this advanced function :
-  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
-  - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks.
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
+  - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
    Worst case evaluation is provided by ZSTD_compressBound().
    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
@@ -685,19 +690,19 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds);  </b>/**< re-use decompression para

  Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
-  Without last block mark, frames will be considered unfinished (corrupted) by decoders.
+  Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.

-  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
+  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
 <BR></pre>

 <h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
-size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   </b>/* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */<b>
-size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */<b>
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </pre></b><BR>
-<a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
+<a name="Chapter20"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
  A ZSTD_DCtx object is required to track streaming operations.
  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
  A ZSTD_DCtx object can be re-used multiple times.
@@ -783,15 +788,38 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 </pre></b><BR>
 <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 </b></pre><BR>
-<h3>New advanced API (experimental, and compression only)</h3><pre></pre><b><pre></pre></b><BR>
+<a name="Chapter21"></a><h2>New advanced API (experimental)</h2><pre></pre>
+
 <pre><b>typedef enum {
+    </b>/* Question : should we have a format ZSTD_f_auto ?<b>
+     * For the time being, it would mean exactly the same as ZSTD_f_zstd1.
+     * But, in the future, should several formats be supported,
+     * on the compression side, it would mean "default format".
+     * On the decompression side, it would mean "multi format",
+     * and ZSTD_f_zstd1 could be reserved to mean "accept *only* zstd frames".
+     * Since meaning is a little different, another option could be to define different enums for compression and decompression.
+     * This question could be kept for later, when there are actually multiple formats to support,
+     * but there is also the question of pinning enum values, and pinning value `0` is especially important */
+    ZSTD_f_zstd1 = 0,        </b>/* zstd frame format, specified in zstd_compression_format.md (default) */<b>
+    ZSTD_f_zstd1_magicless,  </b>/* Variant of zstd frame format, without initial 4-bytes magic number.<b>
+                              * Useful to save 4 bytes per generated frame.
+                              * Decoder cannot recognise automatically this format, requiring instructions. */
+} ZSTD_format_e;
+</b></pre><BR>
+<pre><b>typedef enum {
+    </b>/* compression format */<b>
+    ZSTD_p_format = 10,      </b>/* See ZSTD_format_e enum definition.<b>
+                              * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
+
    </b>/* compression parameters */<b>
    ZSTD_p_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b>
                              * Default level is ZSTD_CLEVEL_DEFAULT==3.
                              * Special: value 0 means "do not change cLevel". */
    ZSTD_p_windowLog,        </b>/* Maximum allowed back-reference distance, expressed as power of 2.<b>
                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * Special: value 0 means "do not change windowLog". */
+                              * Special: value 0 means "do not change windowLog".
+                              * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
+                              * requires setting the maximum window size at least as large during decompression. */
    ZSTD_p_hashLog,          </b>/* Size of the probe table, as a power of 2.<b>
                              * Resulting table size is (1 << (hashLog+2)).
                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -824,18 +852,19 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
                              * Special: value 0 means "do not change strategy". */

    </b>/* frame parameters */<b>
-    ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1)<b>
-                              * note that content size must be known at the beginning,
-                              * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
+    ZSTD_p_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
+                              * Content size must be known at the beginning of compression,
+                              * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
    ZSTD_p_checksumFlag,     </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
-    ZSTD_p_dictIDFlag,       </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
+    ZSTD_p_dictIDFlag,       </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>

    </b>/* multi-threading parameters */<b>
    ZSTD_p_nbThreads=400,    </b>/* Select how many threads a compression job can spawn (default:1)<b>
                              * More threads improve speed, but also increase memory usage.
                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
                              * Special: value 0 means "do not change nbThreads" */
-    ZSTD_p_jobSize,          </b>/* Size of a compression job. Each compression job is completed in parallel.<b>
+    ZSTD_p_jobSize,          </b>/* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.<b>
+                              * Each compression job is completed in parallel, so indirectly controls the nb of active threads.
                              * 0 means default, which is dynamically determined based on compression parameters.
                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                              * The minimum size is automatically and transparently enforced */
@@ -859,7 +888,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
                          * Larger values increase memory usage and compression ratio, but decrease
                          * compression speed.
                          * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
-                          * (default: 20). */
+                          * (default: windowlog - 7). */
    ZSTD_p_ldmMinMatch,  </b>/* Minimum size of searched matches for long distance matcher.<b>
                          * Larger/too small values usually decrease compression ratio.
                          * Must be clamped between ZSTD_LDM_MINMATCH_MIN
@@ -880,7 +909,8 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 <pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
 </b><p>  Set one compression parameter, selected by enum ZSTD_cParameter.
  Note : when `value` is an enum, cast it to unsigned for proper type checking.
-  @result : 0, or an error code (which can be tested with ZSTD_isError()). 
+  @result : informational value (typically, the one being set, possibly corrected),
+            or an error code (which can be tested with ZSTD_isError()). 
 </p></pre><BR>

 <pre><b>size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
@@ -889,7 +919,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 @result : 0, or an error code (which can be tested with ZSTD_isError()).
  Note 1 : 0 means zero, empty.
           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
-           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
  Note 2 : If all data is provided and consumed in a single round,
           this value is overriden by srcSize instead. 
 </p></pre><BR>
@@ -949,7 +979,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
 <pre><b>typedef enum {
    ZSTD_e_continue=0, </b>/* collect more data, encoder transparently decides when to output result, for optimal conditions */<b>
    ZSTD_e_flush,      </b>/* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */<b>
-    ZSTD_e_end         </b>/* flush any remaining data and ends current frame. Any future compression starts a new frame. */<b>
+    ZSTD_e_end         </b>/* flush any remaining data and close current frame. Any additional data starts a new frame. */<b>
 } ZSTD_EndDirective;
 </b></pre><BR>
 <pre><b>size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
@@ -959,15 +989,21 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
 </b><p>  Behave about the same as ZSTD_compressStream. To note :
  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
  - Compression parameters cannot be changed once compression is started.
-  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
-  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
-  - @return provides the minimum amount of data still to flush from internal buffers
+  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
+  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+  - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
+  - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
+                                                     and then immediately returns, just indicating that there is some data remaining to be flushed.
+                                                     The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+  - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
+  - @return provides the minimum amount of data remaining to be flushed from internal buffers
            or an error code, which can be tested using ZSTD_isError().
-            if @return != 0, flush is not fully completed, there is some data left within internal buffers.
-  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
+            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+            This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
+  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
-            It is necessary to fully flush internal buffers
-            before starting a new compression job, or changing compression parameters.
+            Before starting a new compression job, or changing compression parameters,
+            it is required to fully flush internal buffers.
 
 </p></pre><BR>

@@ -976,6 +1012,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
  Useful after an error, or to interrupt an ongoing compression job and start a new one.
  Any internal data not yet flushed is cancelled.
  Dictionary (if any) is dropped.
+  All parameters are back to default values.
  It's possible to modify compression parameters after a reset.
 
 </p></pre><BR>
@@ -987,26 +1024,30 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
                ZSTD_EndDirective endOp);
 </b><p>  Same as ZSTD_compress_generic(),
  but using only integral types as arguments.
-  Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
+  Argument list is larger than ZSTD_{in,out}Buffer,
  but can be helpful for binders from dynamic languages
  which have troubles handling structures containing memory pointers.
 
 </p></pre><BR>

-<a name="Chapter20"></a><h2>ZSTD_CCtx_params</h2><pre>
+<pre><b>ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+</b><p>  Quick howto :
  - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
-  - ZSTD_CCtxParam_setParameter() : Push parameters one by one into an
-  existing ZSTD_CCtx_params structure. This is similar to
-  ZSTD_CCtx_setParameter().
-  - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to an existing CCtx. These
-  parameters will be applied to all subsequent compression jobs.
+  - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
+                                    an existing ZSTD_CCtx_params structure.
+                                    This is similar to
+                                    ZSTD_CCtx_setParameter().
+  - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+                                    an existing CCtx.
+                                    These parameters will be applied to
+                                    all subsequent compression jobs.
  - ZSTD_compress_generic() : Do compression using the CCtx.
  - ZSTD_freeCCtxParams() : Free the memory.

-  This can be used with ZSTD_estimateCCtxSize_opaque() for static allocation
-  for single-threaded compression.
+  This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+  for static allocation for single-threaded compression.
 
-<BR></pre>
+</p></pre><BR>

 <pre><b>size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
 </b><p>  Reset params to default, with the default compression level.
@@ -1030,22 +1071,116 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
  Set one compression parameter, selected by enum ZSTD_cParameter.
  Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
  Note : when `value` is an enum, cast it to unsigned for proper type checking.
-  @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
 
 </p></pre><BR>

 <pre><b>size_t ZSTD_CCtx_setParametersUsingCCtxParams(
        ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
-</b><p> Apply a set of ZSTD_CCtx_params to the compression context.
- This must be done before the dictionary is loaded.
- The pledgedSrcSize is treated as unknown.
- Multithreading parameters are applied only if nbThreads > 1.
+</b><p>  Apply a set of ZSTD_CCtx_params to the compression context.
+  This must be done before the dictionary is loaded.
+  The pledgedSrcSize is treated as unknown.
+  Multithreading parameters are applied only if nbThreads > 1.
 
 </p></pre><BR>

-<a name="Chapter21"></a><h2>Block functions</h2><pre>
-    Block functions produce and decode raw zstd blocks, without frame metadata.
-    Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+<h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR>
+<pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);   </b>/* not implemented */<b>
+size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);   </b>/* not implemented */<b>
+size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);   </b>/* not implemented */<b>
+</b><p>  Create an internal DDict from dict buffer,
+  to be used to decompress next frames.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+            meaning "return to no-dictionary mode".
+  Note 1 : `dict` content will be copied internally.
+            Use ZSTD_DCtx_loadDictionary_byReference()
+            to reference dictionary content instead.
+            In which case, the dictionary buffer must outlive its users.
+  Note 2 : Loading a dictionary involves building tables,
+           which has a non-negligible impact on CPU usage and latency.
+  Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
+           how dictionary content will be interpreted and loaded.
+ 
+</p></pre><BR>
+
+<pre><b>size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);   </b>/* not implemented */<b>
+</b><p>  Reference a prepared dictionary, to be used to decompress next frames.
+  The dictionary remains active for decompression of future frames using same DCtx.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Note 1 : Currently, only one dictionary can be managed.
+           Referencing a new dictionary effectively "discards" any previous one.
+  Special : adding a NULL DDict means "return to no-dictionary mode".
+  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ 
+</p></pre><BR>
+
+<pre><b>size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize);   </b>/* not implemented */<b>
+size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);   </b>/* not implemented */<b>
+</b><p>  Reference a prefix (single-usage dictionary) for next compression job.
+  Prefix is **only used once**. It must be explicitly referenced before each frame.
+  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+  Note 2 : Prefix buffer is referenced. It must outlive compression job.
+  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
+  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ 
+</p></pre><BR>
+
+<pre><b>size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+</b><p>  Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+  This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
+  This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
+  By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
+ @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ 
+</p></pre><BR>
+
+<pre><b>size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+</b><p>  Instruct the decoder context about what kind of data to decode next.
+  This instruction is mandatory to decode data without a fully-formed header,
+  such ZSTD_f_zstd1_magicless for example.
+ @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ 
+</p></pre><BR>
+
+<pre><b>size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
+                               ZSTD_outBuffer* output,
+                               ZSTD_inBuffer* input);
+</b><p>  Behave the same as ZSTD_decompressStream.
+  Decompression parameters cannot be changed once decompression is started.
+ @return : an error code, which can be tested using ZSTD_isError()
+           if >0, a hint, nb of expected input bytes for next invocation.
+           `0` means : a frame has just been fully decoded and flushed.
+ 
+</p></pre><BR>
+
+<pre><b>size_t ZSTD_decompress_generic_simpleArgs (
+                ZSTD_DCtx* dctx,
+                void* dst, size_t dstCapacity, size_t* dstPos,
+          const void* src, size_t srcSize, size_t* srcPos);
+</b><p>  Same as ZSTD_decompress_generic(),
+  but using only integral types as arguments.
+  Argument list is larger than ZSTD_{in,out}Buffer,
+  but can be helpful for binders from dynamic languages
+  which have troubles handling structures containing memory pointers.
+ 
+</p></pre><BR>
+
+<pre><b>void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
+</b><p>  Return a DCtx to clean state.
+  If a decompression was ongoing, any internal data not yet flushed is cancelled.
+  All parameters are back to default values, including sticky ones.
+  Dictionary (if any) is dropped.
+  Parameters can be modified again after a reset.
+ 
+</p></pre><BR>
+
+<a name="Chapter22"></a><h2>Block level API</h2><pre></pre>
+
+<pre><b></b><p>    Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
    User will have to take in charge required information to regenerate data, such as compressed and content sizes.

    A few rules to respect :
@@ -1055,7 +1190,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
      + compression : any ZSTD_compressBegin*() variant, including with dictionary
      + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
      + copyCCtx() and copyDCtx() can be used too
-    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
+    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
      + If input is larger than a block size, it's necessary to split input data into multiple blocks
      + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
        Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -1066,7 +1201,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
      + In case of multiple successive blocks, should some of them be uncompressed,
        decoder must be informed of their existence in order to follow proper history.
        Use ZSTD_insertBlock() for such a case.
-<BR></pre>
+</p></pre><BR>

 <h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
 size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -85,7 +85,7 @@ int main(int argc, char const *argv[]) {
                return 1;
            }
        }
-        
+
        size_t compressedSize;
        {   ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
            ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
@@ -133,7 +133,7 @@ int main(int argc, char const *argv[]) {

        size_t const cstreamSize = ZSTD_sizeof_CStream(cstream);
        size_t const cstreamEstimatedSize = wLog ?
-                ZSTD_estimateCStreamSize_advanced_usingCParams(params.cParams) :
+                ZSTD_estimateCStreamSize_usingCParams(params.cParams) :
                ZSTD_estimateCStreamSize(compressionLevel);
        size_t const dstreamSize = ZSTD_sizeof_DStream(dstream);

@@ -15,15 +15,9 @@ cxx_library(
    header_namespace='',
    visibility=['PUBLIC'],
    exported_headers=subdir_glob([
-        ('compress', 'zstdmt_compress.h'),
+        ('compress', 'zstd*.h'),
    ]),
-    headers=subdir_glob([
-        ('compress', 'zstd_opt.h'),
-    ]),
-    srcs=[
-        'compress/zstd_compress.c',
-        'compress/zstdmt_compress.c',
-    ],
+    srcs=glob(['compress/zstd*.c']),
    deps=[':common'],
 )

@@ -31,7 +25,7 @@ cxx_library(
    name='decompress',
    header_namespace='',
    visibility=['PUBLIC'],
-    srcs=['decompress/zstd_decompress.c'],
+    srcs=glob(['decompress/zstd*.c']),
    deps=[
        ':common',
        ':legacy',
@@ -58,6 +52,9 @@ cxx_library(
    ]),
    srcs=glob(['legacy/*.c']),
    deps=[':common'],
+    exported_preprocessor_flags=[
+        '-DZSTD_LEGACY_SUPPORT=4',
+    ],
 )

 cxx_library(
@@ -74,6 +71,15 @@ cxx_library(
    deps=[':common'],
 )

+cxx_library(
+    name='compiler',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'compiler.h'),
+    ]),
+)
+
 cxx_library(
    name='bitstream',
    header_namespace='',
@@ -100,6 +106,7 @@ cxx_library(
    ],
    deps=[
        ':bitstream',
+        ':compiler',
        ':errors',
        ':mem',
    ],
@@ -133,7 +140,10 @@ cxx_library(
        ('common', 'pool.h'),
    ]),
    srcs=['common/pool.c'],
-    deps=[':threading'],
+    deps=[
+        ':threading',
+        ':zstd_common',
+    ],
 )

 cxx_library(
@@ -144,6 +154,12 @@ cxx_library(
        ('common', 'threading.h'),
    ]),
    srcs=['common/threading.c'],
+    exported_preprocessor_flags=[
+        '-DZSTD_MULTITHREAD',
+    ],
+    exported_linker_flags=[
+        '-pthread',
+    ],
 )

 cxx_library(
@@ -154,6 +170,9 @@ cxx_library(
        ('common', 'xxhash.h'),
    ]),
    srcs=['common/xxhash.c'],
+    exported_preprocessor_flags=[
+        '-DXXH_NAMESPACE=ZSTD_',
+    ],
 )

 cxx_library(
@@ -166,6 +185,7 @@ cxx_library(
    ]),
    srcs=['common/zstd_common.c'],
    deps=[
+        ':compiler',
        ':errors',
        ':mem',
    ],
@@ -175,6 +195,7 @@ cxx_library(
    name='common',
    deps=[
        ':bitstream',
+        ':compiler',
        ':entropy',
        ':errors',
        ':mem',
@@ -167,7 +167,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 /*-**************************************************************
 *  Internal functions
 ****************************************************************/
-MEM_STATIC unsigned BIT_highbit32 (register U32 val)
+MEM_STATIC unsigned BIT_highbit32 (U32 val)
 {
    assert(val != 0);
    {
@@ -194,11 +194,14 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val)
 }

 /*=====    Local Constants   =====*/
-static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F,
-                                    0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF,
-                                    0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
-                                    0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF };   /* up to 26 bits */
-
+static const unsigned BIT_mask[] = {
+    0,          1,         3,         7,         0xF,       0x1F,
+    0x3F,       0x7F,      0xFF,      0x1FF,     0x3FF,     0x7FF,
+    0xFFF,      0x1FFF,    0x3FFF,    0x7FFF,    0xFFFF,    0x1FFFF,
+    0x3FFFF,    0x7FFFF,   0xFFFFF,   0x1FFFFF,  0x3FFFFF,  0x7FFFFF,
+    0xFFFFFF,   0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
+    0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
+#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))

 /*-**************************************************************
 *  bitStream encoding
@@ -220,11 +223,14 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
 }

 /*! BIT_addBits() :
- *  can add up to 26 bits into `bitC`.
+ *  can add up to 31 bits into `bitC`.
 *  Note : does not check for register overflow ! */
 MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
                            size_t value, unsigned nbBits)
 {
+    MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);
+    assert(nbBits < BIT_MASK_SIZE);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
    bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
    bitC->bitPos += nbBits;
 }
@@ -235,6 +241,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
                                size_t value, unsigned nbBits)
 {
    assert((value>>nbBits) == 0);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
    bitC->bitContainer |= value << bitC->bitPos;
    bitC->bitPos += nbBits;
 }
@@ -245,7 +252,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
 MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
 {
    size_t const nbBytes = bitC->bitPos >> 3;
-    assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) );
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
    MEM_writeLEST(bitC->ptr, bitC->bitContainer);
    bitC->ptr += nbBytes;
    assert(bitC->ptr <= bitC->endPtr);
@@ -261,7 +268,7 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
 MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
 {
    size_t const nbBytes = bitC->bitPos >> 3;
-    assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) );
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
    MEM_writeLEST(bitC->ptr, bitC->bitContainer);
    bitC->ptr += nbBytes;
    if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
@@ -353,12 +360,14 @@ MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 co
 #  endif
        return _bextr_u32(bitContainer, start, nbBits);
 #else
+    assert(nbBits < BIT_MASK_SIZE);
    return (bitContainer >> start) & BIT_mask[nbBits];
 #endif
 }

 MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 {
+    assert(nbBits < BIT_MASK_SIZE);
    return bitContainer & BIT_mask[nbBits];
 }

@@ -30,14 +30,15 @@ const char* ERR_getErrorString(ERR_enum code)
    case PREFIX(init_missing): return "Context should be init first";
    case PREFIX(memory_allocation): return "Allocation error : not enough memory";
    case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
-    case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
-    case PREFIX(srcSize_wrong): return "Src size is incorrect";
    case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
    case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
    case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
    case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
    case PREFIX(dictionary_wrong): return "Dictionary mismatch";
    case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
+    case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+    case PREFIX(srcSize_wrong): return "Src size is incorrect";
+        /* following error codes are not stable and may be removed or changed in a future version */
    case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
    case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
    case PREFIX(maxCode):
@@ -184,7 +184,7 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const sh
 /*! Constructor and Destructor of FSE_CTable.
    Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
 typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
-FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
+FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
 FSE_PUBLIC_API void        FSE_freeCTable (FSE_CTable* ct);

 /*! FSE_buildCTable():
@@ -242,7 +242,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,

 /** HUF_readCTable() :
 *   Loading a CTable saved with HUF_writeCTable() */
-size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);


 /*
@@ -56,8 +56,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
  typedef   int32_t S32;
  typedef  uint64_t U64;
  typedef   int64_t S64;
-  typedef  intptr_t iPtrDiff;
-  typedef uintptr_t uPtrDiff;
 #else
  typedef unsigned char      BYTE;
  typedef unsigned short      U16;
@@ -66,8 +64,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
  typedef   signed int        S32;
  typedef unsigned long long  U64;
  typedef   signed long long  S64;
-  typedef ptrdiff_t      iPtrDiff;
-  typedef size_t         uPtrDiff;
 #endif


@@ -123,20 +119,26 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 /* currently only defined for gcc and icc */
 #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
    __pragma( pack(push, 1) )
-    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
+    typedef struct { U16 v; } unalign16;
+    typedef struct { U32 v; } unalign32;
+    typedef struct { U64 v; } unalign64;
+    typedef struct { size_t v; } unalignArch;
    __pragma( pack(pop) )
 #else
-    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+    typedef struct { U16 v; } __attribute__((packed)) unalign16;
+    typedef struct { U32 v; } __attribute__((packed)) unalign32;
+    typedef struct { U64 v; } __attribute__((packed)) unalign64;
+    typedef struct { size_t v; } __attribute__((packed)) unalignArch;
 #endif

-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
+MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }

-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }

 #else

@@ -11,7 +11,6 @@

 /* ======   Dependencies   ======= */
 #include <stddef.h>  /* size_t */
-#include <stdlib.h>  /* malloc, calloc, free */
 #include "pool.h"

 /* ======   Compiler specifics   ====== */
@@ -33,7 +32,7 @@ typedef struct POOL_job_s {
 struct POOL_ctx_s {
    ZSTD_customMem customMem;
    /* Keep track of the threads */
-    pthread_t *threads;
+    ZSTD_pthread_t *threads;
    size_t numThreads;

    /* The queue is a circular buffer */
@@ -48,11 +47,11 @@ struct POOL_ctx_s {
    int queueEmpty;

    /* The mutex protects the queue */
-    pthread_mutex_t queueMutex;
+    ZSTD_pthread_mutex_t queueMutex;
    /* Condition variable for pushers to wait on when the queue is full */
-    pthread_cond_t queuePushCond;
+    ZSTD_pthread_cond_t queuePushCond;
    /* Condition variables for poppers to wait on when the queue is empty */
-    pthread_cond_t queuePopCond;
+    ZSTD_pthread_cond_t queuePopCond;
    /* Indicates if the queue is shutting down */
    int shutdown;
 };
@@ -67,14 +66,14 @@ static void* POOL_thread(void* opaque) {
    if (!ctx) { return NULL; }
    for (;;) {
        /* Lock the mutex and wait for a non-empty queue or until shutdown */
-        pthread_mutex_lock(&ctx->queueMutex);
+        ZSTD_pthread_mutex_lock(&ctx->queueMutex);

        while (ctx->queueEmpty && !ctx->shutdown) {
-            pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+            ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
        }
        /* empty => shutting down: so stop */
        if (ctx->queueEmpty) {
-            pthread_mutex_unlock(&ctx->queueMutex);
+            ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
            return opaque;
        }
        /* Pop a job off the queue */
@@ -83,49 +82,49 @@ static void* POOL_thread(void* opaque) {
            ctx->numThreadsBusy++;
            ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
            /* Unlock the mutex, signal a pusher, and run the job */
-            pthread_mutex_unlock(&ctx->queueMutex);
-            pthread_cond_signal(&ctx->queuePushCond);
+            ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+            ZSTD_pthread_cond_signal(&ctx->queuePushCond);

            job.function(job.opaque);

            /* If the intended queue size was 0, signal after finishing job */
            if (ctx->queueSize == 1) {
-                pthread_mutex_lock(&ctx->queueMutex);
+                ZSTD_pthread_mutex_lock(&ctx->queueMutex);
                ctx->numThreadsBusy--;
-                pthread_mutex_unlock(&ctx->queueMutex);
-                pthread_cond_signal(&ctx->queuePushCond);
+                ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+                ZSTD_pthread_cond_signal(&ctx->queuePushCond);
        }   }
    }  /* for (;;) */
    /* Unreachable */
 }

-POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
    return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
 }

-POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
-    POOL_ctx *ctx;
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
+    POOL_ctx* ctx;
    /* Check the parameters */
    if (!numThreads) { return NULL; }
    /* Allocate the context and zero initialize */
-    ctx = (POOL_ctx *)ZSTD_calloc(sizeof(POOL_ctx), customMem);
+    ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
    if (!ctx) { return NULL; }
    /* Initialize the job queue.
     * It needs one extra space since one space is wasted to differentiate empty
     * and full queues.
     */
    ctx->queueSize = queueSize + 1;
-    ctx->queue = (POOL_job*) malloc(ctx->queueSize * sizeof(POOL_job));
+    ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
    ctx->queueHead = 0;
    ctx->queueTail = 0;
    ctx->numThreadsBusy = 0;
    ctx->queueEmpty = 1;
-    (void)pthread_mutex_init(&ctx->queueMutex, NULL);
-    (void)pthread_cond_init(&ctx->queuePushCond, NULL);
-    (void)pthread_cond_init(&ctx->queuePopCond, NULL);
+    (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
+    (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
+    (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
    ctx->shutdown = 0;
    /* Allocate space for the thread handles */
-    ctx->threads = (pthread_t*)ZSTD_malloc(numThreads * sizeof(pthread_t), customMem);
+    ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
    ctx->numThreads = 0;
    ctx->customMem = customMem;
    /* Check for errors */
@@ -133,7 +132,7 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
    /* Initialize the threads */
    {   size_t i;
        for (i = 0; i < numThreads; ++i) {
-            if (pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+            if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
                ctx->numThreads = i;
                POOL_free(ctx);
                return NULL;
@@ -146,27 +145,27 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
 /*! POOL_join() :
    Shutdown the queue, wake any sleeping threads, and join all of the threads.
 */
-static void POOL_join(POOL_ctx *ctx) {
+static void POOL_join(POOL_ctx* ctx) {
    /* Shut down the queue */
-    pthread_mutex_lock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_lock(&ctx->queueMutex);
    ctx->shutdown = 1;
-    pthread_mutex_unlock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
    /* Wake up sleeping threads */
-    pthread_cond_broadcast(&ctx->queuePushCond);
-    pthread_cond_broadcast(&ctx->queuePopCond);
+    ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
+    ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
    /* Join all of the threads */
    {   size_t i;
        for (i = 0; i < ctx->numThreads; ++i) {
-            pthread_join(ctx->threads[i], NULL);
+            ZSTD_pthread_join(ctx->threads[i], NULL);
    }   }
 }

 void POOL_free(POOL_ctx *ctx) {
    if (!ctx) { return; }
    POOL_join(ctx);
-    pthread_mutex_destroy(&ctx->queueMutex);
-    pthread_cond_destroy(&ctx->queuePushCond);
-    pthread_cond_destroy(&ctx->queuePopCond);
+    ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
+    ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
+    ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
    ZSTD_free(ctx->queue, ctx->customMem);
    ZSTD_free(ctx->threads, ctx->customMem);
    ZSTD_free(ctx, ctx->customMem);
@@ -176,7 +175,7 @@ size_t POOL_sizeof(POOL_ctx *ctx) {
    if (ctx==NULL) return 0;  /* supports sizeof NULL */
    return sizeof(*ctx)
        + ctx->queueSize * sizeof(POOL_job)
-        + ctx->numThreads * sizeof(pthread_t);
+        + ctx->numThreads * sizeof(ZSTD_pthread_t);
 }

 /**
@@ -198,12 +197,12 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
    POOL_ctx* const ctx = (POOL_ctx*)ctxVoid;
    if (!ctx) { return; }

-    pthread_mutex_lock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_lock(&ctx->queueMutex);
    {   POOL_job const job = {function, opaque};

        /* Wait until there is space in the queue for the new job */
        while (isQueueFull(ctx) && !ctx->shutdown) {
-          pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+          ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
        }
        /* The queue is still going => there is space */
        if (!ctx->shutdown) {
@@ -212,8 +211,8 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
            ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
        }
    }
-    pthread_mutex_unlock(&ctx->queueMutex);
-    pthread_cond_signal(&ctx->queuePopCond);
+    ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+    ZSTD_pthread_cond_signal(&ctx->queuePopCond);
 }

 #else  /* ZSTD_MULTITHREAD  not defined */
@@ -35,12 +35,12 @@ int g_ZSTD_threading_useles_symbol;

 static unsigned __stdcall worker(void *arg)
 {
-    pthread_t* const thread = (pthread_t*) arg;
+    ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
    thread->arg = thread->start_routine(thread->arg);
    return 0;
 }

-int pthread_create(pthread_t* thread, const void* unused,
+int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
            void* (*start_routine) (void*), void* arg)
 {
    (void)unused;
@@ -54,16 +54,16 @@ int pthread_create(pthread_t* thread, const void* unused,
        return 0;
 }

-int _pthread_join(pthread_t * thread, void **value_ptr)
+int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
 {
    DWORD result;

-    if (!thread->handle) return 0;
+    if (!thread.handle) return 0;

-    result = WaitForSingleObject(thread->handle, INFINITE);
+    result = WaitForSingleObject(thread.handle, INFINITE);
    switch (result) {
    case WAIT_OBJECT_0:
-        if (value_ptr) *value_ptr = thread->arg;
+        if (value_ptr) *value_ptr = thread.arg;
        return 0;
    case WAIT_ABANDONED:
        return EINVAL;
@@ -44,32 +44,31 @@ extern "C" {


 /* mutex */
-#define pthread_mutex_t           CRITICAL_SECTION
-#define pthread_mutex_init(a,b)   (InitializeCriticalSection((a)), 0)
-#define pthread_mutex_destroy(a)  DeleteCriticalSection((a))
-#define pthread_mutex_lock(a)     EnterCriticalSection((a))
-#define pthread_mutex_unlock(a)   LeaveCriticalSection((a))
+#define ZSTD_pthread_mutex_t           CRITICAL_SECTION
+#define ZSTD_pthread_mutex_init(a, b)  (InitializeCriticalSection((a)), 0)
+#define ZSTD_pthread_mutex_destroy(a)  DeleteCriticalSection((a))
+#define ZSTD_pthread_mutex_lock(a)     EnterCriticalSection((a))
+#define ZSTD_pthread_mutex_unlock(a)   LeaveCriticalSection((a))

 /* condition variable */
-#define pthread_cond_t             CONDITION_VARIABLE
-#define pthread_cond_init(a, b)    (InitializeConditionVariable((a)), 0)
-#define pthread_cond_destroy(a)    /* No delete */
-#define pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
-#define pthread_cond_signal(a)     WakeConditionVariable((a))
-#define pthread_cond_broadcast(a)  WakeAllConditionVariable((a))
+#define ZSTD_pthread_cond_t             CONDITION_VARIABLE
+#define ZSTD_pthread_cond_init(a, b)    (InitializeConditionVariable((a)), 0)
+#define ZSTD_pthread_cond_destroy(a)    /* No delete */
+#define ZSTD_pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
+#define ZSTD_pthread_cond_signal(a)     WakeConditionVariable((a))
+#define ZSTD_pthread_cond_broadcast(a)  WakeAllConditionVariable((a))

-/* pthread_create() and pthread_join() */
+/* ZSTD_pthread_create() and ZSTD_pthread_join() */
 typedef struct {
    HANDLE handle;
    void* (*start_routine)(void*);
    void* arg;
-} pthread_t;
+} ZSTD_pthread_t;

-int pthread_create(pthread_t* thread, const void* unused,
+int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
                   void* (*start_routine) (void*), void* arg);

-#define pthread_join(a, b) _pthread_join(&(a), (b))
-int _pthread_join(pthread_t* thread, void** value_ptr);
+int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);

 /**
 * add here more wrappers as required
@@ -80,23 +79,40 @@ int _pthread_join(pthread_t* thread, void** value_ptr);
 /* ===   POSIX Systems   === */
 #  include <pthread.h>

+#define ZSTD_pthread_mutex_t            pthread_mutex_t
+#define ZSTD_pthread_mutex_init(a, b)   pthread_mutex_init((a), (b))
+#define ZSTD_pthread_mutex_destroy(a)   pthread_mutex_destroy((a))
+#define ZSTD_pthread_mutex_lock(a)      pthread_mutex_lock((a))
+#define ZSTD_pthread_mutex_unlock(a)    pthread_mutex_unlock((a))
+
+#define ZSTD_pthread_cond_t             pthread_cond_t
+#define ZSTD_pthread_cond_init(a, b)    pthread_cond_init((a), (b))
+#define ZSTD_pthread_cond_destroy(a)    pthread_cond_destroy((a))
+#define ZSTD_pthread_cond_wait(a, b)    pthread_cond_wait((a), (b))
+#define ZSTD_pthread_cond_signal(a)     pthread_cond_signal((a))
+#define ZSTD_pthread_cond_broadcast(a)  pthread_cond_broadcast((a))
+
+#define ZSTD_pthread_t                  pthread_t
+#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
+#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+
 #else  /* ZSTD_MULTITHREAD not defined */
 /* No multithreading support */

-#define pthread_mutex_t int   /* #define rather than typedef, because sometimes pthread support is implicit, resulting in duplicated symbols */
-#define pthread_mutex_init(a,b)    ((void)a, 0)
-#define pthread_mutex_destroy(a)
-#define pthread_mutex_lock(a)
-#define pthread_mutex_unlock(a)
+typedef int ZSTD_pthread_mutex_t;
+#define ZSTD_pthread_mutex_init(a, b)   ((void)a, 0)
+#define ZSTD_pthread_mutex_destroy(a)
+#define ZSTD_pthread_mutex_lock(a)
+#define ZSTD_pthread_mutex_unlock(a)

-#define pthread_cond_t int
-#define pthread_cond_init(a,b)     ((void)a, 0)
-#define pthread_cond_destroy(a)
-#define pthread_cond_wait(a,b)
-#define pthread_cond_signal(a)
-#define pthread_cond_broadcast(a)
+typedef int ZSTD_pthread_cond_t;
+#define ZSTD_pthread_cond_init(a, b)    ((void)a, 0)
+#define ZSTD_pthread_cond_destroy(a)
+#define ZSTD_pthread_cond_wait(a, b)
+#define ZSTD_pthread_cond_signal(a)
+#define ZSTD_pthread_cond_broadcast(a)

-/* do not use pthread_t */
+/* do not use ZSTD_pthread_t */

 #endif /* ZSTD_MULTITHREAD */

@@ -31,21 +31,27 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
 *  ZSTD Error Management
 ******************************************/
 /*! ZSTD_isError() :
-*   tells if a return value is an error code */
+ *  tells if a return value is an error code */
 unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }

 /*! ZSTD_getErrorName() :
-*   provides error code string from function result (useful for debugging) */
+ *  provides error code string from function result (useful for debugging) */
 const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }

 /*! ZSTD_getError() :
-*   convert a `size_t` function result into a proper ZSTD_errorCode enum */
+ *  convert a `size_t` function result into a proper ZSTD_errorCode enum */
 ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }

 /*! ZSTD_getErrorString() :
-*   provides error code string from enum */
+ *  provides error code string from enum */
 const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }

+/*! g_debuglog_enable :
+ *  turn on/off debug traces (global switch) */
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 2)
+int g_debuglog_enable = 1;
+#endif
+

 /*=**************************************************************
 *  Custom allocator
@@ -63,9 +63,10 @@ typedef enum {
  ZSTD_error_memory_allocation = 64,
  ZSTD_error_dstSize_tooSmall = 70,
  ZSTD_error_srcSize_wrong    = 72,
+  /* following error codes are not stable and may be removed or changed in a future version */
  ZSTD_error_frameIndex_tooLarge = 100,
  ZSTD_error_seekableIO          = 102,
-  ZSTD_error_maxCode = 120  /* never EVER use this value directly, it may change in future versions! Use ZSTD_isError() instead */
+  ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;

 /*! ZSTD_getErrorCode() :
@@ -11,6 +11,10 @@
 #ifndef ZSTD_CCOMMON_H_MODULE
 #define ZSTD_CCOMMON_H_MODULE

+/* this module contains definitions which must be identical
+ * across compression, decompression and dictBuilder.
+ * It also contains a few functions useful to at least 2 of them
+ * and which benefit from being inlined */

 /*-*************************************
 *  Dependencies
@@ -50,21 +54,26 @@ extern "C" {

 #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
 #  include <stdio.h>
+extern int g_debuglog_enable;
 /* recommended values for ZSTD_DEBUG display levels :
 * 1 : no display, enables assert() only
- * 2 : reserved for currently active debugging path
- * 3 : events once per object lifetime (CCtx, CDict)
+ * 2 : reserved for currently active debug path
+ * 3 : events once per object lifetime (CCtx, CDict, etc.)
 * 4 : events once per frame
 * 5 : events once per block
 * 6 : events once per sequence (*very* verbose) */
-#  define DEBUGLOG(l, ...) {                         \
-                if (l<=ZSTD_DEBUG) {                 \
-                    fprintf(stderr, __FILE__ ": ");  \
-                    fprintf(stderr, __VA_ARGS__);    \
-                    fprintf(stderr, " \n");          \
+#  define RAWLOG(l, ...) {                                      \
+                if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) {    \
+                    fprintf(stderr, __VA_ARGS__);               \
+            }   }
+#  define DEBUGLOG(l, ...) {                                    \
+                if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) {    \
+                    fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
+                    fprintf(stderr, " \n");                     \
            }   }
 #else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#  define RAWLOG(l, ...)      {}    /* disabled */
+#  define DEBUGLOG(l, ...)    {}    /* disabled */
 #endif


@@ -85,9 +94,7 @@ extern "C" {
 #define ZSTD_OPT_NUM    (1<<12)

 #define ZSTD_REP_NUM      3                 /* number of repcodes */
-#define ZSTD_REP_CHECK    (ZSTD_REP_NUM)    /* number of repcodes to check by the optimal parser */
 #define ZSTD_REP_MOVE     (ZSTD_REP_NUM-1)
-#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
 static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };

 #define KB *(1 <<10)
@@ -102,9 +109,13 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 #define BIT0   1

 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
+#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
 static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
 static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };

+#define ZSTD_FRAMEIDSIZE 4
+static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE;  /* magic number size */
+
 #define ZSTD_BLOCKHEADERSIZE 3   /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
 static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
 typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
@@ -123,34 +134,47 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
 #define MaxLit ((1<<Litbits) - 1)
 #define MaxML  52
 #define MaxLL  35
-#define MaxOff 28
+#define DefaultMaxOff 28
+#define MaxOff 31
 #define MaxSeq MAX(MaxLL, MaxML)   /* Assumption : MaxOff < MaxLL,MaxML */
 #define MLFSELog    9
 #define LLFSELog    9
 #define OffFSELog   8

-static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-                                      1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
+static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
+                                      0, 0, 0, 0, 0, 0, 0, 0,
+                                      1, 1, 1, 1, 2, 2, 3, 3,
+                                      4, 6, 7, 8, 9,10,11,12,
                                     13,14,15,16 };
-static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
-                                             2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
+static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2,
+                                             2, 2, 2, 2, 2, 1, 1, 1,
+                                             2, 2, 2, 2, 2, 2, 2, 2,
+                                             2, 3, 2, 1, 1, 1, 1, 1,
                                            -1,-1,-1,-1 };
 #define LL_DEFAULTNORMLOG 6  /* for static allocation */
 static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;

-static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-                                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-                                      1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
+static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
+                                      0, 0, 0, 0, 0, 0, 0, 0,
+                                      0, 0, 0, 0, 0, 0, 0, 0,
+                                      0, 0, 0, 0, 0, 0, 0, 0,
+                                      1, 1, 1, 1, 2, 2, 3, 3,
+                                      4, 4, 5, 7, 8, 9,10,11,
                                     12,13,14,15,16 };
-static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
-                                             1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-                                             1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
+static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2,
+                                             2, 1, 1, 1, 1, 1, 1, 1,
+                                             1, 1, 1, 1, 1, 1, 1, 1,
+                                             1, 1, 1, 1, 1, 1, 1, 1,
+                                             1, 1, 1, 1, 1, 1, 1, 1,
+                                             1, 1, 1, 1, 1, 1,-1,-1,
                                            -1,-1,-1,-1,-1 };
 #define ML_DEFAULTNORMLOG 6  /* for static allocation */
 static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;

-static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
-                                              1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
+static const S16 OF_defaultNorm[DefaultMaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2,
+                                                     2, 1, 1, 1, 1, 1, 1, 1,
+                                                     1, 1, 1, 1, 1, 1, 1, 1,
+                                                    -1,-1,-1,-1,-1 };
 #define OF_DEFAULTNORMLOG 5  /* for static allocation */
 static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;

@@ -162,7 +186,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
 #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }

 /*! ZSTD_wildcopy() :
-*   custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
+ *  custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
 #define WILDCOPY_OVERLENGTH 8
 MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
 {
@@ -186,17 +210,14 @@ MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd)   /* s


 /*-*******************************************
-*  Private interfaces
+*  Private declarations
 *********************************************/
-typedef struct ZSTD_stats_s ZSTD_stats_t;
-
 typedef struct seqDef_s {
    U32 offset;
    U16 litLength;
    U16 matchLength;
 } seqDef;

-
 typedef struct {
    seqDef* sequencesStart;
    seqDef* sequences;
@@ -211,99 +232,8 @@ typedef struct {
    U32   repToConfirm[ZSTD_REP_NUM];
 } seqStore_t;

-typedef struct {
-    U32 off;
-    U32 len;
-} ZSTD_match_t;
-
-typedef struct {
-    U32 price;
-    U32 off;
-    U32 mlen;
-    U32 litlen;
-    U32 rep[ZSTD_REP_NUM];
-} ZSTD_optimal_t;
-
-typedef struct {
-    U32* litFreq;
-    U32* litLengthFreq;
-    U32* matchLengthFreq;
-    U32* offCodeFreq;
-    ZSTD_match_t* matchTable;
-    ZSTD_optimal_t* priceTable;
-
-    U32  matchLengthSum;
-    U32  matchSum;
-    U32  litLengthSum;
-    U32  litSum;
-    U32  offCodeSum;
-    U32  log2matchLengthSum;
-    U32  log2matchSum;
-    U32  log2litLengthSum;
-    U32  log2litSum;
-    U32  log2offCodeSum;
-    U32  factor;
-    U32  staticPrices;
-    U32  cachedPrice;
-    U32  cachedLitLength;
-    const BYTE* cachedLiterals;
-} optState_t;
-
-typedef struct {
-    U32 offset;
-    U32 checksum;
-} ldmEntry_t;
-
-typedef struct {
-    ldmEntry_t* hashTable;
-    BYTE* bucketOffsets;    /* Next position in bucket to insert entry */
-    U64 hashPower;          /* Used to compute the rolling hash.
-                             * Depends on ldmParams.minMatchLength */
-} ldmState_t;
-
-typedef struct {
-    U32 enableLdm;          /* 1 if enable long distance matching */
-    U32 hashLog;            /* Log size of hashTable */
-    U32 bucketSizeLog;      /* Log bucket size for collision resolution, at most 8 */
-    U32 minMatchLength;     /* Minimum match length */
-    U32 hashEveryLog;       /* Log number of entries to skip */
-} ldmParams_t;
-
-typedef struct {
-    U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
-    FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
-    FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
-    FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
-    U32 workspace[HUF_WORKSPACE_SIZE_U32];
-    HUF_repeat hufCTable_repeatMode;
-    FSE_repeat offcode_repeatMode;
-    FSE_repeat matchlength_repeatMode;
-    FSE_repeat litlength_repeatMode;
-} ZSTD_entropyCTables_t;
-
-struct ZSTD_CCtx_params_s {
-    ZSTD_compressionParameters cParams;
-    ZSTD_frameParameters fParams;
-
-    int compressionLevel;
-    U32 forceWindow;           /* force back-references to respect limit of
-                                * 1<<wLog, even for dictionary */
-
-    /* Multithreading: used to pass parameters to mtctx */
-    U32 nbThreads;
-    unsigned jobSize;
-    unsigned overlapSizeLog;
-
-    /* Long distance matching parameters */
-    ldmParams_t ldmParams;
-
-    /* For use with createCCtxParams() and freeCCtxParams() only */
-    ZSTD_customMem customMem;
-
-};  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
-
-const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
+const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
+void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */

 /* custom memory allocation functions */
 void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
@@ -311,9 +241,7 @@ void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
 void ZSTD_free(void* ptr, ZSTD_customMem customMem);


-/*======  common function  ======*/
-
-MEM_STATIC U32 ZSTD_highbit32(U32 val)
+MEM_STATIC U32 ZSTD_highbit32(U32 val)   /* compress, dictBuilder, decodeCorpus */
 {
    assert(val != 0);
    {
@@ -324,67 +252,26 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
        return 31 - __builtin_clz(val);
 #   else   /* Software version */
-        static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
+        static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
        U32 v = val;
-        int r;
        v |= v >> 1;
        v |= v >> 2;
        v |= v >> 4;
        v |= v >> 8;
        v |= v >> 16;
-        r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
-        return r;
+        return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
 #   endif
    }
 }


-/* hidden functions */
-
 /* ZSTD_invalidateRepCodes() :
 * ensures next compression will not use repcodes from previous block.
 * Note : only works with regular variant;
 *        do not use with extDict variant ! */
-void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);   /* zstdmt, adaptive_compression (shouldn't get this definition from here) */


-/*! ZSTD_initCStream_internal() :
- *  Private use only. Init streaming operation.
- *  expects params to be valid.
- *  must receive dict, or cdict, or none, but not both.
- *  @return : 0, or an error code */
-size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
-                     const void* dict, size_t dictSize,
-                     const ZSTD_CDict* cdict,
-                     ZSTD_CCtx_params  params, unsigned long long pledgedSrcSize);
-
-/*! ZSTD_compressStream_generic() :
- *  Private use only. To be called from zstdmt_compress.c in single-thread mode. */
-size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
-                                   ZSTD_outBuffer* output,
-                                   ZSTD_inBuffer* input,
-                                   ZSTD_EndDirective const flushMode);
-
-/*! ZSTD_getCParamsFromCDict() :
- *  as the name implies */
-ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
-
-/* ZSTD_compressBegin_advanced_internal() :
- * Private use only. To be called from zstdmt_compress.c. */
-size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
-                                    const void* dict, size_t dictSize,
-                                    ZSTD_dictMode_e dictMode,
-                                    ZSTD_CCtx_params params,
-                                    unsigned long long pledgedSrcSize);
-
-/* ZSTD_compress_advanced_internal() :
- * Private use only. To be called from zstdmt_compress.c. */
-size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
-                                       void* dst, size_t dstCapacity,
-                                 const void* src, size_t srcSize,
-                                 const void* dict,size_t dictSize,
-                                 ZSTD_CCtx_params params);
-
 typedef struct {
    blockType_e blockType;
    U32 lastBlock;
@@ -392,7 +279,8 @@ typedef struct {
 } blockProperties_t;

 /*! ZSTD_getcBlockSize() :
-*   Provides the size of compressed block from block header `src` */
+ *  Provides the size of compressed block from block header `src` */
+/* Used by: decompress, fullbench (does not get its definition from here) */
 size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
                          blockProperties_t* bpPtr);

@@ -461,6 +461,7 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
    U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
    U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
    U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+    assert(srcSize > 1); /* Not supported, RLE should be used instead */
    return minBits;
 }

@@ -469,6 +470,7 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
    U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
    U32 tableLog = maxTableLog;
    U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
+    assert(srcSize > 1); /* Not supported, RLE should be used instead */
    if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
    if (maxBitsSrc < tableLog) tableLog = maxBitsSrc;   /* Accuracy can be reduced */
    if (minBits > tableLog) tableLog = minBits;   /* Need a minimum to safely represent all symbol values */
@@ -580,7 +582,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);   /* Unsupported size */
    if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC);   /* Too small tableLog, compression potentially impossible */

-    {   U32 const rtbTable[] = {     0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
+    {   static U32 const rtbTable[] = {     0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
        U64 const scale = 62 - tableLog;
        U64 const step = ((U64)1<<62) / total;   /* <== here, one division ! */
        U64 const vStep = 1ULL<<(scale-20);
@@ -167,7 +167,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
 }


-size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
+size_t HUF_readCTable (HUF_CElt* CTable, U32* maxSymbolValuePtr, const void* src, size_t srcSize)
 {
    BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];   /* init not required, even though some static analyzer may complain */
    U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];   /* large enough for values from 0 to 16 */
@@ -179,7 +179,7 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si

    /* check result */
    if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
-    if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
+    if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall);

    /* Prepare base value per rank */
    {   U32 n, nextRankStart = 0;
@@ -208,9 +208,10 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
                min >>= 1;
        }   }
        /* assign value within rank, symbol order */
-        { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
+        { U32 n; for (n=0; n<nbSymbols; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
    }

+    *maxSymbolValuePtr = nbSymbols - 1;
    return readSize;
 }

@@ -26,7 +26,7 @@
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
-#include "zstd_compress.h"
+#include "zstd_compress_internal.h"
 #include "zstd_fast.h"
 #include "zstd_double_fast.h"
 #include "zstd_lazy.h"
@@ -38,20 +38,7 @@
 *  Helper functions
 ***************************************/
 size_t ZSTD_compressBound(size_t srcSize) {
-    size_t const lowLimit = 256 KB;
-    size_t const margin = (srcSize < lowLimit) ? (lowLimit-srcSize) >> 12 : 0;  /* from 64 to 0 */
-    return srcSize + (srcSize >> 8) + margin;
-}
-
-
-/*-*************************************
-*  Sequence storage
-***************************************/
-static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
-{
-    ssPtr->lit = ssPtr->litStart;
-    ssPtr->sequences = ssPtr->sequencesStart;
-    ssPtr->longLengthID = 0;
+    return ZSTD_COMPRESSBOUND(srcSize);
 }


@@ -80,6 +67,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
    if (!cctx) return NULL;
    cctx->customMem = customMem;
    cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
+    cctx->requestedParams.fParams.contentSizeFlag = 1;
    ZSTD_STATIC_ASSERT(zcss_init==0);
    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
    return cctx;
@@ -154,28 +142,34 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStor
 #define ZSTD_CLEVEL_CUSTOM 999

 static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
-        ZSTD_CCtx_params params, U64 srcSizeHint, size_t dictSize)
+        ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize)
 {
-    return (params.compressionLevel == ZSTD_CLEVEL_CUSTOM ?
-                    params.cParams :
-                    ZSTD_getCParams(params.compressionLevel, srcSizeHint, dictSize));
+    DEBUGLOG(4, "ZSTD_getCParamsFromCCtxParams: srcSize = %u, dictSize = %u",
+                (U32)srcSizeHint, (U32)dictSize);
+    return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ?
+                CCtxParams.cParams :
+                ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize);
 }

-static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* params, U64 srcSize)
+static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize)
 {
-    params->cParams = ZSTD_getCParamsFromCCtxParams(*params, srcSize, 0);
-    params->compressionLevel = ZSTD_CLEVEL_CUSTOM;
+    DEBUGLOG(4, "ZSTD_cLevelToCCtxParams_srcSize: srcSize = %u",
+                (U32)srcSize);
+    CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0);
+    CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM;
 }

 static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
 {
+    DEBUGLOG(4, "ZSTD_cLevelToCParams: level=%i", cctx->requestedParams.compressionLevel);
    ZSTD_cLevelToCCtxParams_srcSize(
            &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1);
 }

-static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* params)
+static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams)
 {
-    ZSTD_cLevelToCCtxParams_srcSize(params, 0);
+    DEBUGLOG(4, "ZSTD_cLevelToCCtxParams");
+    ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, ZSTD_CONTENTSIZE_UNKNOWN);
 }

 static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
@@ -242,6 +236,7 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(
    ZSTD_CCtx_params ret = cctxParams;
    ret.cParams = params.cParams;
    ret.fParams = params.fParams;
+    ret.compressionLevel = ZSTD_CLEVEL_CUSTOM;
    return ret;
 }

@@ -252,12 +247,15 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(

 size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
 {
+    DEBUGLOG(4, "ZSTD_CCtx_setParameter (%u, %u)", (U32)param, value);
    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);

    switch(param)
    {
+    case ZSTD_p_format :
+        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
+
    case ZSTD_p_compressionLevel:
-        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
        if (cctx->cdict) return ERROR(stage_wrong);
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);

@@ -268,9 +266,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
    case ZSTD_p_minMatch:
    case ZSTD_p_targetLength:
    case ZSTD_p_compressionStrategy:
-        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
        if (cctx->cdict) return ERROR(stage_wrong);
-        ZSTD_cLevelToCParams(cctx);  /* Can optimize if srcSize is known */
+        if (value>0) ZSTD_cLevelToCParams(cctx);  /* Can optimize if srcSize is known */
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);

    case ZSTD_p_contentSizeFlag:
@@ -279,15 +276,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);

    case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
-                                   * even when referencing into Dictionary content
+                                   * even when referencing into Dictionary content.
                                   * default : 0 when using a CDict, 1 when using a Prefix */
-        cctx->loadedDictEnd = 0;
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);

    case ZSTD_p_nbThreads:
-        if (value==0) return 0;
-        DEBUGLOG(5, " setting nbThreads : %u", value);
-        if (value > 1 && cctx->staticSize) {
+        if ((value > 1) && cctx->staticSize) {
            return ERROR(parameter_unsupported);  /* MT not compatible with static alloc */
        }
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@@ -296,22 +290,15 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);

    case ZSTD_p_overlapSizeLog:
-        DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->requestedParams.nbThreads);
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);

    case ZSTD_p_enableLongDistanceMatching:
        if (cctx->cdict) return ERROR(stage_wrong);
-        if (value != 0) {
-            ZSTD_cLevelToCParams(cctx);
-        }
+        if (value>0) ZSTD_cLevelToCParams(cctx);
        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);

    case ZSTD_p_ldmHashLog:
    case ZSTD_p_ldmMinMatch:
-        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-        if (cctx->cdict) return ERROR(stage_wrong);
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
    case ZSTD_p_ldmBucketSizeLog:
    case ZSTD_p_ldmHashEveryLog:
        if (cctx->cdict) return ERROR(stage_wrong);
@@ -322,154 +309,167 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 }

 size_t ZSTD_CCtxParam_setParameter(
-        ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value)
+        ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, unsigned value)
 {
+    DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%u, %u)", (U32)param, value);
    switch(param)
    {
+    case ZSTD_p_format :
+        if (value > (unsigned)ZSTD_f_zstd1_magicless)
+            return ERROR(parameter_unsupported);
+        CCtxParams->format = (ZSTD_format_e)value;
+        return (size_t)CCtxParams->format;
+
    case ZSTD_p_compressionLevel :
        if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel();
-        if (value == 0) return 0;
-        params->compressionLevel = value;
-        return 0;
+        if (value)  /* 0 : does not change current level */
+            CCtxParams->compressionLevel = value;
+        return CCtxParams->compressionLevel;

    case ZSTD_p_windowLog :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
-        ZSTD_cLevelToCCtxParams(params);
-        params->cParams.windowLog = value;
-        return 0;
+        DEBUGLOG(4, "ZSTD_CCtxParam_setParameter: set windowLog=%u", value);
+        if (value) {  /* 0 : does not change current windowLog */
+            CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.windowLog = value;
+        }
+        return CCtxParams->cParams.windowLog;

    case ZSTD_p_hashLog :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
-        ZSTD_cLevelToCCtxParams(params);
-        params->cParams.hashLog = value;
-        return 0;
+        if (value) { /* 0 : does not change current hashLog */
+            CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.hashLog = value;
+        }
+        return CCtxParams->cParams.hashLog;

    case ZSTD_p_chainLog :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
-        ZSTD_cLevelToCCtxParams(params);
-        params->cParams.chainLog = value;
-        return 0;
+        if (value) { /* 0 : does not change current chainLog */
+            CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.chainLog = value;
+        }
+        return CCtxParams->cParams.chainLog;

    case ZSTD_p_searchLog :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
-        ZSTD_cLevelToCCtxParams(params);
-        params->cParams.searchLog = value;
-        return 0;
+        if (value) { /* 0 : does not change current searchLog */
+            CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.searchLog = value;
+        }
+        return value;

    case ZSTD_p_minMatch :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
-        ZSTD_cLevelToCCtxParams(params);
-        params->cParams.searchLength = value;
-        return 0;
+        if (value) { /* 0 : does not change current minMatch length */
+            CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.searchLength = value;
+        }
+        return CCtxParams->cParams.searchLength;

    case ZSTD_p_targetLength :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
-        ZSTD_cLevelToCCtxParams(params);
-        params->cParams.targetLength = value;
-        return 0;
+        if (value) { /* 0 : does not change current sufficient_len */
+            CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.targetLength = value;
+        }
+        return CCtxParams->cParams.targetLength;

    case ZSTD_p_compressionStrategy :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
-        ZSTD_cLevelToCCtxParams(params);
-        params->cParams.strategy = (ZSTD_strategy)value;
-        return 0;
+        if (value) { /* 0 : does not change currentstrategy */
+            CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.strategy = (ZSTD_strategy)value;
+        }
+        return (size_t)CCtxParams->cParams.strategy;

    case ZSTD_p_contentSizeFlag :
        /* Content size written in frame header _when known_ (default:1) */
-        DEBUGLOG(5, "set content size flag = %u", (value>0));
-        params->fParams.contentSizeFlag = value > 0;
-        return 0;
+        DEBUGLOG(4, "set content size flag = %u", (value>0));
+        CCtxParams->fParams.contentSizeFlag = value > 0;
+        return CCtxParams->fParams.contentSizeFlag;

    case ZSTD_p_checksumFlag :
        /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
-        params->fParams.checksumFlag = value > 0;
-        return 0;
+        CCtxParams->fParams.checksumFlag = value > 0;
+        return CCtxParams->fParams.checksumFlag;

    case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
-        DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
-        params->fParams.noDictIDFlag = (value == 0);
-        return 0;
+        DEBUGLOG(4, "set dictIDFlag = %u", (value>0));
+        CCtxParams->fParams.noDictIDFlag = (value == 0);
+        return !CCtxParams->fParams.noDictIDFlag;

    case ZSTD_p_forceMaxWindow :
-        params->forceWindow = value > 0;
-        return 0;
+        CCtxParams->forceWindow = (value > 0);
+        return CCtxParams->forceWindow;

    case ZSTD_p_nbThreads :
-        if (value == 0) return 0;
+        if (value == 0) return CCtxParams->nbThreads;
 #ifndef ZSTD_MULTITHREAD
        if (value > 1) return ERROR(parameter_unsupported);
-        return 0;
+        return 1;
 #else
-        return ZSTDMT_initializeCCtxParameters(params, value);
+        return ZSTDMT_CCtxParam_setNbThreads(CCtxParams, value);
 #endif

    case ZSTD_p_jobSize :
 #ifndef ZSTD_MULTITHREAD
        return ERROR(parameter_unsupported);
 #else
-        if (params->nbThreads <= 1) return ERROR(parameter_unsupported);
-        return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_sectionSize, value);
+        if (CCtxParams->nbThreads <= 1) return ERROR(parameter_unsupported);
+        return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value);
 #endif

    case ZSTD_p_overlapSizeLog :
 #ifndef ZSTD_MULTITHREAD
        return ERROR(parameter_unsupported);
 #else
-        if (params->nbThreads <= 1) return ERROR(parameter_unsupported);
-        return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_overlapSectionLog, value);
+        if (CCtxParams->nbThreads <= 1) return ERROR(parameter_unsupported);
+        return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog, value);
 #endif

    case ZSTD_p_enableLongDistanceMatching :
-        if (value != 0) {
-            ZSTD_cLevelToCCtxParams(params);
-            params->cParams.windowLog = ZSTD_LDM_WINDOW_LOG;
+        if (value) {
+            ZSTD_cLevelToCCtxParams(CCtxParams);
+            CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
        }
-        return ZSTD_ldm_initializeParameters(&params->ldmParams, value);
+        return ZSTD_ldm_initializeParameters(&CCtxParams->ldmParams, value);

    case ZSTD_p_ldmHashLog :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
-        params->ldmParams.hashLog = value;
-        return 0;
+        if (value) { /* 0 : does not change current ldmHashLog */
+            CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
+            CCtxParams->ldmParams.hashLog = value;
+        }
+        return CCtxParams->ldmParams.hashLog;

    case ZSTD_p_ldmMinMatch :
-        if (value == 0) return 0;
-        CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX);
-        params->ldmParams.minMatchLength = value;
-        return 0;
+        if (value) { /* 0 : does not change current ldmMinMatch */
+            CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX);
+            CCtxParams->ldmParams.minMatchLength = value;
+        }
+        return CCtxParams->ldmParams.minMatchLength;

    case ZSTD_p_ldmBucketSizeLog :
        if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) {
            return ERROR(parameter_outOfBound);
        }
-        params->ldmParams.bucketSizeLog = value;
-        return 0;
+        CCtxParams->ldmParams.bucketSizeLog = value;
+        return value;

    case ZSTD_p_ldmHashEveryLog :
        if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) {
            return ERROR(parameter_outOfBound);
        }
-        params->ldmParams.hashEveryLog = value;
-        return 0;
+        CCtxParams->ldmParams.hashEveryLog = value;
+        return value;

    default: return ERROR(parameter_unsupported);
    }
 }

-/**
- * This function should be updated whenever ZSTD_CCtx_params is updated.
- * Parameters are copied manually before the dictionary is loaded.
- * The multithreading parameters jobSize and overlapSizeLog are set only if
- * nbThreads > 1.
- *
- * Pledged srcSize is treated as unknown.
+/** ZSTD_CCtx_setParametersUsingCCtxParams() :
+ *  just applies `params` into `cctx`
+ *  no action is performed, parameters are merely stored.
 */
 size_t ZSTD_CCtx_setParametersUsingCCtxParams(
        ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params)
@@ -477,33 +477,14 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
    if (cctx->cdict) return ERROR(stage_wrong);

-    /* Assume the compression and frame parameters are validated */
-    cctx->requestedParams.cParams = params->cParams;
-    cctx->requestedParams.fParams = params->fParams;
-    cctx->requestedParams.compressionLevel = params->compressionLevel;
+    cctx->requestedParams = *params;

-    /* Set force window explicitly since it sets cctx->loadedDictEnd */
-    CHECK_F( ZSTD_CCtx_setParameter(
-                   cctx, ZSTD_p_forceMaxWindow, params->forceWindow) );
-
-    /* Set multithreading parameters explicitly */
-    CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) );
-    if (params->nbThreads > 1) {
-        CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) );
-        CHECK_F( ZSTD_CCtx_setParameter(
-                    cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) );
-    }
-
-    /* Copy long distance matching parameters */
-    cctx->requestedParams.ldmParams = params->ldmParams;
-
-    /* customMem is used only for create/free params and can be ignored */
    return 0;
 }

 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(5, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize);
+    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
    cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
    return 0;
@@ -515,14 +496,14 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
 {
    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
    if (cctx->staticSize) return ERROR(memory_allocation);  /* no malloc for static CCtx */
-    DEBUGLOG(5, "load dictionary of size %u", (U32)dictSize);
+    DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
    ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
    if (dict==NULL || dictSize==0) {   /* no dictionary mode */
        cctx->cdictLocal = NULL;
        cctx->cdict = NULL;
    } else {
        ZSTD_compressionParameters const cParams =
-                ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, 0, dictSize);
+                ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize);
        cctx->cdictLocal = ZSTD_createCDict_advanced(
                                dict, dictSize,
                                dictLoadMethod, dictMode,
@@ -631,29 +612,37 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)

 /** ZSTD_adjustCParams_internal() :
    optimize `cPar` for a given input (`srcSize` and `dictSize`).
-    mostly downsizing to reduce memory consumption and initialization.
-    Both `srcSize` and `dictSize` are optional (use 0 if unknown),
-    but if both are 0, no optimization can be done.
-    Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
+    mostly downsizing to reduce memory consumption and initialization latency.
+    Both `srcSize` and `dictSize` are optional (use 0 if unknown).
+    Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */
 ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
 {
+    static const U64 minSrcSize = 513; /* (1<<9) + 1 */
+    static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
    assert(ZSTD_checkCParams(cPar)==0);
-    if (srcSize+dictSize == 0) return cPar;   /* no size information available : no adjustment */

-    /* resize params, to use less memory when necessary */
-    {   U32 const minSrcSize = (srcSize==0) ? 500 : 0;
-        U64 const rSize = srcSize + dictSize + minSrcSize;
-        if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
-            U32 const srcLog =
-                    MAX(ZSTD_HASHLOG_MIN, (rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1);
-            if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
-    }   }
+    if (dictSize && (srcSize+1<2) /* srcSize unknown */ )
+        srcSize = minSrcSize;  /* presumed small when there is a dictionary */
+    else if (srcSize == 0)
+        srcSize = ZSTD_CONTENTSIZE_UNKNOWN;  /* 0 == unknown : presumed large */
+
+    /* resize windowLog if input is small enough, to use less memory */
+    if ( (srcSize < maxWindowResize)
+      && (dictSize < maxWindowResize) )  {
+        U32 const tSize = (U32)(srcSize + dictSize);
+        static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
+        U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
+                            ZSTD_highbit32(tSize-1) + 1;
+        if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
+    }
    if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
    {   U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
-        if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog);
+        if (cycleLog > cPar.windowLog)
+            cPar.chainLog -= (cycleLog - cPar.windowLog);
    }

-    if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */
+    if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
+        cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */

    return cPar;
 }
@@ -664,7 +653,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize);
 }

-size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params)
+size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
    /* Estimate CCtx size is supported for single-threaded compression only. */
    if (params->nbThreads > 1) { return ERROR(GENERIC); }
@@ -675,7 +664,7 @@ size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* pa
        size_t const maxNbSeq = blockSize / divider;
        size_t const tokenSpace = blockSize + 11*maxNbSeq;
        size_t const chainSize =
-                (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog);
+                (cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams.chainLog);
        size_t const hSize = ((size_t)1) << cParams.hashLog;
        U32    const hashLog3 = (cParams.searchLength>3) ?
                                0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
@@ -701,22 +690,22 @@ size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* pa
    }
 }

-size_t ZSTD_estimateCCtxSize_advanced_usingCParams(ZSTD_compressionParameters cParams)
+size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
 {
    ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams);
-    return ZSTD_estimateCCtxSize_advanced_usingCCtxParams(&params);
+    return ZSTD_estimateCCtxSize_usingCCtxParams(&params);
 }

 size_t ZSTD_estimateCCtxSize(int compressionLevel)
 {
    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
-    return ZSTD_estimateCCtxSize_advanced_usingCParams(cParams);
+    return ZSTD_estimateCCtxSize_usingCParams(cParams);
 }

-size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params)
+size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
    if (params->nbThreads > 1) { return ERROR(GENERIC); }
-    {   size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_usingCCtxParams(params);
+    {   size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog);
        size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize;
        size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
@@ -726,24 +715,21 @@ size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params*
    }
 }

-size_t ZSTD_estimateCStreamSize_advanced_usingCParams(ZSTD_compressionParameters cParams)
+size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams)
 {
    ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams);
-    return ZSTD_estimateCStreamSize_advanced_usingCCtxParams(&params);
+    return ZSTD_estimateCStreamSize_usingCCtxParams(&params);
 }

 size_t ZSTD_estimateCStreamSize(int compressionLevel) {
    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
-    return ZSTD_estimateCStreamSize_advanced_usingCParams(cParams);
+    return ZSTD_estimateCStreamSize_usingCParams(cParams);
 }

 static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1,
                                  ZSTD_compressionParameters cParams2)
 {
-    U32 bslog1 = MIN(cParams1.windowLog, ZSTD_BLOCKSIZELOG_MAX);
-    U32 bslog2 = MIN(cParams2.windowLog, ZSTD_BLOCKSIZELOG_MAX);
-    return (bslog1 == bslog2)   /* same block size */
-         & (cParams1.hashLog  == cParams2.hashLog)
+    return (cParams1.hashLog  == cParams2.hashLog)
         & (cParams1.chainLog == cParams2.chainLog)
         & (cParams1.strategy == cParams2.strategy)   /* opt parser space */
         & ((cParams1.searchLength==3) == (cParams2.searchLength==3));  /* hashlog3 space */
@@ -762,12 +748,38 @@ static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1,
            ldmParams1.hashEveryLog == ldmParams2.hashEveryLog);
 }

+typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
+
+/* ZSTD_sufficientBuff() :
+ * check internal buffers exist for streaming if buffPol == ZSTDb_buffered .
+ * Note : they are assumed to be correctly sized if ZSTD_equivalentCParams()==1 */
+static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t blockSize1,
+                            ZSTD_buffered_policy_e buffPol2,
+                            ZSTD_compressionParameters cParams2,
+                            U64 pledgedSrcSize)
+{
+    size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize));
+    size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2);
+    size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0;
+    DEBUGLOG(4, "ZSTD_sufficientBuff: windowSize2=%u from wlog=%u",
+                (U32)windowSize2, cParams2.windowLog);
+    DEBUGLOG(4, "ZSTD_sufficientBuff: blockSize2 %u <=? blockSize1 %u",
+                (U32)blockSize2, (U32)blockSize1);
+    return (blockSize2 <= blockSize1) /* seqStore space depends on blockSize */
+         & (neededBufferSize2 <= bufferSize1);
+}
+
 /** Equivalence for resetCCtx purposes */
 static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1,
-                                 ZSTD_CCtx_params params2)
+                                 ZSTD_CCtx_params params2,
+                                 size_t buffSize1, size_t blockSize1,
+                                 ZSTD_buffered_policy_e buffPol2,
+                                 U64 pledgedSrcSize)
 {
+    DEBUGLOG(4, "ZSTD_equivalentParams: pledgedSrcSize=%u", (U32)pledgedSrcSize);
    return ZSTD_equivalentCParams(params1.cParams, params2.cParams) &&
-           ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams);
+           ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams) &&
+           ZSTD_sufficientBuff(buffSize1, blockSize1, buffPol2, params2.cParams, pledgedSrcSize);
 }

 /*! ZSTD_continueCCtx() :
@@ -775,13 +787,17 @@ static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1,
 static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize)
 {
    U32 const end = (U32)(cctx->nextSrc - cctx->base);
-    DEBUGLOG(5, "continue mode");
+    size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize));
+    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+    DEBUGLOG(4, "ZSTD_continueCCtx");
+
+    cctx->blockSize = blockSize;   /* previous block size could be different even for same windowLog, due to pledgedSrcSize */
    cctx->appliedParams = params;
    cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
    cctx->consumedSrcSize = 0;
    if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)
        cctx->appliedParams.fParams.contentSizeFlag = 0;
-    DEBUGLOG(5, "pledged content size : %u ; flag : %u",
+    DEBUGLOG(4, "pledged content size : %u ; flag : %u",
        (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag);
    cctx->lowLimit = end;
    cctx->dictLimit = end;
@@ -796,7 +812,6 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl
 }

 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e;
-typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;

 /*! ZSTD_resetCCtx_internal() :
    note : `params` are assumed fully validated at this stage */
@@ -805,11 +820,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                      ZSTD_compResetPolicy_e const crp,
                                      ZSTD_buffered_policy_e const zbuff)
 {
+    DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u",
+                (U32)pledgedSrcSize, params.cParams.windowLog);
    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));

    if (crp == ZSTDcrp_continue) {
-        if (ZSTD_equivalentParams(params, zc->appliedParams)) {
-            DEBUGLOG(5, "ZSTD_equivalentParams()==1");
+        if (ZSTD_equivalentParams(zc->appliedParams, params,
+                                zc->inBuffSize, zc->blockSize,
+                                zbuff, pledgedSrcSize)) {
+            DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%u)",
+                        zc->appliedParams.cParams.windowLog, (U32)zc->blockSize);
            assert(!(params.ldmParams.enableLdm &&
                     params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET));
            zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
@@ -818,6 +838,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
            zc->entropy->litlength_repeatMode = FSE_repeat_none;
            return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
    }   }
+    DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");

    if (params.ldmParams.enableLdm) {
        /* Adjust long distance matching parameters */
@@ -828,19 +849,20 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength);
    }

-    {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
+    {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize));
+        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
        U32    const divider = (params.cParams.searchLength==3) ? 3 : 4;
        size_t const maxNbSeq = blockSize / divider;
        size_t const tokenSpace = blockSize + 11*maxNbSeq;
        size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ?
-                                0 : (1 << params.cParams.chainLog);
+                                0 : ((size_t)1 << params.cParams.chainLog);
        size_t const hSize = ((size_t)1) << params.cParams.hashLog;
        U32    const hashLog3 = (params.cParams.searchLength>3) ?
                                0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
        size_t const h3Size = ((size_t)1) << hashLog3;
        size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
        size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0;
-        size_t const buffInSize = (zbuff==ZSTDb_buffered) ? ((size_t)1 << params.cParams.windowLog) + blockSize : 0;
+        size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0;
        void* ptr;

        /* Check if workSpace is large enough, alloc a new one if needed */
@@ -856,11 +878,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                : 0;
            size_t const neededSpace = entropySpace + optSpace + ldmSpace +
                                       tableSpace + tokenSpace + bufferSpace;
+            DEBUGLOG(4, "Need %uKB workspace, including %uKB for tables, and %uKB for buffers",
+                        (U32)(neededSpace>>10), (U32)(tableSpace>>10), (U32)(bufferSpace>>10));
+            DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u - windowSize: %u - blockSize: %u",
+                        (U32)chainSize, (U32)hSize, (U32)h3Size, (U32)windowSize, (U32)blockSize);

            if (zc->workSpaceSize < neededSpace) {  /* too small : resize */
-                DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n",
-                            (unsigned)zc->workSpaceSize>>10,
-                            (unsigned)neededSpace>>10);
+                DEBUGLOG(4, "Need to update workSpaceSize from %uK to %uK",
+                            (unsigned)(zc->workSpaceSize>>10),
+                            (unsigned)(neededSpace>>10));
                /* static cctx : no resize, error out */
                if (zc->staticSize) return ERROR(memory_allocation);

@@ -883,7 +909,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
        zc->consumedSrcSize = 0;
        if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)
            zc->appliedParams.fParams.contentSizeFlag = 0;
-        DEBUGLOG(5, "pledged content size : %u ; flag : %u",
+        DEBUGLOG(4, "pledged content size : %u ; flag : %u",
            (U32)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag);
        zc->blockSize = blockSize;

@@ -909,7 +935,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,

        /* opt parser space */
        if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) {
-            DEBUGLOG(5, "reserving optimal parser space");
+            DEBUGLOG(4, "reserving optimal parser space");
            assert(((size_t)ptr & 3) == 0);  /* ensure ptr is properly aligned */
            zc->optState.litFreq = (U32*)ptr;
            zc->optState.litLengthFreq = zc->optState.litFreq + (1<<Litbits);
@@ -933,6 +959,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
        }

        /* table Space */
+        DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset);
        if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace);   /* reset tables only */
        assert(((size_t)ptr & 3) == 0);  /* ensure ptr is properly aligned */
        zc->hashTable = (U32*)(ptr);
@@ -981,15 +1008,16 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {

 /*! ZSTD_copyCCtx_internal() :
 *  Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
- *  The "context", in this case, refers to the hash and chain tables, entropy
- *  tables, and dictionary offsets.
 *  Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
- *  pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1
- *  @return : 0, or an error code */
+ *  The "context", in this case, refers to the hash and chain tables,
+ *  entropy tables, and dictionary references.
+ * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx.
+ * @return : 0, or an error code */
 static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
                            const ZSTD_CCtx* srcCCtx,
+                            unsigned windowLog,
                            ZSTD_frameParameters fParams,
-                            unsigned long long pledgedSrcSize,
+                            U64 pledgedSrcSize,
                            ZSTD_buffered_policy_e zbuff)
 {
    DEBUGLOG(5, "ZSTD_copyCCtx_internal");
@@ -999,13 +1027,14 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
    {   ZSTD_CCtx_params params = dstCCtx->requestedParams;
        /* Copy only compression parameters related to tables. */
        params.cParams = srcCCtx->appliedParams.cParams;
+        if (windowLog) params.cParams.windowLog = windowLog;
        params.fParams = fParams;
        ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize,
                                ZSTDcrp_noMemset, zbuff);
    }

    /* copy tables */
-    {   size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->appliedParams.cParams.chainLog);
+    {   size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog);
        size_t const hSize =  (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
        size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
        size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
@@ -1027,6 +1056,12 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,

    /* copy entropy tables */
    memcpy(dstCCtx->entropy, srcCCtx->entropy, sizeof(ZSTD_entropyCTables_t));
+    /* copy repcodes */
+    {
+        int i;
+        for (i = 0; i < ZSTD_REP_NUM; ++i)
+            dstCCtx->seqStore.rep[i] = srcCCtx->seqStore.rep[i];
+    }

    return 0;
 }
@@ -1041,9 +1076,12 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
    ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
    ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0);
    ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
-    fParams.contentSizeFlag = pledgedSrcSize>0;
+    if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+    fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);

-    return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff);
+    return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx,
+                                0 /*windowLog from srcCCtx*/, fParams, pledgedSrcSize,
+                                zbuff);
 }


@@ -1220,7 +1258,7 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,
            ostart[4] = (BYTE)(cLitSize >> 10);
            break;
        }
-    default:   /* not possible : lhSize is {3,4,5} */
+    default:  /* not possible : lhSize is {3,4,5} */
        assert(0);
    }
    return lhSize+cLitSize;
@@ -1229,8 +1267,6 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,

 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
 {
-    BYTE const LL_deltaCode = 19;
-    BYTE const ML_deltaCode = 36;
    const seqDef* const sequences = seqStorePtr->sequencesStart;
    BYTE* const llCodeTable = seqStorePtr->llCode;
    BYTE* const ofCodeTable = seqStorePtr->ofCode;
@@ -1240,9 +1276,9 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
    for (u=0; u<nbSeq; u++) {
        U32 const llv = sequences[u].litLength;
        U32 const mlv = sequences[u].matchLength;
-        llCodeTable[u] = (llv> 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv];
+        llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
        ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset);
-        mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv];
+        mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
    }
    if (seqStorePtr->longLengthID==1)
        llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
@@ -1250,28 +1286,52 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
        mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
 }

-MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType(FSE_repeat* repeatMode,
-        size_t const mostFrequent, size_t nbSeq, U32 defaultNormLog)
+typedef enum {
+    ZSTD_defaultDisallowed = 0,
+    ZSTD_defaultAllowed = 1
+} ZSTD_defaultPolicy_e;
+
+MEM_STATIC
+symbolEncodingType_e ZSTD_selectEncodingType(
+        FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq,
+        U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed)
 {
 #define MIN_SEQ_FOR_DYNAMIC_FSE   64
 #define MAX_SEQ_FOR_STATIC_FSE  1000
-
-    if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
+    ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0);
+    if ((mostFrequent == nbSeq) && (!isDefaultAllowed || nbSeq > 2)) {
+        DEBUGLOG(5, "Selected set_rle");
+        /* Prefer set_basic over set_rle when there are 2 or less symbols,
+         * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol.
+         * If basic encoding isn't possible, always choose RLE.
+         */
        *repeatMode = FSE_repeat_check;
        return set_rle;
    }
-    if ((*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+    if ( isDefaultAllowed
+      && (*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+        DEBUGLOG(5, "Selected set_repeat");
        return set_repeat;
    }
-    if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1)))) {
-        *repeatMode = FSE_repeat_valid;
+    if ( isDefaultAllowed
+      && ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1)))) ) {
+        DEBUGLOG(5, "Selected set_basic");
+        /* The format allows default tables to be repeated, but it isn't useful.
+         * When using simple heuristics to select encoding type, we don't want
+         * to confuse these tables with dictionaries. When running more careful
+         * analysis, we don't need to waste time checking both repeating tables
+         * and default tables.
+         */
+        *repeatMode = FSE_repeat_none;
        return set_basic;
    }
+    DEBUGLOG(5, "Selected set_compressed");
    *repeatMode = FSE_repeat_check;
    return set_compressed;
 }

-MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
+MEM_STATIC
+size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
        FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type,
        U32* count, U32 max,
        BYTE const* codeTable, size_t nbSeq,
@@ -1289,7 +1349,7 @@ MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
    case set_repeat:
        return 0;
    case set_basic:
-        CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize));
+        CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize));  /* note : could be pre-calculated */
        return 0;
    case set_compressed: {
        S16 norm[MaxSeq + 1];
@@ -1299,6 +1359,7 @@ MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
            count[codeTable[nbSeq-1]]--;
            nbSeq_1--;
        }
+        assert(nbSeq_1 > 1);
        CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
        {   size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog);   /* overflow protected */
            if (FSE_isError(NCountSize)) return NCountSize;
@@ -1310,11 +1371,13 @@ MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
    }
 }

-MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity,
-    FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
-    FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
-    FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
-    seqDef const* sequences, size_t nbSeq, int longOffsets)
+MEM_STATIC
+size_t ZSTD_encodeSequences(
+            void* dst, size_t dstCapacity,
+            FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+            FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+            FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+            seqDef const* sequences, size_t nbSeq, int longOffsets)
 {
    BIT_CStream_t blockStream;
    FSE_CState_t  stateMatchLength;
@@ -1351,8 +1414,12 @@ MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity,
            BYTE const ofCode = ofCodeTable[n];
            BYTE const mlCode = mlCodeTable[n];
            U32  const llBits = LL_bits[llCode];
-            U32  const ofBits = ofCode;                                     /* 32b*/  /* 64b*/
+            U32  const ofBits = ofCode;
            U32  const mlBits = ML_bits[mlCode];
+            DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
+                        sequences[n].litLength,
+                        sequences[n].matchLength + MINMATCH,
+                        sequences[n].offset);                               /* 32b*/  /* 64b*/
                                                                            /* (7)*/  /* (7)*/
            FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode);       /* 15 */  /* 15 */
            FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode);      /* 24 */  /* 24 */
@@ -1418,14 +1485,18 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                entropy, cParams->strategy, op, dstCapacity, literals, litSize);
        if (ZSTD_isError(cSize))
          return cSize;
+        assert(cSize <= dstCapacity);
        op += cSize;
    }

    /* Sequences Header */
-    if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall);
-    if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq;
-    else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
-    else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
+    if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/) return ERROR(dstSize_tooSmall);
+    if (nbSeq < 0x7F)
+        *op++ = (BYTE)nbSeq;
+    else if (nbSeq < LONGNBSEQ)
+        op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
+    else
+        op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
    if (nbSeq==0) return op - ostart;

    /* seqHead : flags for FSE encoding type */
@@ -1433,30 +1504,35 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,

    /* convert length/distances into codes */
    ZSTD_seqToCodes(seqStorePtr);
-    /* CTable for Literal Lengths */
+    /* build CTable for Literal Lengths */
    {   U32 max = MaxLL;
        size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace);
-        LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog);
+        DEBUGLOG(5, "Building LL table");
+        LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed);
        {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
                    count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
                    entropy->workspace, sizeof(entropy->workspace));
            if (ZSTD_isError(countSize)) return countSize;
            op += countSize;
    }   }
-    /* CTable for Offsets */
+    /* build CTable for Offsets */
    {   U32 max = MaxOff;
        size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace);
-        Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog);
+        /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
+        ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
+        DEBUGLOG(5, "Building OF table");
+        Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy);
        {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
-                    count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, MaxOff,
+                    count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
                    entropy->workspace, sizeof(entropy->workspace));
            if (ZSTD_isError(countSize)) return countSize;
            op += countSize;
    }   }
-    /* CTable for MatchLengths */
+    /* build CTable for MatchLengths */
    {   U32 max = MaxML;
        size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace);
-        MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog);
+        DEBUGLOG(5, "Building ML table");
+        MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed);
        {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
                    count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
                    entropy->workspace, sizeof(entropy->workspace));
@@ -1466,13 +1542,15 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,

    *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));

-    {   size_t const streamSize = ZSTD_encodeSequences(op, oend - op,
-                CTable_MatchLength, mlCodeTable,
-                CTable_OffsetBits, ofCodeTable,
-                CTable_LitLength, llCodeTable,
-                sequences, nbSeq, longOffsets);
-        if (ZSTD_isError(streamSize)) return streamSize;
-        op += streamSize;
+    {   size_t const bitstreamSize = ZSTD_encodeSequences(
+                                        op, oend - op,
+                                        CTable_MatchLength, mlCodeTable,
+                                        CTable_OffsetBits, ofCodeTable,
+                                        CTable_LitLength, llCodeTable,
+                                        sequences, nbSeq,
+                                        longOffsets);
+        if (ZSTD_isError(bitstreamSize)) return bitstreamSize;
+        op += bitstreamSize;
    }

    return op - ostart;
@@ -1486,27 +1564,33 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
 {
    size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams,
                                                         dst, dstCapacity);
-    size_t const minGain = ZSTD_minGain(srcSize);
-    size_t const maxCSize = srcSize - minGain;
    /* If the srcSize <= dstCapacity, then there is enough space to write a
     * raw uncompressed block. Since we ran out of space, the block must not
     * be compressible, so fall back to a raw uncompressed block.
     */
-    int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
-
+    int const uncompressibleError = (cSize == ERROR(dstSize_tooSmall)) && (srcSize <= dstCapacity);
    if (ZSTD_isError(cSize) && !uncompressibleError)
        return cSize;
+    /* We check that dictionaries have offset codes available for the first
+     * block. After the first block, the offcode table might not have large
+     * enough codes to represent the offsets in the data.
+     */
+    if (entropy->offcode_repeatMode == FSE_repeat_valid)
+        entropy->offcode_repeatMode = FSE_repeat_check;
+
    /* Check compressibility */
-    if (cSize >= maxCSize || uncompressibleError) {
-        entropy->hufCTable_repeatMode = HUF_repeat_none;
-        entropy->offcode_repeatMode = FSE_repeat_none;
-        entropy->matchlength_repeatMode = FSE_repeat_none;
-        entropy->litlength_repeatMode = FSE_repeat_none;
-        return 0;
-    }
+    {   size_t const minGain = ZSTD_minGain(srcSize);  /* note : fixed formula, maybe should depend on compression level, or strategy */
+        size_t const maxCSize = srcSize - minGain;
+        if (cSize >= maxCSize || uncompressibleError) {
+            entropy->hufCTable_repeatMode = HUF_repeat_none;
+            entropy->offcode_repeatMode = FSE_repeat_none;
+            entropy->matchlength_repeatMode = FSE_repeat_none;
+            entropy->litlength_repeatMode = FSE_repeat_none;
+            return 0;  /* block not compressed */
+    }   }
    assert(!ZSTD_isError(cSize));

-    /* confirm repcodes */
+    /* block is compressed => confirm repcodes in history */
    { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; }
    return cSize;
 }
@@ -1528,9 +1612,9 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict
          ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict }
    };
    ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
+
    assert((U32)strat >= (U32)ZSTD_fast);
    assert((U32)strat <= (U32)ZSTD_btultra);
-
    return blockCompressor[extDict!=0][(U32)strat];
 }

@@ -1541,30 +1625,38 @@ static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr,
    seqStorePtr->lit += lastLLSize;
 }

+static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
+{
+    ssPtr->lit = ssPtr->litStart;
+    ssPtr->sequences = ssPtr->sequencesStart;
+    ssPtr->longLengthID = 0;
+}
+
 static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    const BYTE* const base = zc->base;
-    const BYTE* const istart = (const BYTE*)src;
-    const U32 current = (U32)(istart-base);
-    size_t lastLLSize;
-    const BYTE* anchor;
-    U32 const extDict = zc->lowLimit < zc->dictLimit;
-    const ZSTD_blockCompressor blockCompressor =
-        zc->appliedParams.ldmParams.enableLdm
-            ? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm)
-            : ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict);
-
-    if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0;   /* don't even attempt compression below a certain srcSize */
+    DEBUGLOG(5, "ZSTD_compressBlock_internal : dstCapacity = %u", (U32)dstCapacity);
+    if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1)
+        return 0;   /* don't even attempt compression below a certain srcSize */
    ZSTD_resetSeqStore(&(zc->seqStore));
-    if (current > zc->nextToUpdate + 384)
-        zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));   /* limited update after finding a very long match */
-
-    lastLLSize = blockCompressor(zc, src, srcSize);
-
-    /* Last literals */
-    anchor = (const BYTE*)src + srcSize - lastLLSize;
-    ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize);

+    /* limited update after a very long match */
+    {   const BYTE* const base = zc->base;
+        const BYTE* const istart = (const BYTE*)src;
+        const U32 current = (U32)(istart-base);
+        if (current > zc->nextToUpdate + 384)
+            zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));
+    }
+    /* find and store sequences */
+    {   U32 const extDict = zc->lowLimit < zc->dictLimit;
+        const ZSTD_blockCompressor blockCompressor =
+            zc->appliedParams.ldmParams.enableLdm
+                ? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm)
+                : ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict);
+        size_t const lastLLSize = blockCompressor(zc, src, srcSize);
+        const BYTE* const anchor = (const BYTE*)src + srcSize - lastLLSize;
+        ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize);
+    }
+    /* encode */
    return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize);
 }

@@ -1586,26 +1678,47 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
    const BYTE* ip = (const BYTE*)src;
    BYTE* const ostart = (BYTE*)dst;
    BYTE* op = ostart;
-    U32 const maxDist = 1 << cctx->appliedParams.cParams.windowLog;
+    U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
+    assert(cctx->appliedParams.cParams.windowLog <= 31);

+    DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (U32)blockSize);
    if (cctx->appliedParams.fParams.checksumFlag && srcSize)
        XXH64_update(&cctx->xxhState, src, srcSize);

    while (remaining) {
        U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
-        size_t cSize;

        if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
            return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
        if (remaining < blockSize) blockSize = remaining;

-        /* preemptive overflow correction */
+        /* preemptive overflow correction:
+         * 1. correction is large enough:
+         *    lowLimit > (3<<29) ==> current > 3<<29 + 1<<windowLog - blockSize
+         *    1<<windowLog <= newCurrent < 1<<chainLog + 1<<windowLog
+         *
+         *    current - newCurrent
+         *    > (3<<29 + 1<<windowLog - blockSize) - (1<<windowLog + 1<<chainLog)
+         *    > (3<<29 - blockSize) - (1<<chainLog)
+         *    > (3<<29 - blockSize) - (1<<30)             (NOTE: chainLog <= 30)
+         *    > 1<<29 - 1<<17
+         *
+         * 2. (ip+blockSize - cctx->base) doesn't overflow:
+         *    In 32 bit mode we limit windowLog to 30 so we don't get
+         *    differences larger than 1<<31-1.
+         * 3. cctx->lowLimit < 1<<32:
+         *    windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
+         */
        if (cctx->lowLimit > (3U<<29)) {
-            U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->appliedParams.cParams.hashLog, cctx->appliedParams.cParams.strategy)) - 1;
+            U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
            U32 const current = (U32)(ip - cctx->base);
-            U32 const newCurrent = (current & cycleMask) + (1 << cctx->appliedParams.cParams.windowLog);
+            U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog);
            U32 const correction = current - newCurrent;
-            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
+            ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+            assert(current > newCurrent);
+            assert(correction > 1<<28); /* Loose bound, should be about 1<<29 */
            ZSTD_reduceIndex(cctx, correction);
            cctx->base += correction;
            cctx->dictBase += correction;
@@ -1613,35 +1726,41 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
            cctx->dictLimit -= correction;
            if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
            else cctx->nextToUpdate -= correction;
+            DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x\n", correction, cctx->lowLimit);
        }
-
+        /* enforce maxDist */
        if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
-            /* enforce maxDist */
            U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
            if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
            if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
        }

-        cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
-        if (ZSTD_isError(cSize)) return cSize;
+        {   size_t cSize = ZSTD_compressBlock_internal(cctx,
+                                op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
+                                ip, blockSize);
+            if (ZSTD_isError(cSize)) return cSize;

-        if (cSize == 0) {  /* block is not compressible */
-            U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3);
-            if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
-            MEM_writeLE32(op, cBlockHeader24);   /* no pb, 4th byte will be overwritten */
-            memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
-            cSize = ZSTD_blockHeaderSize+blockSize;
-        } else {
-            U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
-            MEM_writeLE24(op, cBlockHeader24);
-            cSize += ZSTD_blockHeaderSize;
-        }
+            if (cSize == 0) {  /* block is not compressible */
+                U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3);
+                if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
+                MEM_writeLE32(op, cBlockHeader24);   /* 4th byte will be overwritten */
+                memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
+                cSize = ZSTD_blockHeaderSize + blockSize;
+            } else {
+                U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
+                MEM_writeLE24(op, cBlockHeader24);
+                cSize += ZSTD_blockHeaderSize;
+            }

-        remaining -= blockSize;
-        dstCapacity -= cSize;
-        ip += blockSize;
-        op += cSize;
-    }
+            ip += blockSize;
+            assert(remaining >= blockSize);
+            remaining -= blockSize;
+            op += cSize;
+            assert(dstCapacity >= cSize);
+            dstCapacity -= cSize;
+            DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u",
+                        (U32)cSize);
+    }   }

    if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
    return op-ostart;
@@ -1654,20 +1773,23 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
    U32   const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536);   /* 0-3 */
    U32   const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength;   /* 0-3 */
    U32   const checksumFlag = params.fParams.checksumFlag>0;
-    U32   const windowSize = 1U << params.cParams.windowLog;
+    U32   const windowSize = (U32)1 << params.cParams.windowLog;
    U32   const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
    BYTE  const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
    U32   const fcsCode = params.fParams.contentSizeFlag ?
                     (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0;  /* 0-3 */
    BYTE  const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
-    size_t pos;
+    size_t pos=0;

    if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
-    DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
+    DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
                !params.fParams.noDictIDFlag, dictID,  dictIDSizeCode);

-    MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
-    op[4] = frameHeaderDecriptionByte; pos=5;
+    if (params.format == ZSTD_f_zstd1) {
+        MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
+        pos = 4;
+    }
+    op[pos++] = frameHeaderDecriptionByte;
    if (!singleSegment) op[pos++] = windowLogByte;
    switch(dictIDSizeCode)
    {
@@ -1697,8 +1819,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
    const BYTE* const ip = (const BYTE*) src;
    size_t fhSize = 0;

-    DEBUGLOG(5, "ZSTD_compressContinue_internal");
-    DEBUGLOG(5, "stage: %u", cctx->stage);
+    DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u", cctx->stage);
    if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong);   /* missing init (ZSTD_compressBegin) */

    if (frame && (cctx->stage==ZSTDcs_init)) {
@@ -1710,17 +1831,21 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
        cctx->stage = ZSTDcs_ongoing;
    }

+    if (!srcSize) return fhSize;  /* do not generate an empty block if no input */
+
    /* Check if blocks follow each other */
    if (src != cctx->nextSrc) {
        /* not contiguous */
-        ptrdiff_t const delta = cctx->nextSrc - ip;
+        size_t const distanceFromBase = (size_t)(cctx->nextSrc - cctx->base);
        cctx->lowLimit = cctx->dictLimit;
-        cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
+        assert(distanceFromBase == (size_t)(U32)distanceFromBase);  /* should never overflow */
+        cctx->dictLimit = (U32)distanceFromBase;
        cctx->dictBase = cctx->base;
-        cctx->base -= delta;
+        cctx->base = ip - distanceFromBase;
        cctx->nextToUpdate = cctx->dictLimit;
        if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit;   /* too small extDict */
    }
+    cctx->nextSrc = ip + srcSize;

    /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
    if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
@@ -1729,17 +1854,14 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
        cctx->lowLimit = lowLimitMax;
    }

-    cctx->nextSrc = ip + srcSize;
-
-    if (srcSize) {
-        size_t const cSize = frame ?
+    DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize);
+    {   size_t const cSize = frame ?
                             ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
                             ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
        if (ZSTD_isError(cSize)) return cSize;
        cctx->consumedSrcSize += srcSize;
        return cSize + fhSize;
-    } else
-        return fhSize;
+    }
 }

 size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
@@ -1754,7 +1876,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
    ZSTD_compressionParameters const cParams =
            ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0);
-    return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
+    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
 }

 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -1776,7 +1898,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
    zc->lowLimit = zc->dictLimit;
    zc->dictLimit = (U32)(zc->nextSrc - zc->base);
    zc->dictBase = zc->base;
-    zc->base += ip - zc->nextSrc;
+    zc->base = ip - zc->dictLimit;
    zc->nextToUpdate = zc->dictLimit;
    zc->loadedDictEnd = zc->appliedParams.forceWindow ? 0 : (U32)(iend - zc->base);

@@ -1803,7 +1925,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
    case ZSTD_btopt:
    case ZSTD_btultra:
        if (srcSize >= HASH_READ_SIZE)
-            ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
+            ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
        break;

    default:
@@ -1851,8 +1973,10 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
    cctx->dictID = cctx->appliedParams.fParams.noDictIDFlag ? 0 :  MEM_readLE32(dictPtr);
    dictPtr += 4;

-    {   size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)cctx->entropy->hufCTable, 255, dictPtr, dictEnd-dictPtr);
+    {   unsigned maxSymbolValue = 255;
+        size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)cctx->entropy->hufCTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr);
        if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
+        if (maxSymbolValue < 255) return ERROR(dictionary_corrupted);
        dictPtr += hufHeaderSize;
    }

@@ -1925,7 +2049,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
                                       const void* dict, size_t dictSize,
                                             ZSTD_dictMode_e dictMode)
 {
-    DEBUGLOG(5, "ZSTD_compress_insertDictionary");
+    DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
    if ((dict==NULL) || (dictSize<=8)) return 0;

    /* dict restricted modes */
@@ -1934,7 +2058,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,

    if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
        if (dictMode == ZSTD_dm_auto) {
-            DEBUGLOG(5, "raw content dictionary detected");
+            DEBUGLOG(4, "raw content dictionary detected");
            return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
        }
        if (dictMode == ZSTD_dm_fullDict)
@@ -1948,23 +2072,22 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,

 /*! ZSTD_compressBegin_internal() :
 * @return : 0, or an error code */
-static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
+size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                             const void* dict, size_t dictSize,
                             ZSTD_dictMode_e dictMode,
                             const ZSTD_CDict* cdict,
-                                   ZSTD_CCtx_params params, U64 pledgedSrcSize,
-                                   ZSTD_buffered_policy_e zbuff)
+                             ZSTD_CCtx_params params, U64 pledgedSrcSize,
+                             ZSTD_buffered_policy_e zbuff)
 {
-    DEBUGLOG(4, "ZSTD_compressBegin_internal");
-    DEBUGLOG(4, "dict ? %s", dict ? "dict" : (cdict ? "cdict" : "none"));
-    DEBUGLOG(4, "dictMode : %u", (U32)dictMode);
+    DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params.cParams.windowLog);
    /* params are supposed to be fully validated at this point */
    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */

    if (cdict && cdict->dictContentSize>0) {
+        cctx->requestedParams = params;
        return ZSTD_copyCCtx_internal(cctx, cdict->refContext,
-                                      params.fParams, pledgedSrcSize,
+                                      params.cParams.windowLog, params.fParams, pledgedSrcSize,
                                      zbuff);
    }

@@ -1973,16 +2096,19 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
    return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode);
 }

-size_t ZSTD_compressBegin_advanced_internal(
-                                    ZSTD_CCtx* cctx,
+size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
                                    const void* dict, size_t dictSize,
                                    ZSTD_dictMode_e dictMode,
+                                    const ZSTD_CDict* cdict,
                                    ZSTD_CCtx_params params,
                                    unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog);
    /* compression parameters verification and optimization */
    CHECK_F( ZSTD_checkCParams(params.cParams) );
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
+    return ZSTD_compressBegin_internal(cctx,
+                                       dict, dictSize, dictMode,
+                                       cdict,
                                       params, pledgedSrcSize,
                                       ZSTDb_not_buffered);
 }
@@ -1995,9 +2121,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 {
    ZSTD_CCtx_params const cctxParams =
            ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
-    return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dm_auto,
-                                                cctxParams,
-                                                pledgedSrcSize);
+    return ZSTD_compressBegin_advanced_internal(cctx,
+                                            dict, dictSize, ZSTD_dm_auto,
+                                            NULL /*cdict*/,
+                                            cctxParams, pledgedSrcSize);
 }

 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
@@ -2005,8 +2132,9 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
    ZSTD_CCtx_params const cctxParams =
            ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
+    DEBUGLOG(4, "ZSTD_compressBegin_usingDict");
    return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
-                                       cctxParams, 0, ZSTDb_not_buffered);
+                                       cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
 }

 size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
@@ -2069,9 +2197,9 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
    endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
    if (ZSTD_isError(endResult)) return endResult;
    if (cctx->appliedParams.fParams.contentSizeFlag) {  /* control src size */
-        DEBUGLOG(5, "end of frame : controlling src size");
+        DEBUGLOG(4, "end of frame : controlling src size");
        if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) {
-            DEBUGLOG(5, "error : pledgedSrcSize = %u, while realSrcSize = %u",
+            DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize = %u",
                (U32)cctx->pledgedSrcSizePlusOne-1, (U32)cctx->consumedSrcSize);
            return ERROR(srcSize_wrong);
    }   }
@@ -2087,6 +2215,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
 {
    ZSTD_CCtx_params const cctxParams =
            ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
+    DEBUGLOG(4, "ZSTD_compress_internal");
    return ZSTD_compress_advanced_internal(cctx,
                                          dst, dstCapacity,
                                          src, srcSize,
@@ -2100,6 +2229,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
                         const void* dict,size_t dictSize,
                               ZSTD_parameters params)
 {
+    DEBUGLOG(4, "ZSTD_compress_advanced");
    CHECK_F(ZSTD_checkCParams(params.cParams));
    return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
 }
@@ -2112,6 +2242,7 @@ size_t ZSTD_compress_advanced_internal(
        const void* dict,size_t dictSize,
        ZSTD_CCtx_params params)
 {
+    DEBUGLOG(4, "ZSTD_compress_advanced_internal");
    CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
                                         params, srcSize, ZSTDb_not_buffered) );
    return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
@@ -2120,8 +2251,10 @@ size_t ZSTD_compress_advanced_internal(
 size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize,
                               const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0);
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize ? srcSize : 1, dict ? dictSize : 0);
    params.fParams.contentSizeFlag = 1;
+    DEBUGLOG(4, "ZSTD_compress_usingDict (level=%i, srcSize=%u, dictSize=%u)",
+                compressionLevel, (U32)srcSize, (U32)dictSize);
    return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
 }

@@ -2152,8 +2285,8 @@ size_t ZSTD_estimateCDictSize_advanced(
 {
    DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict));
    DEBUGLOG(5, "CCtx estimate : %u",
-             (U32)ZSTD_estimateCCtxSize_advanced_usingCParams(cParams));
-    return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced_usingCParams(cParams)
+             (U32)ZSTD_estimateCCtxSize_usingCParams(cParams));
+    return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_usingCParams(cParams)
           + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
 }

@@ -2178,7 +2311,7 @@ static size_t ZSTD_initCDict_internal(
                    ZSTD_dictMode_e dictMode,
                    ZSTD_compressionParameters cParams)
 {
-    DEBUGLOG(5, "ZSTD_initCDict_internal, mode %u", (U32)dictMode);
+    DEBUGLOG(3, "ZSTD_initCDict_internal, mode %u", (U32)dictMode);
    if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
        cdict->dictBuffer = NULL;
        cdict->dictContent = dictBuffer;
@@ -2208,7 +2341,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
                                      ZSTD_dictMode_e dictMode,
                                      ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
-    DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
+    DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;

    {   ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@@ -2278,14 +2411,14 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
                                 ZSTD_dictMode_e dictMode,
                                 ZSTD_compressionParameters cParams)
 {
-    size_t const cctxSize = ZSTD_estimateCCtxSize_advanced_usingCParams(cParams);
+    size_t const cctxSize = ZSTD_estimateCCtxSize_usingCParams(cParams);
    size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize)
                            + cctxSize;
    ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
    void* ptr;
-    DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
+    DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
-    DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u",
+    DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
        (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
    if (workspaceSize < neededSize) return NULL;

@@ -2317,11 +2450,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
    ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
    ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
+    DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
    if (cdict==NULL) return ERROR(dictionary_wrong);
    {   ZSTD_CCtx_params params = cctx->requestedParams;
        params.cParams = ZSTD_getCParamsFromCDict(cdict);
        params.fParams = fParams;
-        DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
        return ZSTD_compressBegin_internal(cctx,
                                           NULL, 0, ZSTD_dm_auto,
                                           cdict,
@@ -2336,7 +2469,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
    ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
-    DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
+    DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
    return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
 }

@@ -2371,6 +2504,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,

 ZSTD_CStream* ZSTD_createCStream(void)
 {
+    DEBUGLOG(3, "ZSTD_createCStream");
    return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
 }

@@ -2401,9 +2535,9 @@ size_t ZSTD_CStreamOutSize(void)
 }

 static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
-                    const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
-                    const ZSTD_CDict* cdict,
-                    const ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
+                    const void* const dict, size_t const dictSize, ZSTD_dictMode_e const dictMode,
+                    const ZSTD_CDict* const cdict,
+                    ZSTD_CCtx_params const params, unsigned long long const pledgedSrcSize)
 {
    DEBUGLOG(4, "ZSTD_resetCStream_internal");
    /* params are supposed to be fully validated at this point */
@@ -2411,42 +2545,47 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */

    CHECK_F( ZSTD_compressBegin_internal(zcs,
-                                        dict, dictSize, dictMode,
-                                        cdict,
-                                        params, pledgedSrcSize,
-                                        ZSTDb_buffered) );
+                                         dict, dictSize, dictMode,
+                                         cdict,
+                                         params, pledgedSrcSize,
+                                         ZSTDb_buffered) );

    zcs->inToCompress = 0;
    zcs->inBuffPos = 0;
-    zcs->inBuffTarget = zcs->blockSize;
+    zcs->inBuffTarget = zcs->blockSize
+                      + (zcs->blockSize == pledgedSrcSize);   /* for small input: avoid automatic flush on reaching end of block, since it would require to add a 3-bytes null block to end frame */
    zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
    zcs->streamStage = zcss_load;
    zcs->frameEnded = 0;
    return 0;   /* ready to go */
 }

+/* ZSTD_resetCStream():
+ * pledgedSrcSize == 0 means "unknown" */
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
    ZSTD_CCtx_params params = zcs->requestedParams;
-    params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+    DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize);
+    if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+    params.fParams.contentSizeFlag = 1;
    params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0);
-    DEBUGLOG(5, "ZSTD_resetCStream");
    return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
 }

 /*! ZSTD_initCStream_internal() :
- *  Note : not static, but hidden (not exposed). Used by zstdmt_compress.c
+ *  Note : for lib/compress only. Used by zstdmt_compress.c.
 *  Assumption 1 : params are valid
 *  Assumption 2 : either dict, or cdict, is defined, not both */
 size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
                    const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
                    ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(4, "ZSTD_initCStream_internal");
    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */

    if (dict && dictSize >= 8) {
-        DEBUGLOG(5, "loading dictionary of size %u", (U32)dictSize);
+        DEBUGLOG(4, "loading dictionary of size %u", (U32)dictSize);
        if (zcs->staticSize) {   /* static CCtx : never uses malloc */
            /* incompatible with internal cdict creation */
            return ERROR(memory_allocation);
@@ -2459,14 +2598,14 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
    } else {
        if (cdict) {
-            params.cParams = ZSTD_getCParamsFromCDict(cdict);  /* cParams are enforced from cdict */
+            params.cParams = ZSTD_getCParamsFromCDict(cdict);  /* cParams are enforced from cdict; it includes windowLog */
        }
        ZSTD_freeCDict(zcs->cdictLocal);
        zcs->cdictLocal = NULL;
        zcs->cdict = cdict;
    }

-    params.compressionLevel = ZSTD_CLEVEL_CUSTOM;
+    params.compressionLevel = ZSTD_CLEVEL_CUSTOM; /* enforce usage of cParams, instead of a dynamic derivation from cLevel (but does that happen ?) */
    zcs->requestedParams = params;

    return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
@@ -2478,8 +2617,9 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                                            const ZSTD_CDict* cdict,
                                            ZSTD_frameParameters fParams,
                                            unsigned long long pledgedSrcSize)
-{   /* cannot handle NULL cdict (does not know what to do) */
-    if (!cdict) return ERROR(dictionary_wrong);
+{
+    DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced");
+    if (!cdict) return ERROR(dictionary_wrong); /* cannot handle NULL cdict (does not know what to do) */
    {   ZSTD_CCtx_params params = zcs->requestedParams;
        params.cParams = ZSTD_getCParamsFromCDict(cdict);
        params.fParams = fParams;
@@ -2492,18 +2632,25 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
 /* note : cdict must outlive compression session */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 {
-    ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ };
-    return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0);  /* note : will check that cdict != NULL */
+    ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ };
+    DEBUGLOG(4, "ZSTD_initCStream_usingCDict");
+    return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);  /* note : will check that cdict != NULL */
 }

+/* ZSTD_initCStream_advanced() :
+ * pledgedSrcSize must be correct.
+ * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                 const void* dict, size_t dictSize,
                                 ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    ZSTD_CCtx_params const cctxParams =
-            ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
+    ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
+    DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
+                (U32)pledgedSrcSize, params.fParams.contentSizeFlag);
    CHECK_F( ZSTD_checkCParams(params.cParams) );
-    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize);
+    if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;  /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */
+    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, cctxParams, pledgedSrcSize);
 }

 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
@@ -2511,21 +2658,21 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
    ZSTD_CCtx_params const cctxParams =
            ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
-    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0);
+    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
 }

-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss)
 {
-    ZSTD_CCtx_params cctxParams;
+    U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;  /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */
    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
-    cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
-    cctxParams.fParams.contentSizeFlag = (pledgedSrcSize>0);
+    ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
    return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, cctxParams, pledgedSrcSize);
 }

 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 {
-    return ZSTD_initCStream_srcSize(zcs, compressionLevel, 0);
+    DEBUGLOG(4, "ZSTD_initCStream");
+    return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN);
 }

 /*======   Compression   ======*/
@@ -2540,6 +2687,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,

 /** ZSTD_compressStream_generic():
 *  internal function for all *compressStream*() variants and *compress_generic()
+ *  non-static, because can be called from zstdmt.c
 * @return : hint size for next input */
 size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                                   ZSTD_outBuffer* output,
@@ -2557,9 +2705,9 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
    /* check expectations */
    DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (U32)flushMode);
    assert(zcs->inBuff != NULL);
-    assert(zcs->inBuffSize>0);
-    assert(zcs->outBuff!= NULL);
-    assert(zcs->outBuffSize>0);
+    assert(zcs->inBuffSize > 0);
+    assert(zcs->outBuff !=  NULL);
+    assert(zcs->outBuffSize > 0);
    assert(output->pos <= output->size);
    assert(input->pos <= input->size);

@@ -2699,6 +2847,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
                              ZSTD_inBuffer* input,
                              ZSTD_EndDirective endOp)
 {
+    DEBUGLOG(5, "ZSTD_compress_generic, endOp=%u ", (U32)endOp);
    /* check conditions */
    if (output->pos > output->size) return ERROR(GENERIC);
    if (input->pos  > input->size)  return ERROR(GENERIC);
@@ -2706,41 +2855,47 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,

    /* transparent initialization stage */
    if (cctx->streamStage == zcss_init) {
-        ZSTD_prefixDict const prefixDict = cctx->prefixDict;
        ZSTD_CCtx_params params = cctx->requestedParams;
-        params.cParams = ZSTD_getCParamsFromCCtxParams(
-                cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
+        ZSTD_prefixDict const prefixDict = cctx->prefixDict;
        memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));  /* single usage */
        assert(prefixDict.dict==NULL || cctx->cdict==NULL);   /* only one can be set */
+        DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage");
+        if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1;  /* auto-fix pledgedSrcSize */
+        params.cParams = ZSTD_getCParamsFromCCtxParams(
+                cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);

 #ifdef ZSTD_MULTITHREAD
+        if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN)
+            params.nbThreads = 1; /* do not invoke multi-threading when src size is too small */
        if (params.nbThreads > 1) {
-            if (cctx->mtctx == NULL || cctx->appliedParams.nbThreads != params.nbThreads) {
+            if (cctx->mtctx == NULL || (params.nbThreads != ZSTDMT_getNbThreads(cctx->mtctx))) {
+                DEBUGLOG(4, "ZSTD_compress_generic: creating new mtctx for nbThreads=%u (previous: %u)",
+                            params.nbThreads, ZSTDMT_getNbThreads(cctx->mtctx));
                ZSTDMT_freeCCtx(cctx->mtctx);
                cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbThreads, cctx->customMem);
                if (cctx->mtctx == NULL) return ERROR(memory_allocation);
            }
            DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads);
            CHECK_F( ZSTDMT_initCStream_internal(
-                             cctx->mtctx,
-                             prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent,
-                             cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
+                        cctx->mtctx,
+                        prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent,
+                        cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
            cctx->streamStage = zcss_load;
            cctx->appliedParams.nbThreads = params.nbThreads;
        } else
 #endif
-        {
-            CHECK_F( ZSTD_resetCStream_internal(
+        {   CHECK_F( ZSTD_resetCStream_internal(
                             cctx, prefixDict.dict, prefixDict.dictSize,
                             prefixDict.dictMode, cctx->cdict, params,
                             cctx->pledgedSrcSizePlusOne-1) );
+            assert(cctx->streamStage == zcss_load);
+            assert(cctx->appliedParams.nbThreads <= 1);
    }   }

    /* compression stage */
 #ifdef ZSTD_MULTITHREAD
    if (cctx->appliedParams.nbThreads > 1) {
        size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
-        DEBUGLOG(5, "ZSTDMT_compressStream_generic : %u", (U32)flushMin);
        if ( ZSTD_isError(flushMin)
          || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
            ZSTD_startNewCompression(cctx);
@@ -2790,8 +2945,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
    {   size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
        size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4;
        size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize + lastBlockSize + checksumSize;
-        DEBUGLOG(5, "ZSTD_endStream : remaining to flush : %u",
-                (unsigned)toFlush);
+        DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (U32)toFlush);
        return toFlush;
    }
 }
@@ -2820,12 +2974,12 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
    { 22, 20, 22,  5,  5, 16, ZSTD_lazy2   },  /* level 12 */
    { 22, 21, 22,  5,  5, 16, ZSTD_lazy2   },  /* level 13 */
    { 22, 21, 22,  6,  5, 16, ZSTD_lazy2   },  /* level 14 */
-    { 22, 21, 22,  5,  5, 16, ZSTD_btlazy2 },  /* level 15 */
-    { 23, 22, 22,  5,  5, 16, ZSTD_btlazy2 },  /* level 16 */
-    { 23, 22, 22,  4,  5, 24, ZSTD_btopt   },  /* level 17 */
-    { 23, 22, 22,  5,  4, 32, ZSTD_btopt   },  /* level 18 */
-    { 23, 23, 22,  6,  3, 48, ZSTD_btopt   },  /* level 19 */
-    { 25, 25, 23,  7,  3, 64, ZSTD_btultra },  /* level 20 */
+    { 22, 21, 22,  4,  5, 16, ZSTD_btlazy2 },  /* level 15 */
+    { 22, 21, 22,  4,  5, 48, ZSTD_btopt   },  /* level 16 */
+    { 23, 22, 22,  4,  4, 48, ZSTD_btopt   },  /* level 17 */
+    { 23, 22, 22,  5,  3, 64, ZSTD_btopt   },  /* level 18 */
+    { 23, 23, 22,  7,  3,128, ZSTD_btopt   },  /* level 19 */
+    { 25, 25, 23,  7,  3,128, ZSTD_btultra },  /* level 20 */
    { 26, 26, 24,  7,  3,256, ZSTD_btultra },  /* level 21 */
    { 27, 27, 25,  9,  3,512, ZSTD_btultra },  /* level 22 */
 },
@@ -2944,6 +3098,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
    }
 #endif

+    DEBUGLOG(4, "ZSTD_getCParams: cLevel=%i, srcSize=%u, dictSize=%u => table %u",
+                compressionLevel, (U32)srcSizeHint, (U32)dictSize, tableID);
    if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
    if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
    { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];
@@ -2959,5 +3115,6 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize);
    memset(&params, 0, sizeof(params));
    params.cParams = cParams;
+    params.fParams.contentSizeFlag = 1;
    return params;
 }
@@ -8,6 +8,9 @@
 * You may select, at your option, one of the above-listed licenses.
 */

+/* This header contains definitions
+ * that shall **only** be used by modules within lib/compress.
+ */

 #ifndef ZSTD_COMPRESS_H
 #define ZSTD_COMPRESS_H
@@ -43,6 +46,95 @@ typedef struct ZSTD_prefixDict_s {
    ZSTD_dictMode_e dictMode;
 } ZSTD_prefixDict;

+typedef struct {
+    U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
+    FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+    FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+    FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+    U32 workspace[HUF_WORKSPACE_SIZE_U32];
+    HUF_repeat hufCTable_repeatMode;
+    FSE_repeat offcode_repeatMode;
+    FSE_repeat matchlength_repeatMode;
+    FSE_repeat litlength_repeatMode;
+} ZSTD_entropyCTables_t;
+
+typedef struct {
+    U32 off;
+    U32 len;
+} ZSTD_match_t;
+
+typedef struct {
+    int price;
+    U32 off;
+    U32 mlen;
+    U32 litlen;
+    U32 rep[ZSTD_REP_NUM];
+} ZSTD_optimal_t;
+
+typedef struct {
+    /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
+    U32* litFreq;               /* table of literals statistics, of size 256 */
+    U32* litLengthFreq;         /* table of litLength statistics, of size (MaxLL+1) */
+    U32* matchLengthFreq;       /* table of matchLength statistics, of size (MaxML+1) */
+    U32* offCodeFreq;           /* table of offCode statistics, of size (MaxOff+1) */
+    ZSTD_match_t* matchTable;   /* list of found matches, of size ZSTD_OPT_NUM+1 */
+    ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
+
+    U32  litSum;                 /* nb of literals */
+    U32  litLengthSum;           /* nb of litLength codes */
+    U32  matchLengthSum;         /* nb of matchLength codes */
+    U32  offCodeSum;             /* nb of offset codes */
+    /* begin updated by ZSTD_setLog2Prices */
+    U32  log2litSum;             /* pow2 to compare log2(litfreq) to */
+    U32  log2litLengthSum;       /* pow2 to compare log2(llfreq) to */
+    U32  log2matchLengthSum;     /* pow2 to compare log2(mlfreq) to */
+    U32  log2offCodeSum;         /* pow2 to compare log2(offreq) to */
+    /* end : updated by ZSTD_setLog2Prices */
+    U32  staticPrices;           /* prices follow a pre-defined cost structure, statistics are irrelevant */
+} optState_t;
+
+typedef struct {
+    U32 offset;
+    U32 checksum;
+} ldmEntry_t;
+
+typedef struct {
+    ldmEntry_t* hashTable;
+    BYTE* bucketOffsets;    /* Next position in bucket to insert entry */
+    U64 hashPower;          /* Used to compute the rolling hash.
+                             * Depends on ldmParams.minMatchLength */
+} ldmState_t;
+
+typedef struct {
+    U32 enableLdm;          /* 1 if enable long distance matching */
+    U32 hashLog;            /* Log size of hashTable */
+    U32 bucketSizeLog;      /* Log bucket size for collision resolution, at most 8 */
+    U32 minMatchLength;     /* Minimum match length */
+    U32 hashEveryLog;       /* Log number of entries to skip */
+} ldmParams_t;
+
+struct ZSTD_CCtx_params_s {
+    ZSTD_format_e format;
+    ZSTD_compressionParameters cParams;
+    ZSTD_frameParameters fParams;
+
+    int compressionLevel;
+    U32 forceWindow;           /* force back-references to respect limit of
+                                * 1<<wLog, even for dictionary */
+
+    /* Multithreading: used to pass parameters to mtctx */
+    U32 nbThreads;
+    unsigned jobSize;
+    unsigned overlapSizeLog;
+
+    /* Long distance matching parameters */
+    ldmParams_t ldmParams;
+
+    /* For use with createCCtxParams() and freeCCtxParams() only */
+    ZSTD_customMem customMem;
+
+};  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
+
 struct ZSTD_CCtx_s {
    const BYTE* nextSrc;    /* next block here to continue on current prefix */
    const BYTE* base;       /* All regular indexes relative to this position */
@@ -99,38 +191,51 @@ struct ZSTD_CCtx_s {
 };


-static const BYTE LL_Code[64] = {  0,  1,  2,  3,  4,  5,  6,  7,
-                                   8,  9, 10, 11, 12, 13, 14, 15,
-                                  16, 16, 17, 17, 18, 18, 19, 19,
-                                  20, 20, 20, 20, 21, 21, 21, 21,
-                                  22, 22, 22, 22, 22, 22, 22, 22,
-                                  23, 23, 23, 23, 23, 23, 23, 23,
-                                  24, 24, 24, 24, 24, 24, 24, 24,
-                                  24, 24, 24, 24, 24, 24, 24, 24 };
+MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
+{
+    static const BYTE LL_Code[64] = {  0,  1,  2,  3,  4,  5,  6,  7,
+                                       8,  9, 10, 11, 12, 13, 14, 15,
+                                      16, 16, 17, 17, 18, 18, 19, 19,
+                                      20, 20, 20, 20, 21, 21, 21, 21,
+                                      22, 22, 22, 22, 22, 22, 22, 22,
+                                      23, 23, 23, 23, 23, 23, 23, 23,
+                                      24, 24, 24, 24, 24, 24, 24, 24,
+                                      24, 24, 24, 24, 24, 24, 24, 24 };
+    static const U32 LL_deltaCode = 19;
+    return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
+}

-static const BYTE ML_Code[128] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
-                                  16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
-                                  32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
-                                  38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
-                                  40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
-                                  41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
-                                  42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
-                                  42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
+/* ZSTD_MLcode() :
+ * note : mlBase = matchLength - MINMATCH;
+ *        because it's the format it's stored in seqStore->sequences */
+MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
+{
+    static const BYTE ML_Code[128] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
+                                      16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+                                      32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
+                                      38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
+                                      40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+                                      41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
+                                      42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+                                      42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
+    static const U32 ML_deltaCode = 36;
+    return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
+}

 /*! ZSTD_storeSeq() :
-    Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
-    `offsetCode` : distance to match, or 0 == repCode.
-    `matchCode` : matchLength - MINMATCH
+ *  Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
+ *  `offsetCode` : distance to match + 3 (values 1-3 are repCodes).
+ *  `mlBase` : matchLength - MINMATCH
 */
-MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
+MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase)
 {
 #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
    static const BYTE* g_start = NULL;
-    U32 const pos = (U32)((const BYTE*)literals - g_start);
-    if (g_start==NULL) g_start = (const BYTE*)literals;
-    if ((pos > 0) && (pos < 1000000000))
-        DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
-               pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+    if (g_start==NULL) g_start = (const BYTE*)literals;  /* note : index only works for compression within a single segment */
+    {   U32 const pos = (U32)((const BYTE*)literals - g_start);
+        DEBUGLOG(6, "Cpos%7u :%3u literals, match%3u bytes at dist.code%7u",
+               pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode);
+    }
 #endif
    /* copy Literals */
    assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
@@ -139,6 +244,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v

    /* literal Length */
    if (litLength>0xFFFF) {
+        assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
        seqStorePtr->longLengthID = 1;
        seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
    }
@@ -148,11 +254,12 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
    seqStorePtr->sequences[0].offset = offsetCode + 1;

    /* match Length */
-    if (matchCode>0xFFFF) {
+    if (mlBase>0xFFFF) {
+        assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
        seqStorePtr->longLengthID = 2;
        seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
    }
-    seqStorePtr->sequences[0].matchLength = (U16)matchCode;
+    seqStorePtr->sequences[0].matchLength = (U16)mlBase;

    seqStorePtr->sequences++;
 }
@@ -161,7 +268,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
 /*-*************************************
 *  Match length counter
 ***************************************/
-static unsigned ZSTD_NbCommonBytes (register size_t val)
+static unsigned ZSTD_NbCommonBytes (size_t val)
 {
    if (MEM_isLittleEndian()) {
        if (MEM_64bits()) {
@@ -235,13 +342,17 @@ MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* co
    const BYTE* const pStart = pIn;
    const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);

-    while (pIn < pInLoopLimit) {
-        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
-        if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
-        pIn += ZSTD_NbCommonBytes(diff);
-        return (size_t)(pIn - pStart);
-    }
-    if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+    if (pIn < pInLoopLimit) {
+        { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+          if (diff) return ZSTD_NbCommonBytes(diff); }
+        pIn+=sizeof(size_t); pMatch+=sizeof(size_t);
+        while (pIn < pInLoopLimit) {
+            size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+            if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
+            pIn += ZSTD_NbCommonBytes(diff);
+            return (size_t)(pIn - pStart);
+    }   }
+    if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
    if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
    if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
    return (size_t)(pIn - pStart);
@@ -304,4 +415,48 @@ MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
 }
 #endif

+
+/* ==============================================================
+ * Private declarations
+ * These prototypes shall only be called from within lib/compress
+ * ============================================================== */
+
+/*! ZSTD_initCStream_internal() :
+ *  Private use only. Init streaming operation.
+ *  expects params to be valid.
+ *  must receive dict, or cdict, or none, but not both.
+ *  @return : 0, or an error code */
+size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+                     const void* dict, size_t dictSize,
+                     const ZSTD_CDict* cdict,
+                     ZSTD_CCtx_params  params, unsigned long long pledgedSrcSize);
+
+/*! ZSTD_compressStream_generic() :
+ *  Private use only. To be called from zstdmt_compress.c in single-thread mode. */
+size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
+                                   ZSTD_outBuffer* output,
+                                   ZSTD_inBuffer* input,
+                                   ZSTD_EndDirective const flushMode);
+
+/*! ZSTD_getCParamsFromCDict() :
+ *  as the name implies */
+ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
+
+/* ZSTD_compressBegin_advanced_internal() :
+ * Private use only. To be called from zstdmt_compress.c. */
+size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
+                                    const void* dict, size_t dictSize,
+                                    ZSTD_dictMode_e dictMode,
+                                    const ZSTD_CDict* cdict,
+                                    ZSTD_CCtx_params params,
+                                    unsigned long long pledgedSrcSize);
+
+/* ZSTD_compress_advanced_internal() :
+ * Private use only. To be called from zstdmt_compress.c. */
+size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
+                                       void* dst, size_t dstCapacity,
+                                 const void* src, size_t srcSize,
+                                 const void* dict,size_t dictSize,
+                                 ZSTD_CCtx_params params);
+
 #endif /* ZSTD_COMPRESS_H */
@@ -8,6 +8,7 @@
 * You may select, at your option, one of the above-listed licenses.
 */

+#include "zstd_compress_internal.h"
 #include "zstd_double_fast.h"


@@ -11,12 +11,13 @@
 #ifndef ZSTD_DOUBLE_FAST_H
 #define ZSTD_DOUBLE_FAST_H

-#include "zstd_compress.h"
-
 #if defined (__cplusplus)
 extern "C" {
 #endif

+#include "mem.h"      /* U32 */
+#include "zstd.h"     /* ZSTD_CCtx, size_t */
+
 void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls);
 size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
 size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
@@ -8,6 +8,7 @@
 * You may select, at your option, one of the above-listed licenses.
 */

+#include "zstd_compress_internal.h"
 #include "zstd_fast.h"


@@ -11,12 +11,13 @@
 #ifndef ZSTD_FAST_H
 #define ZSTD_FAST_H

-#include "zstd_compress.h"
-
 #if defined (__cplusplus)
 extern "C" {
 #endif

+#include "mem.h"      /* U32 */
+#include "zstd.h"     /* ZSTD_CCtx, size_t */
+
 void ZSTD_fillHashTable(ZSTD_CCtx* zc, const void* end, const U32 mls);
 size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
                         const void* src, size_t srcSize);
@@ -8,6 +8,7 @@
 * You may select, at your option, one of the above-listed licenses.
 */

+#include "zstd_compress_internal.h"
 #include "zstd_lazy.h"


@@ -15,10 +16,11 @@
 *  Binary Tree search
 ***************************************/
 /** ZSTD_insertBt1() : add one or multiple positions to tree.
-*   ip : assumed <= iend-8 .
-*   @return : nb of positions added */
-static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares,
-                          U32 extDict)
+ *  ip : assumed <= iend-8 .
+ * @return : nb of positions added */
+static U32 ZSTD_insertBt1(ZSTD_CCtx* zc,
+                const BYTE* const ip, const BYTE* const iend,
+                U32 nbCompares, U32 const mls, U32 const extDict)
 {
    U32*   const hashTable = zc->hashTable;
    U32    const hashLog = zc->appliedParams.cParams.hashLog;
@@ -40,7 +42,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
    U32* largerPtr  = smallerPtr + 1;
    U32 dummy32;   /* to be nullified at the end */
    U32 const windowLow = zc->lowLimit;
-    U32 matchEndIdx = current+8;
+    U32 matchEndIdx = current+8+1;
    size_t bestLength = 8;
 #ifdef ZSTD_C_PREDICT
    U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0);
@@ -49,12 +51,15 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
    predictedLarge += (predictedLarge>0);
 #endif /* ZSTD_C_PREDICT */

+    DEBUGLOG(8, "ZSTD_insertBt1 (%u)", current);
+
    assert(ip <= iend-8);   /* required for h calculation */
    hashTable[h] = current;   /* Update Hash Table */

    while (nbCompares-- && (matchIndex > windowLow)) {
        U32* const nextPtr = bt + 2*(matchIndex & btMask);
        size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger);   /* guaranteed minimum nb of common bytes */
+        assert(matchIndex < current);

 #ifdef ZSTD_C_PREDICT   /* note : can create issues when hlog small <= 11 */
        const U32* predictPtr = bt + 2*((matchIndex-1) & btMask);   /* written this way, as bt is a roll buffer */
@@ -76,10 +81,11 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
            continue;
        }
 #endif
+
        if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
+            assert(matchIndex+matchLength >= dictLimit);   /* might be wrong if extDict is incorrectly set to 0 */
            match = base + matchIndex;
-            if (match[matchLength] == ip[matchLength])
-                matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
+            matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
        } else {
            match = dictBase + matchIndex;
            matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
@@ -93,16 +99,17 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
                matchEndIdx = matchIndex + (U32)matchLength;
        }

-        if (ip+matchLength == iend)   /* equal : no way to know if inf or sup */
+        if (ip+matchLength == iend) {   /* equal : no way to know if inf or sup */
            break;   /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
+        }

        if (match[matchLength] < ip[matchLength]) {  /* necessarily within buffer */
-            /* match+1 is smaller than current */
+            /* match is smaller than current */
            *smallerPtr = matchIndex;             /* update smaller idx */
            commonLengthSmaller = matchLength;    /* all smaller will now have at least this guaranteed common length */
            if (matchIndex <= btLow) { smallerPtr=&dummy32; break; }   /* beyond tree size, stop searching */
-            smallerPtr = nextPtr+1;               /* new "smaller" => larger of match */
-            matchIndex = nextPtr[1];              /* new matchIndex larger than previous (closer to current) */
+            smallerPtr = nextPtr+1;               /* new "candidate" => larger than match, which was smaller than target */
+            matchIndex = nextPtr[1];              /* new matchIndex, larger than previous and closer to current */
        } else {
            /* match is larger than current */
            *largerPtr = matchIndex;
@@ -114,8 +121,38 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co

    *smallerPtr = *largerPtr = 0;
    if (bestLength > 384) return MIN(192, (U32)(bestLength - 384));   /* speed optimization */
-    if (matchEndIdx > current + 8) return matchEndIdx - (current + 8);
-    return 1;
+    assert(matchEndIdx > current + 8);
+    return matchEndIdx - (current + 8);
+}
+
+FORCE_INLINE_TEMPLATE
+void ZSTD_updateTree_internal(ZSTD_CCtx* zc,
+                const BYTE* const ip, const BYTE* const iend,
+                const U32 nbCompares, const U32 mls, const U32 extDict)
+{
+    const BYTE* const base = zc->base;
+    U32 const target = (U32)(ip - base);
+    U32 idx = zc->nextToUpdate;
+    DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u  (extDict:%u)",
+                idx, target, extDict);
+
+    while(idx < target)
+        idx += ZSTD_insertBt1(zc, base+idx, iend, nbCompares, mls, extDict);
+    zc->nextToUpdate = target;
+}
+
+void ZSTD_updateTree(ZSTD_CCtx* zc,
+                const BYTE* const ip, const BYTE* const iend,
+                const U32 nbCompares, const U32 mls)
+{
+    ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 0 /*extDict*/);
+}
+
+void ZSTD_updateTree_extDict(ZSTD_CCtx* zc,
+                const BYTE* const ip, const BYTE* const iend,
+                const U32 nbCompares, const U32 mls)
+{
+    ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 1 /*extDict*/);
 }


@@ -144,7 +181,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
    const U32 windowLow = zc->lowLimit;
    U32* smallerPtr = bt + 2*(current&btMask);
    U32* largerPtr  = bt + 2*(current&btMask) + 1;
-    U32 matchEndIdx = current+8;
+    U32 matchEndIdx = current+8+1;
    U32 dummy32;   /* to be nullified at the end */
    size_t bestLength = 0;

@@ -158,8 +195,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (

        if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
            match = base + matchIndex;
-            if (match[matchLength] == ip[matchLength])
-                matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
+            matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
        } else {
            match = dictBase + matchIndex;
            matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
@@ -172,8 +208,9 @@ static size_t ZSTD_insertBtAndFindBestMatch (
                matchEndIdx = matchIndex + (U32)matchLength;
            if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
                bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
-            if (ip+matchLength == iend)   /* equal : no way to know if inf or sup */
+            if (ip+matchLength == iend) {   /* equal : no way to know if inf or sup */
                break;   /* drop, to guarantee consistency (miss a little bit of compression) */
+            }
        }

        if (match[matchLength] < ip[matchLength]) {
@@ -194,21 +231,12 @@ static size_t ZSTD_insertBtAndFindBestMatch (

    *smallerPtr = *largerPtr = 0;

-    zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1;
+    assert(matchEndIdx > current+8);
+    zc->nextToUpdate = matchEndIdx - 8;   /* skip repetitive patterns */
    return bestLength;
 }


-void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
-{
-    const BYTE* const base = zc->base;
-    const U32 target = (U32)(ip - base);
-    U32 idx = zc->nextToUpdate;
-
-    while(idx < target)
-        idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0);
-}
-
 /** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
 static size_t ZSTD_BtFindBestMatch (
                        ZSTD_CCtx* zc,
@@ -239,16 +267,6 @@ static size_t ZSTD_BtFindBestMatch_selectMLS (
 }


-void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
-{
-    const BYTE* const base = zc->base;
-    const U32 target = (U32)(ip - base);
-    U32 idx = zc->nextToUpdate;
-
-    while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1);
-}
-
-
 /** Tree updater, providing best match */
 static size_t ZSTD_BtFindBestMatch_extDict (
                        ZSTD_CCtx* zc,
@@ -335,14 +353,14 @@ size_t ZSTD_HcFindBestMatch_generic (
    U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);

    for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
-        const BYTE* match;
        size_t currentMl=0;
        if ((!extDict) || matchIndex >= dictLimit) {
-            match = base + matchIndex;
+            const BYTE* const match = base + matchIndex;
            if (match[ml] == ip[ml])   /* potentially better */
                currentMl = ZSTD_count(ip, match, iLimit);
        } else {
-            match = dictBase + matchIndex;
+            const BYTE* const match = dictBase + matchIndex;
+            assert(match+4 <= dictEnd);
            if (MEM_read32(match) == MEM_read32(ip))   /* assumption : matchIndex <= dictLimit-4 (by table construction) */
                currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
        }
@@ -380,10 +398,10 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (


 FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
-                        ZSTD_CCtx* zc,
+                        ZSTD_CCtx* const zc,
                        const BYTE* ip, const BYTE* const iLimit,
-                        size_t* offsetPtr,
-                        const U32 maxNbAttempts, const U32 matchLengthSearch)
+                        size_t* const offsetPtr,
+                        U32 const maxNbAttempts, U32 const matchLengthSearch)
 {
    switch(matchLengthSearch)
    {
@@ -502,9 +520,8 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
         */
        /* catch up */
        if (offset) {
-            while ( (start > anchor)
-                 && (start > base+offset-ZSTD_REP_MOVE)
-                 && (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1]) )  /* only search for offset within prefix */
+            while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > base))
+                 && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) )  /* only search for offset within prefix */
                { start--; matchLength++; }
            offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE);
        }
@@ -516,9 +533,8 @@ _storeSequence:
        }

        /* check immediate repcode */
-        while ( (ip <= ilimit)
-             && ((offset_2>0)
-             & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
+        while ( ((ip <= ilimit) & (offset_2>0))
+             && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
            /* store sequence */
            matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
            offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */
@@ -11,12 +11,13 @@
 #ifndef ZSTD_LAZY_H
 #define ZSTD_LAZY_H

-#include "zstd_compress.h"
-
 #if defined (__cplusplus)
 extern "C" {
 #endif

+#include "mem.h"    /* U32 */
+#include "zstd.h"   /* ZSTD_CCtx, size_t */
+
 U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls);
 void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls);
 void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls);
@@ -14,14 +14,14 @@

 #define LDM_BUCKET_SIZE_LOG 3
 #define LDM_MIN_MATCH_LENGTH 64
-#define LDM_HASH_LOG 20
+#define LDM_HASH_RLOG 7
 #define LDM_HASH_CHAR_OFFSET 10

 size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
 {
    ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
    params->enableLdm = enableLdm>0;
-    params->hashLog = LDM_HASH_LOG;
+    params->hashLog = 0;
    params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
    params->minMatchLength = LDM_MIN_MATCH_LENGTH;
    params->hashEveryLog = ZSTD_LDM_HASHEVERYLOG_NOTSET;
@@ -30,6 +30,10 @@ size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)

 void ZSTD_ldm_adjustParameters(ldmParams_t* params, U32 windowLog)
 {
+    if (params->hashLog == 0) {
+        params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG);
+        assert(params->hashLog <= ZSTD_HASHLOG_MAX);
+    }
    if (params->hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET) {
        params->hashEveryLog =
                windowLog < params->hashLog ? 0 : windowLog - params->hashLog;
@@ -295,7 +299,7 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx,
    const U32   lowestIndex = cctx->dictLimit;
    const BYTE* const lowest = base + lowestIndex;
    const BYTE* const iend = istart + srcSize;
-    const BYTE* const ilimit = iend - ldmParams.minMatchLength;
+    const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE);

    const ZSTD_blockCompressor blockCompressor =
        ZSTD_selectBlockCompressor(cctx->appliedParams.cParams.strategy, 0);
@@ -499,7 +503,7 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic(
    const BYTE* const lowPrefixPtr = base + dictLimit;
    const BYTE* const dictEnd = dictBase + dictLimit;
    const BYTE* const iend = istart + srcSize;
-    const BYTE* const ilimit = iend - ldmParams.minMatchLength;
+    const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE);

    const ZSTD_blockCompressor blockCompressor =
        ZSTD_selectBlockCompressor(ctx->appliedParams.cParams.strategy, 1);
@@ -10,17 +10,18 @@
 #ifndef ZSTD_LDM_H
 #define ZSTD_LDM_H

-#include "zstd_compress.h"
-
 #if defined (__cplusplus)
 extern "C" {
 #endif

+#include "zstd_compress_internal.h"   /* ldmParams_t, U32 */
+#include "zstd.h"   /* ZSTD_CCtx, size_t */
+
 /*-*************************************
 *  Long distance matching
 ***************************************/

-#define ZSTD_LDM_WINDOW_LOG 27
+#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
 #define ZSTD_LDM_HASHEVERYLOG_NOTSET 9999

 /** ZSTD_compressBlock_ldm_generic() :
@@ -8,36 +8,35 @@
 * You may select, at your option, one of the above-listed licenses.
 */

+#include "zstd_compress_internal.h"
 #include "zstd_opt.h"
-#include "zstd_lazy.h"
+#include "zstd_lazy.h"   /* ZSTD_updateTree, ZSTD_updateTree_extDict */


-#define ZSTD_LITFREQ_ADD    2
-#define ZSTD_FREQ_DIV       4
-#define ZSTD_MAX_PRICE      (1<<30)
+#define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats. Also used for matchSum (?) */
+#define ZSTD_FREQ_DIV       4   /* log factor when using previous stats to init next stats */
+#define ZSTD_MAX_PRICE     (1<<30)
+

 /*-*************************************
 *  Price functions for optimal parser
 ***************************************/
 static void ZSTD_setLog2Prices(optState_t* optPtr)
 {
-    optPtr->log2matchLengthSum = ZSTD_highbit32(optPtr->matchLengthSum+1);
-    optPtr->log2litLengthSum = ZSTD_highbit32(optPtr->litLengthSum+1);
    optPtr->log2litSum = ZSTD_highbit32(optPtr->litSum+1);
+    optPtr->log2litLengthSum = ZSTD_highbit32(optPtr->litLengthSum+1);
+    optPtr->log2matchLengthSum = ZSTD_highbit32(optPtr->matchLengthSum+1);
    optPtr->log2offCodeSum = ZSTD_highbit32(optPtr->offCodeSum+1);
-    optPtr->factor = 1 + ((optPtr->litSum>>5) / optPtr->litLengthSum) + ((optPtr->litSum<<1) / (optPtr->litSum + optPtr->matchSum));
 }


-static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSize)
+static void ZSTD_rescaleFreqs(optState_t* const optPtr,
+                              const BYTE* const src, size_t const srcSize)
 {
-    unsigned u;
-
-    optPtr->cachedLiterals = NULL;
-    optPtr->cachedPrice = optPtr->cachedLitLength = 0;
    optPtr->staticPrices = 0;

-    if (optPtr->litLengthSum == 0) {
+    if (optPtr->litLengthSum == 0) {  /* first init */
+        unsigned u;
        if (srcSize <= 1024) optPtr->staticPrices = 1;

        assert(optPtr->litFreq!=NULL);
@@ -45,44 +44,41 @@ static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSiz
            optPtr->litFreq[u] = 0;
        for (u=0; u<srcSize; u++)
            optPtr->litFreq[src[u]]++;
-
        optPtr->litSum = 0;
-        optPtr->litLengthSum = MaxLL+1;
-        optPtr->matchLengthSum = MaxML+1;
-        optPtr->offCodeSum = (MaxOff+1);
-        optPtr->matchSum = (ZSTD_LITFREQ_ADD<<Litbits);
-
        for (u=0; u<=MaxLit; u++) {
-            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> ZSTD_FREQ_DIV);
            optPtr->litSum += optPtr->litFreq[u];
        }
+
        for (u=0; u<=MaxLL; u++)
            optPtr->litLengthFreq[u] = 1;
+        optPtr->litLengthSum = MaxLL+1;
        for (u=0; u<=MaxML; u++)
            optPtr->matchLengthFreq[u] = 1;
+        optPtr->matchLengthSum = MaxML+1;
        for (u=0; u<=MaxOff; u++)
            optPtr->offCodeFreq[u] = 1;
-    } else {
-        optPtr->matchLengthSum = 0;
-        optPtr->litLengthSum = 0;
-        optPtr->offCodeSum = 0;
-        optPtr->matchSum = 0;
-        optPtr->litSum = 0;
+        optPtr->offCodeSum = (MaxOff+1);

+    } else {
+        unsigned u;
+
+        optPtr->litSum = 0;
        for (u=0; u<=MaxLit; u++) {
-            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1));
+            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> (ZSTD_FREQ_DIV+1));
            optPtr->litSum += optPtr->litFreq[u];
        }
+        optPtr->litLengthSum = 0;
        for (u=0; u<=MaxLL; u++) {
            optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1));
            optPtr->litLengthSum += optPtr->litLengthFreq[u];
        }
+        optPtr->matchLengthSum = 0;
        for (u=0; u<=MaxML; u++) {
            optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV);
            optPtr->matchLengthSum += optPtr->matchLengthFreq[u];
-            optPtr->matchSum += optPtr->matchLengthFreq[u] * (u + 3);
        }
-        optPtr->matchSum *= ZSTD_LITFREQ_ADD;
+        optPtr->offCodeSum = 0;
        for (u=0; u<=MaxOff; u++) {
            optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV);
            optPtr->offCodeSum += optPtr->offCodeFreq[u];
@@ -93,114 +89,146 @@ static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSiz
 }


-static U32 ZSTD_getLiteralPrice(optState_t* optPtr, U32 litLength, const BYTE* literals)
+/* ZSTD_rawLiteralsCost() :
+ * cost of literals (only) in given segment (which length can be null)
+ * does not include cost of literalLength symbol */
+static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
+                                const optState_t* const optPtr)
 {
-    U32 price, u;
-
-    if (optPtr->staticPrices)
-        return ZSTD_highbit32((U32)litLength+1) + (litLength*6);
-
-    if (litLength == 0)
-        return optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[0]+1);
+    if (optPtr->staticPrices) return (litLength*6);  /* 6 bit per literal - no statistic used */
+    if (litLength == 0) return 0;

    /* literals */
-    if (optPtr->cachedLiterals == literals) {
-        U32 const additional = litLength - optPtr->cachedLitLength;
-        const BYTE* literals2 = optPtr->cachedLiterals + optPtr->cachedLitLength;
-        price = optPtr->cachedPrice + additional * optPtr->log2litSum;
-        for (u=0; u < additional; u++)
-            price -= ZSTD_highbit32(optPtr->litFreq[literals2[u]]+1);
-        optPtr->cachedPrice = price;
-        optPtr->cachedLitLength = litLength;
-    } else {
-        price = litLength * optPtr->log2litSum;
+    {   U32 u;
+        U32 cost = litLength * optPtr->log2litSum;
        for (u=0; u < litLength; u++)
-            price -= ZSTD_highbit32(optPtr->litFreq[literals[u]]+1);
-
-        if (litLength >= 12) {
-            optPtr->cachedLiterals = literals;
-            optPtr->cachedPrice = price;
-            optPtr->cachedLitLength = litLength;
-        }
+            cost -= ZSTD_highbit32(optPtr->litFreq[literals[u]]+1);
+        return cost;
    }
-
-    /* literal Length */
-    {   const BYTE LL_deltaCode = 19;
-        const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
-        price += LL_bits[llCode] + optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1);
-    }
-
-    return price;
 }

-
-FORCE_INLINE_TEMPLATE U32 ZSTD_getPrice(optState_t* optPtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
+/* ZSTD_litLengthPrice() :
+ * cost of literalLength symbol */
+static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr)
 {
-    /* offset */
-    U32 price;
-    BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
+    if (optPtr->staticPrices) return ZSTD_highbit32((U32)litLength+1);

-    if (optPtr->staticPrices)
-        return ZSTD_getLiteralPrice(optPtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode;
+    /* literal Length */
+    {   U32 const llCode = ZSTD_LLcode(litLength);
+        U32 const price = LL_bits[llCode] + optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1);
+        return price;
+    }
+}
+
+/* ZSTD_litLengthPrice() :
+ * cost of the literal part of a sequence,
+ * including literals themselves, and literalLength symbol */
+static U32 ZSTD_fullLiteralsCost(const BYTE* const literals, U32 const litLength,
+                                 const optState_t* const optPtr)
+{
+    return ZSTD_rawLiteralsCost(literals, litLength, optPtr)
+         + ZSTD_litLengthPrice(litLength, optPtr);
+}
+
+/* ZSTD_litLengthContribution() :
+ * @return ( cost(litlength) - cost(0) )
+ * this value can then be added to rawLiteralsCost()
+ * to provide a cost which is directly comparable to a match ending at same position */
+static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr)
+{
+    if (optPtr->staticPrices) return ZSTD_highbit32(litLength+1);
+
+    /* literal Length */
+    {   U32 const llCode = ZSTD_LLcode(litLength);
+        int const contribution = LL_bits[llCode]
+                        + ZSTD_highbit32(optPtr->litLengthFreq[0]+1)
+                        - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1);
+#if 1
+        return contribution;
+#else
+        return MAX(0, contribution); /* sometimes better, sometimes not ... */
+#endif
+    }
+}
+
+/* ZSTD_literalsContribution() :
+ * creates a fake cost for the literals part of a sequence
+ * which can be compared to the ending cost of a match
+ * should a new match start at this position */
+static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLength,
+                                     const optState_t* const optPtr)
+{
+    int const contribution = ZSTD_rawLiteralsCost(literals, litLength, optPtr)
+                           + ZSTD_litLengthContribution(litLength, optPtr);
+    return contribution;
+}
+
+/* ZSTD_getMatchPrice() :
+ * Provides the cost of the match part (offset + matchLength) of a sequence
+ * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
+ * optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) */
+FORCE_INLINE_TEMPLATE U32 ZSTD_getMatchPrice(
+                                    U32 const offset, U32 const matchLength,
+                                    const optState_t* const optPtr,
+                                    int const optLevel)
+{
+    U32 price;
+    U32 const offCode = ZSTD_highbit32(offset+1);
+    U32 const mlBase = matchLength - MINMATCH;
+    assert(matchLength >= MINMATCH);
+
+    if (optPtr->staticPrices)  /* fixed scheme, do not use statistics */
+        return ZSTD_highbit32((U32)mlBase+1) + 16 + offCode;

    price = offCode + optPtr->log2offCodeSum - ZSTD_highbit32(optPtr->offCodeFreq[offCode]+1);
-    if (!ultra && offCode >= 20) price += (offCode-19)*2;
+    if ((optLevel<2) /*static*/ && offCode >= 20) price += (offCode-19)*2; /* handicap for long distance offsets, favor decompression speed */

    /* match Length */
-    {   const BYTE ML_deltaCode = 36;
-        const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
+    {   U32 const mlCode = ZSTD_MLcode(mlBase);
        price += ML_bits[mlCode] + optPtr->log2matchLengthSum - ZSTD_highbit32(optPtr->matchLengthFreq[mlCode]+1);
    }

-    return price + ZSTD_getLiteralPrice(optPtr, litLength, literals) + optPtr->factor;
+    DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price);
+    return price;
 }

-
-static void ZSTD_updatePrice(optState_t* optPtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
+static void ZSTD_updateStats(optState_t* const optPtr,
+                             U32 litLength, const BYTE* literals,
+                             U32 offsetCode, U32 matchLength)
 {
-    U32 u;
-
    /* literals */
-    optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
-    for (u=0; u < litLength; u++)
-        optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
+    {   U32 u;
+        for (u=0; u < litLength; u++)
+            optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
+        optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
+    }

    /* literal Length */
-    {   const BYTE LL_deltaCode = 19;
-        const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
+    {   U32 const llCode = ZSTD_LLcode(litLength);
        optPtr->litLengthFreq[llCode]++;
        optPtr->litLengthSum++;
    }

-    /* match offset */
-    {   BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
-        optPtr->offCodeSum++;
+    /* match offset code (0-2=>repCode; 3+=>offset+2) */
+    {   U32 const offCode = ZSTD_highbit32(offsetCode+1);
+        assert(offCode <= MaxOff);
        optPtr->offCodeFreq[offCode]++;
+        optPtr->offCodeSum++;
    }

    /* match Length */
-    {   const BYTE ML_deltaCode = 36;
-        const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
+    {   U32 const mlBase = matchLength - MINMATCH;
+        U32 const mlCode = ZSTD_MLcode(mlBase);
        optPtr->matchLengthFreq[mlCode]++;
        optPtr->matchLengthSum++;
    }
-
-    ZSTD_setLog2Prices(optPtr);
 }


-#define SET_PRICE(pos, mlen_, offset_, litlen_, price_)   \
-    {                                                 \
-        while (last_pos < pos)  { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \
-        opt[pos].mlen = mlen_;                         \
-        opt[pos].off = offset_;                        \
-        opt[pos].litlen = litlen_;                     \
-        opt[pos].price = price_;                       \
-    }
-
-
-/* function safe only for comparisons */
-static U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
+/* ZSTD_readMINMATCH() :
+ * function safe only for comparisons
+ * assumption : memPtr must be at least 4 bytes before end of buffer */
+MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
 {
    switch (length)
    {
@@ -216,15 +244,14 @@ static U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)

 /* Update hashTable3 up to ip (excluded)
   Assumption : always within prefix (i.e. not within extDict) */
-static
-U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
+static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* const cctx, const BYTE* const ip)
 {
-    U32* const hashTable3  = zc->hashTable3;
-    U32 const hashLog3  = zc->hashLog3;
-    const BYTE* const base = zc->base;
-    U32 idx = zc->nextToUpdate3;
-    const U32 target = zc->nextToUpdate3 = (U32)(ip - base);
-    const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3);
+    U32* const hashTable3  = cctx->hashTable3;
+    U32 const hashLog3  = cctx->hashLog3;
+    const BYTE* const base = cctx->base;
+    U32 idx = cctx->nextToUpdate3;
+    U32 const target = cctx->nextToUpdate3 = (U32)(ip - base);
+    size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3);

    while(idx < target) {
        hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx;
@@ -238,102 +265,147 @@ U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
 /*-*************************************
 *  Binary Tree search
 ***************************************/
-static U32 ZSTD_insertBtAndGetAllMatches (
-                        ZSTD_CCtx* zc,
-                        const BYTE* const ip, const BYTE* const iLimit,
-                        U32 nbCompares, const U32 mls,
-                        U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen)
+FORCE_INLINE_TEMPLATE
+U32 ZSTD_insertBtAndGetAllMatches (
+                    ZSTD_CCtx* zc,
+                    const BYTE* const ip, const BYTE* const iLimit, int const extDict,
+                    U32 nbCompares, U32 const mls, U32 const sufficient_len,
+                    U32 rep[ZSTD_REP_NUM], U32 const ll0,
+                    ZSTD_match_t* matches, const U32 lengthToBeat)
 {
    const BYTE* const base = zc->base;
-    const U32 current = (U32)(ip-base);
-    const U32 hashLog = zc->appliedParams.cParams.hashLog;
-    const size_t h  = ZSTD_hashPtr(ip, hashLog, mls);
+    U32 const current = (U32)(ip-base);
+    U32 const hashLog = zc->appliedParams.cParams.hashLog;
+    U32 const minMatch = (mls==3) ? 3 : 4;
    U32* const hashTable = zc->hashTable;
+    size_t const h  = ZSTD_hashPtr(ip, hashLog, mls);
    U32 matchIndex  = hashTable[h];
    U32* const bt   = zc->chainTable;
-    const U32 btLog = zc->appliedParams.cParams.chainLog - 1;
-    const U32 btMask= (1U << btLog) - 1;
+    U32 const btLog = zc->appliedParams.cParams.chainLog - 1;
+    U32 const btMask= (1U << btLog) - 1;
    size_t commonLengthSmaller=0, commonLengthLarger=0;
    const BYTE* const dictBase = zc->dictBase;
-    const U32 dictLimit = zc->dictLimit;
+    U32 const dictLimit = zc->dictLimit;
    const BYTE* const dictEnd = dictBase + dictLimit;
    const BYTE* const prefixStart = base + dictLimit;
-    const U32 btLow = btMask >= current ? 0 : current - btMask;
-    const U32 windowLow = zc->lowLimit;
+    U32 const btLow = btMask >= current ? 0 : current - btMask;
+    U32 const windowLow = zc->lowLimit;
    U32* smallerPtr = bt + 2*(current&btMask);
    U32* largerPtr  = bt + 2*(current&btMask) + 1;
-    U32 matchEndIdx = current+8;
+    U32 matchEndIdx = current+8+1;   /* farthest referenced position of any match => detects repetitive patterns */
    U32 dummy32;   /* to be nullified at the end */
    U32 mnum = 0;

-    const U32 minMatch = (mls == 3) ? 3 : 4;
-    size_t bestLength = minMatchLen-1;
+    size_t bestLength = lengthToBeat-1;
+    DEBUGLOG(7, "ZSTD_insertBtAndGetAllMatches");

-    if (minMatch == 3) { /* HC3 match finder */
+    /* check repCode */
+    {   U32 const lastR = ZSTD_REP_NUM + ll0;
+        U32 repCode;
+        for (repCode = ll0; repCode < lastR; repCode++) {
+            U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
+            U32 const repIndex = current - repOffset;
+            U32 repLen = 0;
+            assert(current >= dictLimit);
+            if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < current-dictLimit) {  /* equivalent to `current > repIndex >= dictLimit` */
+                if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch)) {
+                    repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch;
+                }
+            } else {  /* repIndex < dictLimit || repIndex >= current */
+                const BYTE* const repMatch = dictBase + repIndex;
+                assert(current >= windowLow);
+                if ( extDict /* this case only valid in extDict mode */
+                  && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow)  /* equivalent to `current > repIndex >= windowLow` */
+                     & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)
+                  && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
+                    repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
+            }   }
+            /* save longer solution */
+            if (repLen > bestLength) {
+                DEBUGLOG(8, "found rep-match %u of length %u",
+                            repCode - ll0, (U32)repLen);
+                bestLength = repLen;
+                matches[mnum].off = repCode - ll0;
+                matches[mnum].len = (U32)repLen;
+                mnum++;
+                if ( (repLen > sufficient_len)
+                   | (ip+repLen == iLimit) ) {  /* best possible */
+                    return mnum;
+    }   }   }   }
+
+    /* HC3 match finder */
+    if ((mls == 3) /*static*/ && (bestLength < mls)) {
        U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip);
-        if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) {
-            const BYTE* match;
-            size_t currentMl=0;
-            if ((!extDict) || matchIndex3 >= dictLimit) {
-                match = base + matchIndex3;
-                if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit);
+        if ((matchIndex3 > windowLow)
+          & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) {
+            size_t mlen;
+            if ((!extDict) /*static*/ || (matchIndex3 >= dictLimit)) {
+                const BYTE* const match = base + matchIndex3;
+                mlen = ZSTD_count(ip, match, iLimit);
            } else {
-                match = dictBase + matchIndex3;
-                if (ZSTD_readMINMATCH(match, MINMATCH) == ZSTD_readMINMATCH(ip, MINMATCH))    /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
-                    currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
+                const BYTE* const match = dictBase + matchIndex3;
+                mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart);
            }

            /* save best solution */
-            if (currentMl > bestLength) {
-                bestLength = currentMl;
-                matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3;
-                matches[mnum].len = (U32)currentMl;
-                mnum++;
-                if (currentMl > ZSTD_OPT_NUM) goto update;
-                if (ip+currentMl == iLimit) goto update; /* best possible, and avoid read overflow*/
-            }
-        }
-    }
+            if (mlen >= mls /* == 3 > bestLength */) {
+                DEBUGLOG(8, "found small match with hlog3, of length %u",
+                            (U32)mlen);
+                bestLength = mlen;
+                assert(current > matchIndex3);
+                assert(mnum==0);  /* no prior solution */
+                matches[0].off = (current - matchIndex3) + ZSTD_REP_MOVE;
+                matches[0].len = (U32)mlen;
+                mnum = 1;
+                if ( (mlen > sufficient_len) |
+                     (ip+mlen == iLimit) ) {  /* best possible length */
+                    zc->nextToUpdate = current+1;  /* skip insertion */
+                    return 1;
+    }   }   }   }

    hashTable[h] = current;   /* Update Hash Table */

    while (nbCompares-- && (matchIndex > windowLow)) {
-        U32* nextPtr = bt + 2*(matchIndex & btMask);
+        U32* const nextPtr = bt + 2*(matchIndex & btMask);
        size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger);   /* guaranteed minimum nb of common bytes */
        const BYTE* match;
+        assert(current > matchIndex);

        if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
+            assert(matchIndex+matchLength >= dictLimit);  /* ensure the condition is correct when !extDict */
            match = base + matchIndex;
-            if (match[matchLength] == ip[matchLength]) {
-                matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1;
-            }
+            matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit);
        } else {
            match = dictBase + matchIndex;
            matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
            if (matchIndex+matchLength >= dictLimit)
-                match = base + matchIndex;   /* to prepare for next usage of match[matchLength] */
+                match = base + matchIndex;   /* prepare for match[matchLength] */
        }

        if (matchLength > bestLength) {
-            if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength;
+            DEBUGLOG(8, "found match of length %u at distance %u",
+                    (U32)matchLength, current - matchIndex);
+            assert(matchEndIdx > matchIndex);
+            if (matchLength > matchEndIdx - matchIndex)
+                matchEndIdx = matchIndex + (U32)matchLength;
            bestLength = matchLength;
-            matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex;
+            matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE;
            matches[mnum].len = (U32)matchLength;
            mnum++;
            if (matchLength > ZSTD_OPT_NUM) break;
-            if (ip+matchLength == iLimit)   /* equal : no way to know if inf or sup */
-                break;   /* drop, to guarantee consistency (miss a little bit of compression) */
+            if (ip+matchLength == iLimit) {  /* equal : no way to know if inf or sup */
+                break;   /* drop, to preserve bt consistency (miss a little bit of compression) */
+            }
        }

        if (match[matchLength] < ip[matchLength]) {
-            /* match is smaller than current */
+            /* match smaller than current */
            *smallerPtr = matchIndex;             /* update smaller idx */
            commonLengthSmaller = matchLength;    /* all smaller will now have at least this guaranteed common length */
            if (matchIndex <= btLow) { smallerPtr=&dummy32; break; }   /* beyond tree size, stop the search */
-            smallerPtr = nextPtr+1;               /* new "smaller" => larger of match */
-            matchIndex = nextPtr[1];              /* new matchIndex larger than previous (closer to current) */
+            smallerPtr = nextPtr+1;               /* new candidate => larger than match, which was smaller than current */
+            matchIndex = nextPtr[1];              /* new matchIndex, larger than previous, closer to current */
        } else {
-            /* match is larger than current */
            *largerPtr = matchIndex;
            commonLengthLarger = matchLength;
            if (matchIndex <= btLow) { largerPtr=&dummy32; break; }   /* beyond tree size, stop the search */
@@ -343,65 +415,31 @@ static U32 ZSTD_insertBtAndGetAllMatches (

    *smallerPtr = *largerPtr = 0;

-update:
-    zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1;
+    assert(matchEndIdx > current+8);
+    zc->nextToUpdate = matchEndIdx - 8;  /* skip repetitive patterns */
    return mnum;
 }


-/** Tree updater, providing best match */
-static U32 ZSTD_BtGetAllMatches (
-                        ZSTD_CCtx* zc,
-                        const BYTE* const ip, const BYTE* const iLimit,
-                        const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
-{
-    if (ip < zc->base + zc->nextToUpdate) return 0;   /* skipped area */
-    ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
-    return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen);
-}
-
-
-static U32 ZSTD_BtGetAllMatches_selectMLS (
+FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
                        ZSTD_CCtx* zc,   /* Index table will be updated */
-                        const BYTE* ip, const BYTE* const iHighLimit,
-                        const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
+                        const BYTE* ip, const BYTE* const iHighLimit, int const extDict,
+                        U32 const maxNbAttempts, U32 const matchLengthSearch, U32 const sufficient_len,
+                        U32 rep[ZSTD_REP_NUM], U32 const ll0,
+                        ZSTD_match_t* matches, U32 const lengthToBeat)
 {
+    DEBUGLOG(7, "ZSTD_BtGetAllMatches");
+    if (ip < zc->base + zc->nextToUpdate) return 0;   /* skipped area */
+    if (extDict) ZSTD_updateTree_extDict(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch);
+    else ZSTD_updateTree(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch);
    switch(matchLengthSearch)
    {
-    case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
+    case 3 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 3, sufficient_len, rep, ll0, matches, lengthToBeat);
    default :
-    case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
-    case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
+    case 4 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 4, sufficient_len, rep, ll0, matches, lengthToBeat);
+    case 5 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 5, sufficient_len, rep, ll0, matches, lengthToBeat);
    case 7 :
-    case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
-    }
-}
-
-/** Tree updater, providing best match */
-static U32 ZSTD_BtGetAllMatches_extDict (
-                        ZSTD_CCtx* zc,
-                        const BYTE* const ip, const BYTE* const iLimit,
-                        const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
-{
-    if (ip < zc->base + zc->nextToUpdate) return 0;   /* skipped area */
-    ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
-    return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen);
-}
-
-
-static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
-                        ZSTD_CCtx* zc,   /* Index table will be updated */
-                        const BYTE* ip, const BYTE* const iHighLimit,
-                        const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
-{
-    switch(matchLengthSearch)
-    {
-    case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
-    default :
-    case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
-    case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
-    case 7 :
-    case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
+    case 6 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 6, sufficient_len, rep, ll0, matches, lengthToBeat);
    }
 }

@@ -409,12 +447,92 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
 /*-*******************************
 *  Optimal parser
 *********************************/
+typedef struct repcodes_s {
+    U32 rep[3];
+} repcodes_t;
+
+repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
+{
+    repcodes_t newReps;
+    if (offset >= ZSTD_REP_NUM) {  /* full offset */
+        newReps.rep[2] = rep[1];
+        newReps.rep[1] = rep[0];
+        newReps.rep[0] = offset - ZSTD_REP_MOVE;
+    } else {   /* repcode */
+        U32 const repCode = offset + ll0;
+        if (repCode > 0) {  /* note : if repCode==0, no change */
+            U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
+            newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2];
+            newReps.rep[1] = rep[0];
+            newReps.rep[0] = currentOffset;
+        } else {   /* repCode == 0 */
+            memcpy(&newReps, rep, sizeof(newReps));
+        }
+    }
+    return newReps;
+}
+
+
+typedef struct {
+    const BYTE* anchor;
+    U32 litlen;
+    U32 rawLitCost;
+} cachedLiteralPrice_t;
+
+static U32 ZSTD_rawLiteralsCost_cached(
+                            cachedLiteralPrice_t* const cachedLitPrice,
+                            const BYTE* const anchor, U32 const litlen,
+                            const optState_t* const optStatePtr)
+{
+    U32 startCost;
+    U32 remainingLength;
+    const BYTE* startPosition;
+
+    if (anchor == cachedLitPrice->anchor) {
+        startCost = cachedLitPrice->rawLitCost;
+        startPosition = anchor + cachedLitPrice->litlen;
+        assert(litlen >= cachedLitPrice->litlen);
+        remainingLength = litlen - cachedLitPrice->litlen;
+    } else {
+        startCost = 0;
+        startPosition = anchor;
+        remainingLength = litlen;
+    }
+
+    {   U32 const rawLitCost = startCost + ZSTD_rawLiteralsCost(startPosition, remainingLength, optStatePtr);
+        cachedLitPrice->anchor = anchor;
+        cachedLitPrice->litlen = litlen;
+        cachedLitPrice->rawLitCost = rawLitCost;
+        return rawLitCost;
+    }
+}
+
+static U32 ZSTD_fullLiteralsCost_cached(
+                            cachedLiteralPrice_t* const cachedLitPrice,
+                            const BYTE* const anchor, U32 const litlen,
+                            const optState_t* const optStatePtr)
+{
+    return ZSTD_rawLiteralsCost_cached(cachedLitPrice, anchor, litlen, optStatePtr)
+         + ZSTD_litLengthPrice(litlen, optStatePtr);
+}
+
+static int ZSTD_literalsContribution_cached(
+                            cachedLiteralPrice_t* const cachedLitPrice,
+                            const BYTE* const anchor, U32 const litlen,
+                            const optState_t* const optStatePtr)
+{
+    int const contribution = ZSTD_rawLiteralsCost_cached(cachedLitPrice, anchor, litlen, optStatePtr)
+                           + ZSTD_litLengthContribution(litlen, optStatePtr);
+    return contribution;
+}
+
 FORCE_INLINE_TEMPLATE
 size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
-                                      const void* src, size_t srcSize, const int ultra)
+                                      const void* src, size_t srcSize,
+                                      const int optLevel, const int extDict)
 {
-    seqStore_t* seqStorePtr = &(ctx->seqStore);
-    optState_t* optStatePtr = &(ctx->optState);
+    seqStore_t* const seqStorePtr = &(ctx->seqStore);
+    optState_t* const optStatePtr = &(ctx->optState);
    const BYTE* const istart = (const BYTE*)src;
    const BYTE* ip = istart;
    const BYTE* anchor = istart;
@@ -423,230 +541,219 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
    const BYTE* const base = ctx->base;
    const BYTE* const prefixStart = base + ctx->dictLimit;

-    const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
-    const U32 sufficient_len = ctx->appliedParams.cParams.targetLength;
-    const U32 mls = ctx->appliedParams.cParams.searchLength;
-    const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
+    U32 const maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
+    U32 const sufficient_len = MIN(ctx->appliedParams.cParams.targetLength, ZSTD_OPT_NUM -1);
+    U32 const mls = ctx->appliedParams.cParams.searchLength;
+    U32 const minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;

-    ZSTD_optimal_t* opt = optStatePtr->priceTable;
-    ZSTD_match_t* matches = optStatePtr->matchTable;
-    const BYTE* inr;
-    U32 offset, rep[ZSTD_REP_NUM];
+    ZSTD_optimal_t* const opt = optStatePtr->priceTable;
+    ZSTD_match_t* const matches = optStatePtr->matchTable;
+    cachedLiteralPrice_t cachedLitPrice;
+    U32 rep[ZSTD_REP_NUM];

    /* init */
+    DEBUGLOG(5, "ZSTD_compressBlock_opt_generic");
    ctx->nextToUpdate3 = ctx->nextToUpdate;
    ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize);
    ip += (ip==prefixStart);
-    { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
+    memset(&cachedLitPrice, 0, sizeof(cachedLitPrice));

    /* Match Loop */
    while (ip < ilimit) {
-        U32 cur, match_num, last_pos, litlen, price;
-        U32 u, mlen, best_mlen, best_off, litLength;
-        memset(opt, 0, sizeof(ZSTD_optimal_t));
-        last_pos = 0;
-        litlen = (U32)(ip - anchor);
+        U32 cur, last_pos = 0;
+        U32 best_mlen, best_off;

-        /* check repCode */
-        {   U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
-            for (i=(ip == anchor); i<last_i; i++) {
-                const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
-                if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart))
-                    && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
-                    mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch;
-                    if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
-                        best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
-                        goto _storeSequence;
-                    }
-                    best_off = i - (ip == anchor);
-                    do {
-                        price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-                        if (mlen > last_pos || price < opt[mlen].price)
-                            SET_PRICE(mlen, mlen, i, litlen, price);   /* note : macro modifies last_pos */
-                        mlen--;
-                    } while (mlen >= minMatch);
-        }   }   }
+        /* find first match */
+        {   U32 const litlen = (U32)(ip - anchor);
+            U32 const ll0 = !litlen;
+            U32 const nbMatches = ZSTD_BtGetAllMatches(ctx, ip, iend, extDict, maxSearches, mls, sufficient_len, rep, ll0, matches, minMatch);
+            if (!nbMatches) { ip++; continue; }

-        match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
+            /* initialize opt[0] */
+            { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
+            opt[0].mlen = 1;
+            opt[0].litlen = litlen;

-        if (!last_pos && !match_num) { ip++; continue; }
+            /* large match -> immediate encoding */
+            {   U32 const maxML = matches[nbMatches-1].len;
+                DEBUGLOG(7, "found %u matches of maxLength=%u and offset=%u at cPos=%u => start new serie",
+                            nbMatches, maxML, matches[nbMatches-1].off, (U32)(ip-prefixStart));

-        if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) {
-            best_mlen = matches[match_num-1].len;
-            best_off = matches[match_num-1].off;
-            cur = 0;
-            last_pos = 1;
-            goto _storeSequence;
+                if (maxML > sufficient_len) {
+                    best_mlen = maxML;
+                    best_off = matches[nbMatches-1].off;
+                    DEBUGLOG(7, "large match (%u>%u), immediate encoding",
+                                best_mlen, sufficient_len);
+                    cur = 0;
+                    last_pos = 1;
+                    goto _shortestPath;
+            }   }
+
+            /* set prices for first matches starting position == 0 */
+            {   U32 const literalsPrice = ZSTD_fullLiteralsCost_cached(&cachedLitPrice, anchor, litlen, optStatePtr);
+                U32 pos;
+                U32 matchNb;
+                for (pos = 0; pos < minMatch; pos++) {
+                    opt[pos].mlen = 1;
+                    opt[pos].price = ZSTD_MAX_PRICE;
+                }
+                for (matchNb = 0; matchNb < nbMatches; matchNb++) {
+                    U32 const offset = matches[matchNb].off;
+                    U32 const end = matches[matchNb].len;
+                    repcodes_t const repHistory = ZSTD_updateRep(rep, offset, ll0);
+                    for ( ; pos <= end ; pos++ ) {
+                        U32 const matchPrice = literalsPrice + ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel);
+                        DEBUGLOG(7, "rPos:%u => set initial price : %u",
+                                    pos, matchPrice);
+                        opt[pos].mlen = pos;
+                        opt[pos].off = offset;
+                        opt[pos].litlen = litlen;
+                        opt[pos].price = matchPrice;
+                        memcpy(opt[pos].rep, &repHistory, sizeof(repHistory));
+                }   }
+                last_pos = pos-1;
+            }
        }

-        /* set prices using matches at position = 0 */
-        best_mlen = (last_pos) ? last_pos : minMatch;
-        for (u = 0; u < match_num; u++) {
-            mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
-            best_mlen = matches[u].len;
-            while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
-                if (mlen > last_pos || price < opt[mlen].price)
-                    SET_PRICE(mlen, mlen, matches[u].off, litlen, price);   /* note : macro modifies last_pos */
-                mlen++;
-        }   }
-
-        if (last_pos < minMatch) { ip++; continue; }
-
-        /* initialize opt[0] */
-        { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
-        opt[0].mlen = 1;
-        opt[0].litlen = litlen;
-
-         /* check further positions */
+        /* check further positions */
        for (cur = 1; cur <= last_pos; cur++) {
-           inr = ip + cur;
+            const BYTE* const inr = ip + cur;
+            assert(cur < ZSTD_OPT_NUM);

-           if (opt[cur-1].mlen == 1) {
-                litlen = opt[cur-1].litlen + 1;
+            /* Fix current position with one literal if cheaper */
+            {   U32 const litlen = (opt[cur-1].mlen == 1) ? opt[cur-1].litlen + 1 : 1;
+                int price;  /* note : contribution can be negative */
                if (cur > litlen) {
-                    price = opt[cur - litlen].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-litlen);
-                } else
-                    price = ZSTD_getLiteralPrice(optStatePtr, litlen, anchor);
-           } else {
-                litlen = 1;
-                price = opt[cur - 1].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-1);
-           }
+                    price = opt[cur - litlen].price + ZSTD_literalsContribution(inr-litlen, litlen, optStatePtr);
+                } else {
+                    price = ZSTD_literalsContribution_cached(&cachedLitPrice, anchor, litlen, optStatePtr);
+                }
+                assert(price < 1000000000); /* overflow check */
+                if (price <= opt[cur].price) {
+                    DEBUGLOG(7, "rPos:%u : better price (%u<%u) using literal",
+                                cur, price, opt[cur].price);
+                    opt[cur].mlen = 1;
+                    opt[cur].off = 0;
+                    opt[cur].litlen = litlen;
+                    opt[cur].price = price;
+                    memcpy(opt[cur].rep, opt[cur-1].rep, sizeof(opt[cur].rep));
+            }   }

-           if (cur > last_pos || price <= opt[cur].price)
-                SET_PRICE(cur, 1, 0, litlen, price);
+            /* last match must start at a minimum distance of 8 from oend */
+            if (inr > ilimit) continue;

-           if (cur == last_pos) break;
+            if (cur == last_pos) break;

-           if (inr > ilimit)  /* last match must start at a minimum distance of 8 from oend */
-               continue;
+             if ( (optLevel==0) /*static*/
+               && (opt[cur+1].price <= opt[cur].price) )
+                continue;  /* skip unpromising positions; about ~+6% speed, -0.01 ratio */

-           mlen = opt[cur].mlen;
-           if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
-                opt[cur].rep[2] = opt[cur-mlen].rep[1];
-                opt[cur].rep[1] = opt[cur-mlen].rep[0];
-                opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
-           } else {
-                opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
-                opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
-                opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
-           }
+            {   U32 const ll0 = (opt[cur].mlen != 1);
+                U32 const litlen = (opt[cur].mlen == 1) ? opt[cur].litlen : 0;
+                U32 const previousPrice = (cur > litlen) ? opt[cur-litlen].price : 0;
+                U32 const basePrice = previousPrice + ZSTD_fullLiteralsCost(inr-litlen, litlen, optStatePtr);
+                U32 const nbMatches = ZSTD_BtGetAllMatches(ctx, inr, iend, extDict, maxSearches, mls, sufficient_len, opt[cur].rep, ll0, matches, minMatch);
+                U32 matchNb;
+                if (!nbMatches) continue;

-            best_mlen = minMatch;
-            {   U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
-                for (i=(opt[cur].mlen != 1); i<last_i; i++) {  /* check rep */
-                    const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
-                    if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart))
-                       && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
-                       mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch;
+                {   U32 const maxML = matches[nbMatches-1].len;
+                    DEBUGLOG(7, "rPos:%u, found %u matches, of maxLength=%u",
+                                cur, nbMatches, maxML);

-                       if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
-                            best_mlen = mlen; best_off = i; last_pos = cur + 1;
-                            goto _storeSequence;
-                       }
-
-                       best_off = i - (opt[cur].mlen != 1);
-                       if (mlen > best_mlen) best_mlen = mlen;
-
-                       do {
-                           if (opt[cur].mlen == 1) {
-                                litlen = opt[cur].litlen;
-                                if (cur > litlen) {
-                                    price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
-                                } else
-                                    price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-                            } else {
-                                litlen = 0;
-                                price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
-                            }
-
-                            if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
-                                SET_PRICE(cur + mlen, mlen, i, litlen, price);
-                            mlen--;
-                        } while (mlen >= minMatch);
-            }   }   }
-
-            match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
-
-            if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
-                best_mlen = matches[match_num-1].len;
-                best_off = matches[match_num-1].off;
-                last_pos = cur + 1;
-                goto _storeSequence;
-            }
-
-            /* set prices using matches at position = cur */
-            for (u = 0; u < match_num; u++) {
-                mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
-                best_mlen = matches[u].len;
-
-                while (mlen <= best_mlen) {
-                    if (opt[cur].mlen == 1) {
-                        litlen = opt[cur].litlen;
-                        if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
-                        else
-                            price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
-                    } else {
-                        litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
+                    if ( (maxML > sufficient_len)
+                       | (cur + maxML >= ZSTD_OPT_NUM) ) {
+                        best_mlen = maxML;
+                        best_off = matches[nbMatches-1].off;
+                        last_pos = cur + 1;
+                        goto _shortestPath;
                    }
+                }

-                    if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
-                        SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
+                /* set prices using matches found at position == cur */
+                for (matchNb = 0; matchNb < nbMatches; matchNb++) {
+                    U32 const offset = matches[matchNb].off;
+                    repcodes_t const repHistory = ZSTD_updateRep(opt[cur].rep, offset, ll0);
+                    U32 const lastML = matches[matchNb].len;
+                    U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
+                    U32 mlen;

-                    mlen++;
-        }   }   }
+                    DEBUGLOG(7, "testing match %u => offCode=%u, mlen=%u, llen=%u",
+                                matchNb, matches[matchNb].off, lastML, litlen);
+
+                    for (mlen = lastML; mlen >= startML; mlen--) {
+                        U32 const pos = cur + mlen;
+                        int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
+
+                        if ((pos > last_pos) || (price < opt[pos].price)) {
+                            DEBUGLOG(7, "rPos:%u => new better price (%u<%u)",
+                                        pos, price, opt[pos].price);
+                            while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; }
+                            opt[pos].mlen = mlen;
+                            opt[pos].off = offset;
+                            opt[pos].litlen = litlen;
+                            opt[pos].price = price;
+                            memcpy(opt[pos].rep, &repHistory, sizeof(repHistory));
+                        } else {
+                            if (optLevel==0) break;  /* gets ~+10% speed for about -0.01 ratio loss */
+                        }
+            }   }   }
+        }  /* for (cur = 1; cur <= last_pos; cur++) */

        best_mlen = opt[last_pos].mlen;
        best_off = opt[last_pos].off;
        cur = last_pos - best_mlen;

-        /* store sequence */
-_storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
-        opt[0].mlen = 1;
+_shortestPath:   /* cur, last_pos, best_mlen, best_off have to be set */
+        assert(opt[0].mlen == 1);

-        while (1) {
-            mlen = opt[cur].mlen;
-            offset = opt[cur].off;
-            opt[cur].mlen = best_mlen;
-            opt[cur].off = best_off;
-            best_mlen = mlen;
-            best_off = offset;
-            if (mlen > cur) break;
-            cur -= mlen;
-        }
+        /* reverse traversal */
+        DEBUGLOG(7, "start reverse traversal (last_pos:%u, cur:%u)",
+                    last_pos, cur);
+        {   U32 selectedMatchLength = best_mlen;
+            U32 selectedOffset = best_off;
+            U32 pos = cur;
+            while (1) {
+                U32 const mlen = opt[pos].mlen;
+                U32 const off = opt[pos].off;
+                opt[pos].mlen = selectedMatchLength;
+                opt[pos].off = selectedOffset;
+                selectedMatchLength = mlen;
+                selectedOffset = off;
+                if (mlen > pos) break;
+                pos -= mlen;
+        }   }

-        for (u = 0; u <= last_pos;) {
-            u += opt[u].mlen;
-        }
+        /* save sequences */
+        {   U32 pos;
+            for (pos=0; pos < last_pos; ) {
+                U32 const llen = (U32)(ip - anchor);
+                U32 const mlen = opt[pos].mlen;
+                U32 const offset = opt[pos].off;
+                if (mlen == 1) { ip++; pos++; continue; }  /* literal position => move on */
+                pos += mlen; ip += mlen;

-        for (cur=0; cur < last_pos; ) {
-            mlen = opt[cur].mlen;
-            if (mlen == 1) { ip++; cur++; continue; }
-            offset = opt[cur].off;
-            cur += mlen;
-            litLength = (U32)(ip - anchor);
-
-            if (offset > ZSTD_REP_MOVE_OPT) {
-                rep[2] = rep[1];
-                rep[1] = rep[0];
-                rep[0] = offset - ZSTD_REP_MOVE_OPT;
-                offset--;
-            } else {
-                if (offset != 0) {
-                    best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
-                    if (offset != 1) rep[2] = rep[1];
+                /* repcodes update : like ZSTD_updateRep(), but update in place */
+                if (offset >= ZSTD_REP_NUM) {  /* full offset */
+                    rep[2] = rep[1];
                    rep[1] = rep[0];
-                    rep[0] = best_off;
+                    rep[0] = offset - ZSTD_REP_MOVE;
+                } else {   /* repcode */
+                    U32 const repCode = offset + (llen==0);
+                    if (repCode) {  /* note : if repCode==0, no change */
+                        U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
+                        if (repCode >= 2) rep[2] = rep[1];
+                        rep[1] = rep[0];
+                        rep[0] = currentOffset;
+                    }
                }
-                if (litLength==0) offset--;
-            }

-            ZSTD_updatePrice(optStatePtr, litLength, anchor, offset, mlen-MINMATCH);
-            ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
-            anchor = ip = ip + mlen;
-    }    }   /* for (cur=0; cur < last_pos; ) */
+                ZSTD_updateStats(optStatePtr, llen, anchor, offset, mlen);
+                ZSTD_storeSeq(seqStorePtr, llen, anchor, offset, mlen-MINMATCH);
+                anchor = ip;
+        }   }
+        ZSTD_setLog2Prices(optStatePtr);
+    }   /* while (ip < ilimit) */

    /* Save reps for next block */
    { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->repToConfirm[i] = rep[i]; }
@@ -658,292 +765,21 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */

 size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
+    DEBUGLOG(5, "ZSTD_compressBlock_btopt");
+    return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0 /*optLevel*/, 0 /*extDict*/);
 }

 size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
+    return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/);
 }

-
-FORCE_INLINE_TEMPLATE
-size_t ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
-                                     const void* src, size_t srcSize, const int ultra)
-{
-    seqStore_t* seqStorePtr = &(ctx->seqStore);
-    optState_t* optStatePtr = &(ctx->optState);
-    const BYTE* const istart = (const BYTE*)src;
-    const BYTE* ip = istart;
-    const BYTE* anchor = istart;
-    const BYTE* const iend = istart + srcSize;
-    const BYTE* const ilimit = iend - 8;
-    const BYTE* const base = ctx->base;
-    const U32 lowestIndex = ctx->lowLimit;
-    const U32 dictLimit = ctx->dictLimit;
-    const BYTE* const prefixStart = base + dictLimit;
-    const BYTE* const dictBase = ctx->dictBase;
-    const BYTE* const dictEnd  = dictBase + dictLimit;
-
-    const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
-    const U32 sufficient_len = ctx->appliedParams.cParams.targetLength;
-    const U32 mls = ctx->appliedParams.cParams.searchLength;
-    const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
-
-    ZSTD_optimal_t* opt = optStatePtr->priceTable;
-    ZSTD_match_t* matches = optStatePtr->matchTable;
-    const BYTE* inr;
-
-    /* init */
-    U32 offset, rep[ZSTD_REP_NUM];
-    { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
-
-    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize);
-    ip += (ip==prefixStart);
-
-    /* Match Loop */
-    while (ip < ilimit) {
-        U32 cur, match_num, last_pos, litlen, price;
-        U32 u, mlen, best_mlen, best_off, litLength;
-        U32 current = (U32)(ip-base);
-        memset(opt, 0, sizeof(ZSTD_optimal_t));
-        last_pos = 0;
-        opt[0].litlen = (U32)(ip - anchor);
-
-        /* check repCode */
-        {   U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
-            for (i = (ip==anchor); i<last_i; i++) {
-                const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
-                const U32 repIndex = (U32)(current - repCur);
-                const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
-                const BYTE* const repMatch = repBase + repIndex;
-                if ( (repCur > 0 && repCur <= (S32)current)
-                   && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex))  /* intentional overflow */
-                   && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
-                    /* repcode detected we should take it */
-                    const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
-                    mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
-
-                    if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
-                        best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
-                        goto _storeSequence;
-                    }
-
-                    best_off = i - (ip==anchor);
-                    litlen = opt[0].litlen;
-                    do {
-                        price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-                        if (mlen > last_pos || price < opt[mlen].price)
-                            SET_PRICE(mlen, mlen, i, litlen, price);   /* note : macro modifies last_pos */
-                        mlen--;
-                    } while (mlen >= minMatch);
-        }   }   }
-
-        match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch);  /* first search (depth 0) */
-
-        if (!last_pos && !match_num) { ip++; continue; }
-
-        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
-        opt[0].mlen = 1;
-
-        if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) {
-            best_mlen = matches[match_num-1].len;
-            best_off = matches[match_num-1].off;
-            cur = 0;
-            last_pos = 1;
-            goto _storeSequence;
-        }
-
-        best_mlen = (last_pos) ? last_pos : minMatch;
-
-        /* set prices using matches at position = 0 */
-        for (u = 0; u < match_num; u++) {
-            mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
-            best_mlen = matches[u].len;
-            litlen = opt[0].litlen;
-            while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
-                if (mlen > last_pos || price < opt[mlen].price)
-                    SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
-                mlen++;
-        }   }
-
-        if (last_pos < minMatch) {
-            ip++; continue;
-        }
-
-        /* check further positions */
-        for (cur = 1; cur <= last_pos; cur++) {
-            inr = ip + cur;
-
-            if (opt[cur-1].mlen == 1) {
-                litlen = opt[cur-1].litlen + 1;
-                if (cur > litlen) {
-                    price = opt[cur - litlen].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-litlen);
-                } else
-                    price = ZSTD_getLiteralPrice(optStatePtr, litlen, anchor);
-            } else {
-                litlen = 1;
-                price = opt[cur - 1].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-1);
-            }
-
-            if (cur > last_pos || price <= opt[cur].price)
-                SET_PRICE(cur, 1, 0, litlen, price);
-
-            if (cur == last_pos) break;
-
-            if (inr > ilimit)  /* last match must start at a minimum distance of 8 from oend */
-                continue;
-
-            mlen = opt[cur].mlen;
-            if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
-                opt[cur].rep[2] = opt[cur-mlen].rep[1];
-                opt[cur].rep[1] = opt[cur-mlen].rep[0];
-                opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
-            } else {
-                opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
-                opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
-                opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
-            }
-
-            best_mlen = minMatch;
-            {   U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
-                for (i = (mlen != 1); i<last_i; i++) {
-                    const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
-                    const U32 repIndex = (U32)(current+cur - repCur);
-                    const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
-                    const BYTE* const repMatch = repBase + repIndex;
-                    if ( (repCur > 0 && repCur <= (S32)(current+cur))
-                      && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex))  /* intentional overflow */
-                      && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
-                        /* repcode detected */
-                        const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
-                        mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
-
-                        if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
-                            best_mlen = mlen; best_off = i; last_pos = cur + 1;
-                            goto _storeSequence;
-                        }
-
-                        best_off = i - (opt[cur].mlen != 1);
-                        if (mlen > best_mlen) best_mlen = mlen;
-
-                        do {
-                            if (opt[cur].mlen == 1) {
-                                litlen = opt[cur].litlen;
-                                if (cur > litlen) {
-                                    price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
-                                } else
-                                    price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-                            } else {
-                                litlen = 0;
-                                price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
-                            }
-
-                            if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
-                                SET_PRICE(cur + mlen, mlen, i, litlen, price);
-                            mlen--;
-                        } while (mlen >= minMatch);
-            }   }   }
-
-            match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
-
-            if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
-                best_mlen = matches[match_num-1].len;
-                best_off = matches[match_num-1].off;
-                last_pos = cur + 1;
-                goto _storeSequence;
-            }
-
-            /* set prices using matches at position = cur */
-            for (u = 0; u < match_num; u++) {
-                mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
-                best_mlen = matches[u].len;
-
-                while (mlen <= best_mlen) {
-                    if (opt[cur].mlen == 1) {
-                        litlen = opt[cur].litlen;
-                        if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
-                        else
-                            price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
-                    } else {
-                        litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
-                    }
-
-                    if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
-                        SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
-
-                    mlen++;
-        }   }   }   /* for (cur = 1; cur <= last_pos; cur++) */
-
-        best_mlen = opt[last_pos].mlen;
-        best_off = opt[last_pos].off;
-        cur = last_pos - best_mlen;
-
-        /* store sequence */
-_storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
-        opt[0].mlen = 1;
-
-        while (1) {
-            mlen = opt[cur].mlen;
-            offset = opt[cur].off;
-            opt[cur].mlen = best_mlen;
-            opt[cur].off = best_off;
-            best_mlen = mlen;
-            best_off = offset;
-            if (mlen > cur) break;
-            cur -= mlen;
-        }
-
-        for (u = 0; u <= last_pos; ) {
-            u += opt[u].mlen;
-        }
-
-        for (cur=0; cur < last_pos; ) {
-            mlen = opt[cur].mlen;
-            if (mlen == 1) { ip++; cur++; continue; }
-            offset = opt[cur].off;
-            cur += mlen;
-            litLength = (U32)(ip - anchor);
-
-            if (offset > ZSTD_REP_MOVE_OPT) {
-                rep[2] = rep[1];
-                rep[1] = rep[0];
-                rep[0] = offset - ZSTD_REP_MOVE_OPT;
-                offset--;
-            } else {
-                if (offset != 0) {
-                    best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
-                    if (offset != 1) rep[2] = rep[1];
-                    rep[1] = rep[0];
-                    rep[0] = best_off;
-                }
-
-                if (litLength==0) offset--;
-            }
-
-            ZSTD_updatePrice(optStatePtr, litLength, anchor, offset, mlen-MINMATCH);
-            ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
-            anchor = ip = ip + mlen;
-    }    }   /* for (cur=0; cur < last_pos; ) */
-
-    /* Save reps for next block */
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->repToConfirm[i] = rep[i]; }
-
-    /* Return the last literals size */
-    return iend - anchor;
-}
-
-
 size_t ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
+    return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0 /*optLevel*/, 1 /*extDict*/);
 }

 size_t ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
+    return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2 /*optLevel*/, 1 /*extDict*/);
 }
@@ -11,12 +11,12 @@
 #ifndef ZSTD_OPT_H
 #define ZSTD_OPT_H

-#include "zstd_compress.h"
-
 #if defined (__cplusplus)
 extern "C" {
 #endif

+#include "zstd.h"   /* ZSTD_CCtx, size_t */
+
 size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
 size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize);

@@ -10,7 +10,7 @@


 /* ======   Tuning parameters   ====== */
-#define ZSTDMT_NBTHREADS_MAX 256
+#define ZSTDMT_NBTHREADS_MAX 200
 #define ZSTDMT_OVERLAPLOG_DEFAULT 6


@@ -24,7 +24,7 @@
 #include <string.h>      /* memcpy, memset */
 #include "pool.h"        /* threadpool */
 #include "threading.h"   /* mutex */
-#include "zstd_internal.h"  /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
+#include "zstd_compress_internal.h"  /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
 #include "zstdmt_compress.h"


@@ -53,22 +53,24 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
 }

 #define MUTEX_WAIT_TIME_DLEVEL 6
-#define PTHREAD_MUTEX_LOCK(mutex) {               \
-    if (ZSTD_DEBUG>=MUTEX_WAIT_TIME_DLEVEL) {   \
+#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) {          \
+    if (ZSTD_DEBUG >= MUTEX_WAIT_TIME_DLEVEL) {   \
        unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
-        pthread_mutex_lock(mutex);                \
+        ZSTD_pthread_mutex_lock(mutex);           \
        {   unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
            unsigned long long const elapsedTime = (afterTime-beforeTime); \
            if (elapsedTime > 1000) {  /* or whatever threshold you like; I'm using 1 millisecond here */ \
                DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
                   elapsedTime, #mutex);          \
        }   }                                     \
-    } else pthread_mutex_lock(mutex);             \
+    } else {                                      \
+        ZSTD_pthread_mutex_lock(mutex);           \
+    }                                             \
 }

 #else

-#  define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
+#  define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m)
 #  define DEBUG_PRINTHEX(l,p,n) {}

 #endif
@@ -85,7 +87,7 @@ typedef struct buffer_s {
 static const buffer_t g_nullBuffer = { NULL, 0 };

 typedef struct ZSTDMT_bufferPool_s {
-    pthread_mutex_t poolMutex;
+    ZSTD_pthread_mutex_t poolMutex;
    size_t bufferSize;
    unsigned totalBuffers;
    unsigned nbBuffers;
@@ -99,7 +101,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_custo
    ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
        sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
    if (bufPool==NULL) return NULL;
-    if (pthread_mutex_init(&bufPool->poolMutex, NULL)) {
+    if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) {
        ZSTD_free(bufPool, cMem);
        return NULL;
    }
@@ -113,10 +115,13 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_custo
 static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
 {
    unsigned u;
+    DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool);
    if (!bufPool) return;   /* compatibility with free on NULL */
-    for (u=0; u<bufPool->totalBuffers; u++)
+    for (u=0; u<bufPool->totalBuffers; u++) {
+        DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start);
        ZSTD_free(bufPool->bTable[u].start, bufPool->cMem);
-    pthread_mutex_destroy(&bufPool->poolMutex);
+    }
+    ZSTD_pthread_mutex_destroy(&bufPool->poolMutex);
    ZSTD_free(bufPool, bufPool->cMem);
 }

@@ -127,17 +132,20 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
                            + (bufPool->totalBuffers - 1) * sizeof(buffer_t);
    unsigned u;
    size_t totalBufferSize = 0;
-    pthread_mutex_lock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
    for (u=0; u<bufPool->totalBuffers; u++)
        totalBufferSize += bufPool->bTable[u].size;
-    pthread_mutex_unlock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);

    return poolSize + totalBufferSize;
 }

-static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize)
+static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize)
 {
+    ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
+    DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize);
    bufPool->bufferSize = bSize;
+    ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
 }

 /** ZSTDMT_getBuffer() :
@@ -145,27 +153,31 @@ static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize)
 static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
 {
    size_t const bSize = bufPool->bufferSize;
-    DEBUGLOG(5, "ZSTDMT_getBuffer");
-    pthread_mutex_lock(&bufPool->poolMutex);
+    DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize);
+    ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
    if (bufPool->nbBuffers) {   /* try to use an existing buffer */
        buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)];
        size_t const availBufferSize = buf.size;
-        if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) {
+        bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer;
+        if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) {
            /* large enough, but not too much */
-            pthread_mutex_unlock(&bufPool->poolMutex);
+            DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u",
+                        bufPool->nbBuffers, (U32)buf.size);
+            ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
            return buf;
        }
        /* size conditions not respected : scratch this buffer, create new one */
-        DEBUGLOG(5, "existing buffer does not meet size conditions => freeing");
+        DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing");
        ZSTD_free(buf.start, bufPool->cMem);
    }
-    pthread_mutex_unlock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
    /* create new buffer */
-    DEBUGLOG(5, "create a new buffer");
+    DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer");
    {   buffer_t buffer;
        void* const start = ZSTD_malloc(bSize, bufPool->cMem);
        buffer.start = start;   /* note : start can be NULL if malloc fails ! */
        buffer.size = (start==NULL) ? 0 : bSize;
+        DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize);
        return buffer;
    }
 }
@@ -175,15 +187,17 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
 {
    if (buf.start == NULL) return;   /* compatible with release on NULL */
    DEBUGLOG(5, "ZSTDMT_releaseBuffer");
-    pthread_mutex_lock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
    if (bufPool->nbBuffers < bufPool->totalBuffers) {
        bufPool->bTable[bufPool->nbBuffers++] = buf;  /* stored for later use */
-        pthread_mutex_unlock(&bufPool->poolMutex);
+        DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u",
+                    (U32)buf.size, (U32)(bufPool->nbBuffers-1));
+        ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
        return;
    }
-    pthread_mutex_unlock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
    /* Reached bufferPool capacity (should not happen) */
-    DEBUGLOG(5, "buffer pool capacity reached => freeing ");
+    DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing ");
    ZSTD_free(buf.start, bufPool->cMem);
 }

@@ -206,7 +220,7 @@ static ZSTD_CCtx_params ZSTDMT_makeJobCCtxParams(ZSTD_CCtx_params const params)
 /* a single CCtx Pool can be invoked from multiple threads in parallel */

 typedef struct {
-    pthread_mutex_t poolMutex;
+    ZSTD_pthread_mutex_t poolMutex;
    unsigned totalCCtx;
    unsigned availCCtx;
    ZSTD_customMem cMem;
@@ -219,7 +233,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
    unsigned u;
    for (u=0; u<pool->totalCCtx; u++)
        ZSTD_freeCCtx(pool->cctx[u]);  /* note : compatible with free on NULL */
-    pthread_mutex_destroy(&pool->poolMutex);
+    ZSTD_pthread_mutex_destroy(&pool->poolMutex);
    ZSTD_free(pool, pool->cMem);
 }

@@ -231,7 +245,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
    ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
        sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
    if (!cctxPool) return NULL;
-    if (pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
+    if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
        ZSTD_free(cctxPool, cMem);
        return NULL;
    }
@@ -247,7 +261,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
 /* only works during initialization phase, not during compression */
 static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
 {
-    pthread_mutex_lock(&cctxPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
    {   unsigned const nbThreads = cctxPool->totalCCtx;
        size_t const poolSize = sizeof(*cctxPool)
                                + (nbThreads-1)*sizeof(ZSTD_CCtx*);
@@ -256,7 +270,7 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
        for (u=0; u<nbThreads; u++) {
            totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
        }
-        pthread_mutex_unlock(&cctxPool->poolMutex);
+        ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
        return poolSize + totalCCtxSize;
    }
 }
@@ -264,14 +278,14 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
 static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
 {
    DEBUGLOG(5, "ZSTDMT_getCCtx");
-    pthread_mutex_lock(&cctxPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
    if (cctxPool->availCCtx) {
        cctxPool->availCCtx--;
        {   ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx];
-            pthread_mutex_unlock(&cctxPool->poolMutex);
+            ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
            return cctx;
    }   }
-    pthread_mutex_unlock(&cctxPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
    DEBUGLOG(5, "create one more CCtx");
    return ZSTD_createCCtx_advanced(cctxPool->cMem);   /* note : can be NULL, when creation fails ! */
 }
@@ -279,7 +293,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
 static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
 {
    if (cctx==NULL) return;   /* compatibility with release on NULL */
-    pthread_mutex_lock(&pool->poolMutex);
+    ZSTD_pthread_mutex_lock(&pool->poolMutex);
    if (pool->availCCtx < pool->totalCCtx)
        pool->cctx[pool->availCCtx++] = cctx;
    else {
@@ -287,7 +301,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
        DEBUGLOG(5, "CCtx pool overflow : free cctx");
        ZSTD_freeCCtx(cctx);
    }
-    pthread_mutex_unlock(&pool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&pool->poolMutex);
 }


@@ -296,7 +310,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
 typedef struct {
    buffer_t src;
    const void* srcStart;
-    size_t   dictSize;
+    size_t   prefixSize;
    size_t   srcSize;
    buffer_t dstBuff;
    size_t   cSize;
@@ -305,8 +319,8 @@ typedef struct {
    unsigned lastChunk;
    unsigned jobCompleted;
    unsigned jobScanned;
-    pthread_mutex_t* jobCompleted_mutex;
-    pthread_cond_t* jobCompleted_cond;
+    ZSTD_pthread_mutex_t* jobCompleted_mutex;
+    ZSTD_pthread_cond_t* jobCompleted_cond;
    ZSTD_CCtx_params params;
    const ZSTD_CDict* cdict;
    ZSTDMT_CCtxPool* cctxPool;
@@ -318,11 +332,11 @@ typedef struct {
 void ZSTDMT_compressChunk(void* jobDescription)
 {
    ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
-    ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
-    const void* const src = (const char*)job->srcStart + job->dictSize;
+    ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool);
+    const void* const src = (const char*)job->srcStart + job->prefixSize;
    buffer_t dstBuff = job->dstBuff;
-    DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
-                 job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
+    DEBUGLOG(5, "ZSTDMT_compressChunk: job (first:%u) (last:%u) : prefixSize %u, srcSize %u ",
+                 job->firstChunk, job->lastChunk, (U32)job->prefixSize, (U32)job->srcSize);

    if (cctx==NULL) {
        job->cSize = ERROR(memory_allocation);
@@ -336,48 +350,58 @@ void ZSTDMT_compressChunk(void* jobDescription)
            goto _endJob;
        }
        job->dstBuff = dstBuff;
+        DEBUGLOG(5, "ZSTDMT_compressChunk: received dstBuff of size %u", (U32)dstBuff.size);
    }

-    if (job->cdict) {  /* should only happen for first segment */
-        size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
-        DEBUGLOG(5, "using CDict");
+    if (job->cdict) {
+        size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dm_auto, job->cdict, job->params, job->fullFrameSize);
+        DEBUGLOG(4, "ZSTDMT_compressChunk: init using CDict (windowLog=%u)", job->params.cParams.windowLog);
+        assert(job->firstChunk);  /* only allowed for first job */
        if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
    } else {  /* srcStart points at reloaded section */
-        if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
-        { ZSTD_CCtx_params jobParams = job->params;
-          size_t const forceWindowError =
-              ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
-          /* Force loading dictionary in "content-only" mode (no header analysis) */
-          size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize);
-            if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
+        U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
+        ZSTD_CCtx_params jobParams = job->params;   /* do not modify job->params ! copy it, modify the copy */
+        size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
+        if (ZSTD_isError(forceWindowError)) {
+            DEBUGLOG(5, "ZSTD_CCtxParam_setParameter error : %s ", ZSTD_getErrorName(forceWindowError));
+            job->cSize = forceWindowError;
+            goto _endJob;
+        }
+        DEBUGLOG(5, "ZSTDMT_compressChunk: invoking ZSTD_compressBegin_advanced_internal with windowLog = %u ", jobParams.cParams.windowLog);
+        {   size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
+                                        job->srcStart, job->prefixSize, ZSTD_dm_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
+                                        NULL,
+                                        jobParams, pledgedSrcSize);
+            if (ZSTD_isError(initError)) {
+                DEBUGLOG(5, "ZSTD_compressBegin_advanced_internal error : %s ", ZSTD_getErrorName(initError));
                job->cSize = initError;
                goto _endJob;
-            }
-    }   }
-    if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
+        }   }
+    }
+    if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first job */
        size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
-        if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
+        if (ZSTD_isError(hSize)) { job->cSize = hSize; /* save error code */ goto _endJob; }
        ZSTD_invalidateRepCodes(cctx);
    }

-    DEBUGLOG(5, "Compressing : ");
-    DEBUG_PRINTHEX(4, job->srcStart, 12);
+    DEBUGLOG(5, "Compressing into dstBuff of size %u", (U32)dstBuff.size);
+    DEBUG_PRINTHEX(6, job->srcStart, 12);
    job->cSize = (job->lastChunk) ?
                 ZSTD_compressEnd     (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
                 ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
-    DEBUGLOG(5, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
+    DEBUGLOG(5, "compressed %u bytes into %u bytes   (first:%u) (last:%u) ",
                (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
-    DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
+    DEBUGLOG(5, "dstBuff.size : %u ; => %s ", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));

 _endJob:
    ZSTDMT_releaseCCtx(job->cctxPool, cctx);
    ZSTDMT_releaseBuffer(job->bufPool, job->src);
    job->src = g_nullBuffer; job->srcStart = NULL;
-    PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
+    ZSTD_PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
    job->jobCompleted = 1;
    job->jobScanned = 0;
-    pthread_cond_signal(job->jobCompleted_cond);
-    pthread_mutex_unlock(job->jobCompleted_mutex);
+    ZSTD_pthread_cond_signal(job->jobCompleted_cond);
+    ZSTD_pthread_mutex_unlock(job->jobCompleted_mutex);
 }


@@ -395,15 +419,16 @@ struct ZSTDMT_CCtx_s {
    ZSTDMT_jobDescription* jobs;
    ZSTDMT_bufferPool* bufPool;
    ZSTDMT_CCtxPool* cctxPool;
-    pthread_mutex_t jobCompleted_mutex;
-    pthread_cond_t jobCompleted_cond;
+    ZSTD_pthread_mutex_t jobCompleted_mutex;
+    ZSTD_pthread_cond_t jobCompleted_cond;
+    ZSTD_CCtx_params params;
    size_t targetSectionSize;
    size_t inBuffSize;
    size_t dictSize;
    size_t targetDictSize;
    inBuff_t inBuff;
-    ZSTD_CCtx_params params;
    XXH64_state_t xxhState;
+    unsigned singleThreaded;
    unsigned jobIDMask;
    unsigned doneJobID;
    unsigned nextJobID;
@@ -424,20 +449,32 @@ static ZSTDMT_jobDescription* ZSTDMT_allocJobsTable(U32* nbJobsPtr, ZSTD_customM
                            nbJobs * sizeof(ZSTDMT_jobDescription), cMem);
 }

-/* Internal only */
-size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads)
+/* ZSTDMT_CCtxParam_setNbThreads():
+ * Internal use only */
+size_t ZSTDMT_CCtxParam_setNbThreads(ZSTD_CCtx_params* params, unsigned nbThreads)
 {
+    if (nbThreads > ZSTDMT_NBTHREADS_MAX) nbThreads = ZSTDMT_NBTHREADS_MAX;
+    if (nbThreads < 1) nbThreads = 1;
    params->nbThreads = nbThreads;
    params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT;
    params->jobSize = 0;
-    return 0;
+    return nbThreads;
+}
+
+/* ZSTDMT_getNbThreads():
+ * @return nb threads currently active in mtctx.
+ * mtctx must be valid */
+size_t ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx)
+{
+    assert(mtctx != NULL);
+    return mtctx->params.nbThreads;
 }

 ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
 {
    ZSTDMT_CCtx* mtctx;
    U32 nbJobs = nbThreads + 2;
-    DEBUGLOG(3, "ZSTDMT_createCCtx_advanced");
+    DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbThreads = %u)", nbThreads);

    if (nbThreads < 1) return NULL;
    nbThreads = MIN(nbThreads , ZSTDMT_NBTHREADS_MAX);
@@ -447,7 +484,7 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)

    mtctx = (ZSTDMT_CCtx*) ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem);
    if (!mtctx) return NULL;
-    ZSTDMT_initializeCCtxParameters(&mtctx->params, nbThreads);
+    ZSTDMT_CCtxParam_setNbThreads(&mtctx->params, nbThreads);
    mtctx->cMem = cMem;
    mtctx->allJobsCompleted = 1;
    mtctx->factory = POOL_create_advanced(nbThreads, 0, cMem);
@@ -459,11 +496,11 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
        ZSTDMT_freeCCtx(mtctx);
        return NULL;
    }
-    if (pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) {
+    if (ZSTD_pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) {
        ZSTDMT_freeCCtx(mtctx);
        return NULL;
    }
-    if (pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) {
+    if (ZSTD_pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) {
        ZSTDMT_freeCCtx(mtctx);
        return NULL;
    }
@@ -483,28 +520,46 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
    unsigned jobID;
    DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
    for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
+        DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start);
        ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
        mtctx->jobs[jobID].dstBuff = g_nullBuffer;
+        DEBUGLOG(4, "job%02u: release src address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].src.start);
        ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].src);
        mtctx->jobs[jobID].src = g_nullBuffer;
    }
    memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
+    DEBUGLOG(4, "input: release address %08X", (U32)(size_t)mtctx->inBuff.buffer.start);
    ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->inBuff.buffer);
    mtctx->inBuff.buffer = g_nullBuffer;
    mtctx->allJobsCompleted = 1;
 }

+static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
+{
+    DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted");
+    while (zcs->doneJobID < zcs->nextJobID) {
+        unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
+        ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+        while (zcs->jobs[jobID].jobCompleted==0) {
+            DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID);   /* we want to block when waiting for data to flush */
+            ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
+        }
+        ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+        zcs->doneJobID++;
+    }
+}
+
 size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
 {
    if (mtctx==NULL) return 0;   /* compatible with free on NULL */
-    POOL_free(mtctx->factory);
-    if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
-    ZSTDMT_freeBufferPool(mtctx->bufPool);  /* release job resources into pools first */
+    POOL_free(mtctx->factory);   /* stop and free worker threads */
+    ZSTDMT_releaseAllJobResources(mtctx);  /* release job resources into pools first */
    ZSTD_free(mtctx->jobs, mtctx->cMem);
+    ZSTDMT_freeBufferPool(mtctx->bufPool);
    ZSTDMT_freeCCtxPool(mtctx->cctxPool);
    ZSTD_freeCDict(mtctx->cdictLocal);
-    pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
-    pthread_cond_destroy(&mtctx->jobCompleted_cond);
+    ZSTD_pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
+    ZSTD_pthread_cond_destroy(&mtctx->jobCompleted_cond);
    ZSTD_free(mtctx, mtctx->cMem);
    return 0;
 }
@@ -521,17 +576,23 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
 }

 /* Internal only */
-size_t ZSTDMT_CCtxParam_setMTCtxParameter(
-    ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value) {
+size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
+                                ZSTDMT_parameter parameter, unsigned value) {
+    DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter");
    switch(parameter)
    {
-    case ZSTDMT_p_sectionSize :
+    case ZSTDMT_p_jobSize :
+        DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value);
+        if ( (value > 0)  /* value==0 => automatic job size */
+           & (value < ZSTDMT_JOBSIZE_MIN) )
+            value = ZSTDMT_JOBSIZE_MIN;
        params->jobSize = value;
-        return 0;
+        return value;
    case ZSTDMT_p_overlapSectionLog :
+        if (value > 9) value = 9;
        DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
        params->overlapSizeLog = (value >= 9) ? 9 : value;
-        return 0;
+        return value;
    default :
        return ERROR(parameter_unsupported);
    }
@@ -539,9 +600,10 @@ size_t ZSTDMT_CCtxParam_setMTCtxParameter(

 size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value)
 {
+    DEBUGLOG(4, "ZSTDMT_setMTCtxParameter");
    switch(parameter)
    {
-    case ZSTDMT_p_sectionSize :
+    case ZSTDMT_p_jobSize :
        return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
    case ZSTDMT_p_overlapSectionLog :
        return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
@@ -577,7 +639,7 @@ static size_t ZSTDMT_compress_advanced_internal(
    size_t const overlapSize = (overlapRLog>=9) ? 0 : (size_t)1 << (params.cParams.windowLog - overlapRLog);
    unsigned nbChunks = computeNbChunks(srcSize, params.cParams.windowLog, params.nbThreads);
    size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
-    size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize;   /* avoid too small last block */
+    size_t const avgChunkSize = (((proposedChunkSize-1) & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize;   /* avoid too small last block */
    const char* const srcStart = (const char*)src;
    size_t remainingSrcSize = srcSize;
    unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize));  /* presumes avgChunkSize >= 256 KB, which should be the case */
@@ -586,7 +648,8 @@ static size_t ZSTDMT_compress_advanced_internal(
    assert(jobParams.nbThreads == 0);
    assert(mtctx->cctxPool->totalCCtx == params.nbThreads);

-    DEBUGLOG(4, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
+    DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: nbChunks=%2u (rawSize=%u bytes; fixedSize=%u)  ",
+                nbChunks, (U32)proposedChunkSize, (U32)avgChunkSize);
    if (nbChunks==1) {   /* fallback to single-thread mode */
        ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
        if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams);
@@ -615,9 +678,9 @@ static size_t ZSTDMT_compress_advanced_internal(

            mtctx->jobs[u].src = g_nullBuffer;
            mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
-            mtctx->jobs[u].dictSize = dictSize;
+            mtctx->jobs[u].prefixSize = dictSize;
            mtctx->jobs[u].srcSize = chunkSize;
-            mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL;
+            mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
            mtctx->jobs[u].fullFrameSize = srcSize;
            mtctx->jobs[u].params = jobParams;
            /* do not calculate checksum within sections, but write it in header for first section */
@@ -635,7 +698,7 @@ static size_t ZSTDMT_compress_advanced_internal(
                XXH64_update(&xxh64, srcStart + frameStartPos, chunkSize);
            }

-            DEBUGLOG(5, "posting job %u   (%u bytes)", u, (U32)chunkSize);
+            DEBUGLOG(5, "ZSTDMT_compress_advanced_internal: posting job %u  (%u bytes)", u, (U32)chunkSize);
            DEBUG_PRINTHEX(6, mtctx->jobs[u].srcStart, 12);
            POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);

@@ -649,12 +712,12 @@ static size_t ZSTDMT_compress_advanced_internal(
        unsigned chunkID;
        for (chunkID=0; chunkID<nbChunks; chunkID++) {
            DEBUGLOG(5, "waiting for chunk %u ", chunkID);
-            PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
+            ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
            while (mtctx->jobs[chunkID].jobCompleted==0) {
                DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", chunkID);
-                pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
+                ZSTD_pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
            }
-            pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
+            ZSTD_pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
            DEBUGLOG(5, "ready to write chunk %u ", chunkID);

            mtctx->jobs[chunkID].srcStart = NULL;
@@ -667,9 +730,8 @@ static size_t ZSTDMT_compress_advanced_internal(
                    if (chunkID >= compressWithinDst) {  /* chunk compressed into its own buffer, which must be released */
                        DEBUGLOG(5, "releasing buffer %u>=%u", chunkID, compressWithinDst);
                        ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[chunkID].dstBuff);
-                    }
-                    mtctx->jobs[chunkID].dstBuff = g_nullBuffer;
-                }
+                }   }
+                mtctx->jobs[chunkID].dstBuff = g_nullBuffer;
                dstPos += cSize ;
            }
        }  /* for (chunkID=0; chunkID<nbChunks; chunkID++) */
@@ -724,34 +786,20 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
 /* =======      Streaming API     ======= */
 /* ====================================== */

-static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
-{
-    DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted");
-    while (zcs->doneJobID < zcs->nextJobID) {
-        unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
-        PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
-        while (zcs->jobs[jobID].jobCompleted==0) {
-            DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID);   /* we want to block when waiting for data to flush */
-            pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
-        }
-        pthread_mutex_unlock(&zcs->jobCompleted_mutex);
-        zcs->doneJobID++;
-    }
-}
-
 size_t ZSTDMT_initCStream_internal(
        ZSTDMT_CCtx* zcs,
        const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
        const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
        unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(4, "ZSTDMT_initCStream_internal");
+    DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
    /* params are supposed to be fully validated at this point */
    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
    assert(zcs->cctxPool->totalCCtx == params.nbThreads);
+    zcs->singleThreaded = (params.nbThreads==1) | (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN);  /* do not trigger multi-threading when srcSize is too small */

-    if (params.nbThreads==1) {
+    if (zcs->singleThreaded) {
        ZSTD_CCtx_params const singleThreadParams = ZSTDMT_makeJobCCtxParams(params);
        DEBUGLOG(4, "single thread mode");
        assert(singleThreadParams.nbThreads == 0);
@@ -759,6 +807,7 @@ size_t ZSTDMT_initCStream_internal(
                                         dict, dictSize, cdict,
                                         singleThreadParams, pledgedSrcSize);
    }
+    DEBUGLOG(4, "multi-threading mode (%u threads)", params.nbThreads);

    if (zcs->allJobsCompleted == 0) {   /* previous compression not correctly finished */
        ZSTDMT_waitForAllJobsCompleted(zcs);
@@ -769,7 +818,6 @@ size_t ZSTDMT_initCStream_internal(
    zcs->params = params;
    zcs->frameContentSize = pledgedSrcSize;
    if (dict) {
-        DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
        ZSTD_freeCDict(zcs->cdictLocal);
        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
                                                    ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
@@ -777,20 +825,20 @@ size_t ZSTDMT_initCStream_internal(
        zcs->cdict = zcs->cdictLocal;
        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
    } else {
-        DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
        ZSTD_freeCDict(zcs->cdictLocal);
        zcs->cdictLocal = NULL;
        zcs->cdict = cdict;
    }

+    assert(params.overlapSizeLog <= 9);
    zcs->targetDictSize = (params.overlapSizeLog==0) ? 0 : (size_t)1 << (params.cParams.windowLog - (9 - params.overlapSizeLog));
-    DEBUGLOG(4, "overlapLog : %u ", params.overlapSizeLog);
-    DEBUGLOG(4, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
+    DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(zcs->targetDictSize>>10));
    zcs->targetSectionSize = params.jobSize ? params.jobSize : (size_t)1 << (params.cParams.windowLog + 2);
-    zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
-    zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
-    DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
+    if (zcs->targetSectionSize < ZSTDMT_JOBSIZE_MIN) zcs->targetSectionSize = ZSTDMT_JOBSIZE_MIN;
+    if (zcs->targetSectionSize < zcs->targetDictSize) zcs->targetSectionSize = zcs->targetDictSize;  /* job size must be >= overlap size */
+    DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(zcs->targetSectionSize>>10), params.jobSize);
    zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize;
+    DEBUGLOG(4, "inBuff Size : %u KB", (U32)(zcs->inBuffSize>>10));
    ZSTDMT_setBufferSize(zcs->bufPool, MAX(zcs->inBuffSize, ZSTD_compressBound(zcs->targetSectionSize)) );
    zcs->inBuff.buffer = g_nullBuffer;
    zcs->dictSize = 0;
@@ -808,7 +856,7 @@ size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
                                   unsigned long long pledgedSrcSize)
 {
    ZSTD_CCtx_params cctxParams = mtctx->params;
-    DEBUGLOG(5, "ZSTDMT_initCStream_advanced");
+    DEBUGLOG(5, "ZSTDMT_initCStream_advanced (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
    cctxParams.cParams = params.cParams;
    cctxParams.fParams = params.fParams;
    return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dm_auto, NULL,
@@ -830,9 +878,12 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,


 /* ZSTDMT_resetCStream() :
- * pledgedSrcSize is optional and can be zero == unknown */
+ * pledgedSrcSize can be zero == unknown (for the time being)
+ * prefer using ZSTD_CONTENTSIZE_UNKNOWN,
+ * as `0` might mean "empty" in the future */
 size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
 {
+    if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
    if (zcs->params.nbThreads==1)
        return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize);
    return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params,
@@ -844,7 +895,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
    ZSTD_CCtx_params cctxParams = zcs->params;
    cctxParams.cParams = params.cParams;
    cctxParams.fParams = params.fParams;
-    return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, 0);
+    return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
 }


@@ -852,12 +903,12 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
 {
    unsigned const jobID = zcs->nextJobID & zcs->jobIDMask;

-    DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ",
+    DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ",
                zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
    zcs->jobs[jobID].src = zcs->inBuff.buffer;
    zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
    zcs->jobs[jobID].srcSize = srcSize;
-    zcs->jobs[jobID].dictSize = zcs->dictSize;
+    zcs->jobs[jobID].prefixSize = zcs->dictSize;
    assert(zcs->inBuff.filled >= srcSize + zcs->dictSize);
    zcs->jobs[jobID].params = zcs->params;
    /* do not calculate checksum within sections, but write it in header for first section */
@@ -903,7 +954,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
            zcs->params.fParams.checksumFlag = 0;
    }   }

-    DEBUGLOG(4, "posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)",
+    DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)",
                zcs->nextJobID,
                (U32)zcs->jobs[jobID].srcSize,
                zcs->jobs[jobID].lastChunk,
@@ -922,19 +973,21 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
 static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush)
 {
    unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
+    DEBUGLOG(5, "ZSTDMT_flushNextJob");
    if (zcs->doneJobID == zcs->nextJobID) return 0;   /* all flushed ! */
-    PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+    ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
    while (zcs->jobs[wJobID].jobCompleted==0) {
        DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID);
-        if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; }  /* nothing ready to be flushed => skip */
-        pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);  /* block when nothing available to flush */
+        if (!blockToFlush) { ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; }  /* nothing ready to be flushed => skip */
+        ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);  /* block when nothing available to flush */
    }
-    pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+    ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex);
    /* compression job completed : output can be flushed */
    {   ZSTDMT_jobDescription job = zcs->jobs[wJobID];
        if (!job.jobScanned) {
            if (ZSTD_isError(job.cSize)) {
-                DEBUGLOG(5, "compression error detected ");
+                DEBUGLOG(5, "job %u : compression error detected : %s",
+                            zcs->doneJobID, ZSTD_getErrorName(job.cSize));
                ZSTDMT_waitForAllJobsCompleted(zcs);
                ZSTDMT_releaseAllJobResources(zcs);
                return job.cSize;
@@ -973,7 +1026,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi


 /** ZSTDMT_compressStream_generic() :
- *  internal use only
+ *  internal use only - exposed to be invoked from zstd_compress.c
 *  assumption : output and input are valid (pos <= size)
 * @return : minimum amount of data remaining to flush, 0 if none */
 size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
@@ -982,20 +1035,24 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
                                     ZSTD_EndDirective endOp)
 {
    size_t const newJobThreshold = mtctx->dictSize + mtctx->targetSectionSize;
+    unsigned forwardInputProgress = 0;
+    DEBUGLOG(5, "ZSTDMT_compressStream_generic ");
    assert(output->pos <= output->size);
    assert(input->pos  <= input->size);
-    if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
-        /* current frame being ended. Only flush/end are allowed. Or start new frame with init */
-        return ERROR(stage_wrong);
-    }
-    if (mtctx->params.nbThreads==1) {  /* delegate to single-thread (synchronous) */
+
+    if (mtctx->singleThreaded) {  /* delegate to single-thread (synchronous) */
        return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp);
    }

-    /* single-pass shortcut (note : this is synchronous-mode) */
-    if ( (mtctx->nextJobID==0)      /* just started */
-      && (mtctx->inBuff.filled==0)  /* nothing buffered */
-      && (endOp==ZSTD_e_end)        /* end order */
+    if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
+        /* current frame being ended. Only flush/end are allowed */
+        return ERROR(stage_wrong);
+    }
+
+    /* single-pass shortcut (note : synchronous-mode) */
+    if ( (mtctx->nextJobID == 0)     /* just started */
+      && (mtctx->inBuff.filled == 0) /* nothing buffered */
+      && (endOp == ZSTD_e_end)       /* end order */
      && (output->size - output->pos >= ZSTD_compressBound(input->size - input->pos)) ) { /* enough room */
        size_t const cSize = ZSTDMT_compress_advanced_internal(mtctx,
                (char*)output->dst + output->pos, output->size - output->pos,
@@ -1013,15 +1070,16 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
    /* fill input buffer */
    if (input->size > input->pos) {   /* support NULL input */
        if (mtctx->inBuff.buffer.start == NULL) {
-            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->bufPool);
-            if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
+            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->bufPool);  /* note : may fail, in which case, no forward input progress */
            mtctx->inBuff.filled = 0;
        }
-        {   size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
+        if (mtctx->inBuff.buffer.start) {
+            size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
            DEBUGLOG(5, "inBuff:%08X;  inBuffSize=%u;  ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
            memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
            input->pos += toLoad;
            mtctx->inBuff.filled += toLoad;
+            forwardInputProgress = toLoad>0;
    }   }

    if ( (mtctx->inBuff.filled >= newJobThreshold)  /* filled enough : let's compress */
@@ -1030,7 +1088,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
    }

    /* check for potential compressed data ready to be flushed */
-    CHECK_F( ZSTDMT_flushNextJob(mtctx, output, (mtctx->inBuff.filled == mtctx->inBuffSize) /* blockToFlush */) ); /* block if it wasn't possible to create new job due to saturation */
+    CHECK_F( ZSTDMT_flushNextJob(mtctx, output, !forwardInputProgress /* blockToFlush */) ); /* block if there was no forward input progress */

    if (input->pos < input->size)  /* input not consumed : do not flush yet */
        endOp = ZSTD_e_continue;
@@ -1058,32 +1116,34 @@ size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBu
 }


-static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned endFrame)
+static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned endFrame)
 {
-    size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
+    size_t const srcSize = mtctx->inBuff.filled - mtctx->dictSize;
+    DEBUGLOG(5, "ZSTDMT_flushStream_internal");

-    if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
-       && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
-        CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) );
+    if ( ((srcSize > 0) || (endFrame && !mtctx->frameEnded))
+       && (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) {
+           DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job");
+        CHECK_F( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
    }

    /* check if there is any data available to flush */
-    return ZSTDMT_flushNextJob(zcs, output, 1 /* blockToFlush */);
+    return ZSTDMT_flushNextJob(mtctx, output, 1 /* blockToFlush */);
 }


-size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output)
 {
    DEBUGLOG(5, "ZSTDMT_flushStream");
-    if (zcs->params.nbThreads==1)
-        return ZSTD_flushStream(zcs->cctxPool->cctx[0], output);
-    return ZSTDMT_flushStream_internal(zcs, output, 0 /* endFrame */);
+    if (mtctx->singleThreaded)
+        return ZSTD_flushStream(mtctx->cctxPool->cctx[0], output);
+    return ZSTDMT_flushStream_internal(mtctx, output, 0 /* endFrame */);
 }

-size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output)
 {
    DEBUGLOG(4, "ZSTDMT_endStream");
-    if (zcs->params.nbThreads==1)
-        return ZSTD_endStream(zcs->cctxPool->cctx[0], output);
-    return ZSTDMT_flushStream_internal(zcs, output, 1 /* endFrame */);
+    if (mtctx->singleThreaded)
+        return ZSTD_endStream(mtctx->cctxPool->cctx[0], output);
+    return ZSTDMT_flushStream_internal(mtctx, output, 1 /* endFrame */);
 }
@@ -50,7 +50,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
 /* ===   Streaming functions   === */

 ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
-ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);    /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);  /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */

 ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);

@@ -60,8 +60,8 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);

 /* ===   Advanced functions and parameters  === */

-#ifndef ZSTDMT_SECTION_SIZE_MIN
-#  define ZSTDMT_SECTION_SIZE_MIN (1U << 20)   /* 1 MB - Minimum size of each compression job */
+#ifndef ZSTDMT_JOBSIZE_MIN
+#  define ZSTDMT_JOBSIZE_MIN (1U << 20)   /* 1 MB - Minimum size of each compression job */
 #endif

 ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
@@ -84,13 +84,13 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
 /* ZSTDMT_parameter :
 * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
 typedef enum {
-    ZSTDMT_p_sectionSize,        /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
-    ZSTDMT_p_overlapSectionLog   /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
+    ZSTDMT_p_jobSize,           /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
+    ZSTDMT_p_overlapSectionLog  /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
 } ZSTDMT_parameter;

 /* ZSTDMT_setMTCtxParameter() :
 * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
- * The function must be called typically after ZSTD_createCCtx().
+ * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
 * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
 * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
@@ -112,7 +112,15 @@ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,

 size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);

-size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads);
+/* ZSTDMT_CCtxParam_setNbThreads()
+ * Set nbThreads, and clamp it correctly,
+ * also reset jobSize and overlapLog */
+size_t ZSTDMT_CCtxParam_setNbThreads(ZSTD_CCtx_params* params, unsigned nbThreads);
+
+/* ZSTDMT_getNbThreads():
+ * @return nb threads currently active in mtctx.
+ * mtctx must be valid */
+size_t ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx);

 /*! ZSTDMT_initCStream_internal() :
 *  Private use only. Init streaming operation.
@@ -35,7 +35,7 @@
 *  Frames requiring more memory will be rejected.
 */
 #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-#  define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1)   /* defined within zstd.h */
+#  define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
 #endif


@@ -110,6 +110,7 @@ struct ZSTD_DCtx_s
    XXH64_state_t xxhState;
    size_t headerSize;
    U32 dictID;
+    ZSTD_format_e format;
    const BYTE* litPtr;
    ZSTD_customMem customMem;
    size_t litSize;
@@ -149,30 +150,21 @@ size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)

 size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }

-size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
+
+static size_t ZSTD_startingInputLength(ZSTD_format_e format)
 {
-    dctx->expected = ZSTD_frameHeaderSize_prefix;
-    dctx->stage = ZSTDds_getFrameHeaderSize;
-    dctx->decodedSize = 0;
-    dctx->previousDstEnd = NULL;
-    dctx->base = NULL;
-    dctx->vBase = NULL;
-    dctx->dictEnd = NULL;
-    dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
-    dctx->litEntropy = dctx->fseEntropy = 0;
-    dctx->dictID = 0;
-    MEM_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
-    memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
-    dctx->LLTptr = dctx->entropy.LLTable;
-    dctx->MLTptr = dctx->entropy.MLTable;
-    dctx->OFTptr = dctx->entropy.OFTable;
-    dctx->HUFptr = dctx->entropy.hufTable;
-    return 0;
+    size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
+                    ZSTD_frameHeaderSize_prefix - ZSTD_frameIdSize :
+                    ZSTD_frameHeaderSize_prefix;
+    ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
+    /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
+    assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
+    return startingInputLength;
 }

 static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
 {
-    ZSTD_decompressBegin(dctx);   /* cannot fail */
+    dctx->format = ZSTD_f_zstd1;  /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */
    dctx->staticSize  = 0;
    dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
    dctx->ddict       = NULL;
@@ -183,6 +175,19 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
    dctx->streamStage = zdss_init;
 }

+ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
+{
+    ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace;
+
+    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
+    if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL;  /* minimum size */
+
+    ZSTD_initDCtx_internal(dctx);
+    dctx->staticSize = workspaceSize;
+    dctx->inBuff = (char*)(dctx+1);
+    return dctx;
+}
+
 ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
 {
    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
@@ -197,19 +202,6 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
    }
 }

-ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
-{
-    ZSTD_DCtx* dctx = (ZSTD_DCtx*) workspace;
-
-    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
-    if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL;  /* minimum size */
-
-    ZSTD_initDCtx_internal(dctx);
-    dctx->staticSize = workspaceSize;
-    dctx->inBuff = (char*)(dctx+1);
-    return dctx;
-}
-
 ZSTD_DCtx* ZSTD_createDCtx(void)
 {
    return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
@@ -252,7 +244,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 *  Note 3 : Skippable Frame Identifiers are considered valid. */
 unsigned ZSTD_isFrame(const void* buffer, size_t size)
 {
-    if (size < 4) return 0;
+    if (size < ZSTD_frameIdSize) return 0;
    {   U32 const magic = MEM_readLE32(buffer);
        if (magic == ZSTD_MAGICNUMBER) return 1;
        if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
@@ -263,40 +255,56 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
    return 0;
 }

-
-/** ZSTD_frameHeaderSize() :
-*   srcSize must be >= ZSTD_frameHeaderSize_prefix.
-*   @return : size of the Frame Header */
-size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
+/** ZSTD_frameHeaderSize_internal() :
+ *  srcSize must be large enough to reach header size fields.
+ *  note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
+ * @return : size of the Frame Header
+ *           or an error code, which can be tested with ZSTD_isError() */
+static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
 {
-    if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
-    {   BYTE const fhd = ((const BYTE*)src)[4];
+    size_t const minInputSize = ZSTD_startingInputLength(format);
+    if (srcSize < minInputSize) return ERROR(srcSize_wrong);
+
+    {   BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
        U32 const dictID= fhd & 3;
        U32 const singleSegment = (fhd >> 5) & 1;
        U32 const fcsId = fhd >> 6;
-        return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
-                + (singleSegment && !fcsId);
+        return minInputSize + !singleSegment
+             + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+             + (singleSegment && !fcsId);
    }
 }

+/** ZSTD_frameHeaderSize() :
+ *  srcSize must be >= ZSTD_frameHeaderSize_prefix.
+ * @return : size of the Frame Header */
+size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
+{
+    return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);
+}

-/** ZSTD_getFrameHeader() :
-*   decode Frame Header, or require larger `srcSize`.
-*   @return : 0, `zfhPtr` is correctly filled,
-*            >0, `srcSize` is too small, result is expected `srcSize`,
-*             or an error code, which can be tested using ZSTD_isError() */
-size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
+
+/** ZSTD_getFrameHeader_internal() :
+ *  decode Frame Header, or require larger `srcSize`.
+ *  note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
+ * @return : 0, `zfhPtr` is correctly filled,
+ *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ *           or an error code, which can be tested using ZSTD_isError() */
+static size_t ZSTD_getFrameHeader_internal(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
 {
    const BYTE* ip = (const BYTE*)src;
-    if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix;
+    size_t const minInputSize = ZSTD_startingInputLength(format);

-    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
+    if (srcSize < minInputSize) return minInputSize;
+
+    if ( (format != ZSTD_f_zstd1_magicless)
+      && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
        if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
            /* skippable frame */
            if (srcSize < ZSTD_skippableHeaderSize)
                return ZSTD_skippableHeaderSize; /* magic number + frame length */
            memset(zfhPtr, 0, sizeof(*zfhPtr));
-            zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
+            zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_frameIdSize);
            zfhPtr->frameType = ZSTD_skippableFrame;
            return 0;
        }
@@ -304,13 +312,13 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
    }

    /* ensure there is enough `srcSize` to fully read/decode frame header */
-    {   size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
+    {   size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format);
        if (srcSize < fhsize) return fhsize;
        zfhPtr->headerSize = (U32)fhsize;
    }

-    {   BYTE const fhdByte = ip[4];
-        size_t pos = 5;
+    {   BYTE const fhdByte = ip[minInputSize-1];
+        size_t pos = minInputSize;
        U32 const dictIDSizeCode = fhdByte&3;
        U32 const checksumFlag = (fhdByte>>2)&1;
        U32 const singleSegment = (fhdByte>>5)&1;
@@ -357,6 +365,18 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
    return 0;
 }

+/** ZSTD_getFrameHeader() :
+ *  decode Frame Header, or require larger `srcSize`.
+ *  note : this function does not consume input, it only reads it.
+ * @return : 0, `zfhPtr` is correctly filled,
+ *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ *           or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
+{
+    return ZSTD_getFrameHeader_internal(zfhPtr, src, srcSize, ZSTD_f_zstd1);
+}
+
+
 /** ZSTD_getFrameContentSize() :
 *  compatible with legacy mode
 * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
@@ -390,14 +410,14 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
    unsigned long long totalDstSize = 0;

    while (srcSize >= ZSTD_frameHeaderSize_prefix) {
-        const U32 magicNumber = MEM_readLE32(src);
+        U32 const magicNumber = MEM_readLE32(src);

        if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
            size_t skippableSize;
            if (srcSize < ZSTD_skippableHeaderSize)
                return ERROR(srcSize_wrong);
-            skippableSize = MEM_readLE32((const BYTE *)src + 4) +
-                            ZSTD_skippableHeaderSize;
+            skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_frameIdSize)
+                          + ZSTD_skippableHeaderSize;
            if (srcSize < skippableSize) {
                return ZSTD_CONTENTSIZE_ERROR;
            }
@@ -422,11 +442,9 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
            src = (const BYTE *)src + frameSrcSize;
            srcSize -= frameSrcSize;
        }
-    }
+    }  /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */

-    if (srcSize) {
-        return ZSTD_CONTENTSIZE_ERROR;
-    }
+    if (srcSize) return ZSTD_CONTENTSIZE_ERROR;

    return totalDstSize;
 }
@@ -442,7 +460,8 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
 {
    unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
-    return ret >= ZSTD_CONTENTSIZE_ERROR ? 0 : ret;
+    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);
+    return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;
 }


@@ -451,9 +470,9 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
 *   @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
 static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
 {
-    size_t const result = ZSTD_getFrameHeader(&(dctx->fParams), src, headerSize);
-    if (ZSTD_isError(result)) return result;  /* invalid header */
-    if (result>0) return ERROR(srcSize_wrong);   /* headerSize too small */
+    size_t const result = ZSTD_getFrameHeader_internal(&(dctx->fParams), src, headerSize, dctx->format);
+    if (ZSTD_isError(result)) return result;    /* invalid header */
+    if (result>0) return ERROR(srcSize_wrong);  /* headerSize too small */
    if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
        return ERROR(dictionary_wrong);
    if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
@@ -499,7 +518,8 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
 }

 /*! ZSTD_decodeLiteralsBlock() :
-    @return : nb of bytes read from src (< srcSize ) */
+ * @return : nb of bytes read from src (< srcSize )
+ *  note : symbol not declared but exposed for fullbench */
 size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                          const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
 {
@@ -700,9 +720,9 @@ static const FSE_decode_t4 OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
 };   /* OF_defaultDTable */

 /*! ZSTD_buildSeqTable() :
-    @return : nb bytes read from src,
-              or an error code if it fails, testable with ZSTD_isError()
-*/
+ * @return : nb bytes read from src,
+ *           or an error code if it fails, testable with ZSTD_isError()
+ */
 static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTablePtr,
                                 symbolEncodingType_e type, U32 max, U32 maxLog,
                                 const void* src, size_t srcSize,
@@ -807,9 +827,9 @@ typedef struct {
    FSE_DState_t stateOffb;
    FSE_DState_t stateML;
    size_t prevOffset[ZSTD_REP_NUM];
-    const BYTE* base;
+    const BYTE* prefixStart;
+    const BYTE* dictEnd;
    size_t pos;
-    uPtrDiff gotoDict;
 } seqState_t;


@@ -862,6 +882,15 @@ size_t ZSTD_execSequenceLast7(BYTE* op,

 typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;

+/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
+ * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
+ * bits before reloading. This value is the maximum number of bytes we read
+ * after reloading when we are decoding long offets.
+ */
+#define LONG_OFFSETS_MAX_EXTRA_BITS_32                                         \
+    (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32                         \
+        ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32                    \
+        : 0)

 static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
 {
@@ -869,7 +898,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l

    U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
    U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
-    U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb);   /* <= maxOff, by table construction */
+    U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb);   /* <= MaxOff, by table construction */

    U32 const llBits = LL_bits[llCode];
    U32 const mlBits = ML_bits[mlCode];
@@ -896,7 +925,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l
                     0,        1,       1,       5,     0xD,     0x1D,     0x3D,     0x7D,
                     0xFD,   0x1FD,   0x3FD,   0x7FD,   0xFFD,   0x1FFD,   0x3FFD,   0x7FFD,
                     0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
-                     0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
+                     0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };

    /* sequence */
    {   size_t offset;
@@ -904,8 +933,10 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l
            offset = 0;
        else {
            ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
-            if (longOffsets) {
-                int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
+            ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
+            assert(ofBits <= MaxOff);
+            if (MEM_32bits() && longOffsets) {
+                U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
                offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
                if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream);
                if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
@@ -936,13 +967,17 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l

    seq.matchLength = ML_base[mlCode]
                    + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0);  /* <=  16 bits */
-    if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
+    if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
+        BIT_reloadDStream(&seqState->DStream);
+    if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
+        BIT_reloadDStream(&seqState->DStream);
+    /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */
+    ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);

    seq.litLength = LL_base[llCode]
                  + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0);    /* <=  16 bits */
-    if (  MEM_32bits()
-      || (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) )
-       BIT_reloadDStream(&seqState->DStream);
+    if (MEM_32bits())
+        BIT_reloadDStream(&seqState->DStream);

    DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
                (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
@@ -1102,7 +1137,6 @@ static size_t ZSTD_decompressSequences(
 }


-
 HINT_INLINE
 seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets)
 {
@@ -1110,7 +1144,7 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long

    U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
    U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
-    U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb);   /* <= maxOff, by table construction */
+    U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb);   /* <= MaxOff, by table construction */

    U32 const llBits = LL_bits[llCode];
    U32 const mlBits = ML_bits[mlCode];
@@ -1137,7 +1171,7 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
                     0,        1,       1,       5,     0xD,     0x1D,     0x3D,     0x7D,
                     0xFD,   0x1FD,   0x3FD,   0x7FD,   0xFFD,   0x1FFD,   0x3FFD,   0x7FFD,
                     0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
-                     0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
+                     0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };

    /* sequence */
    {   size_t offset;
@@ -1145,8 +1179,10 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
            offset = 0;
        else {
            ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
-            if (longOffsets) {
-                int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
+            ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
+            assert(ofBits <= MaxOff);
+            if (MEM_32bits() && longOffsets) {
+                U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
                offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
                if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream);
                if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
@@ -1176,15 +1212,21 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
    }

    seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0);  /* <=  16 bits */
-    if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
+    if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
+        BIT_reloadDStream(&seqState->DStream);
+    if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
+        BIT_reloadDStream(&seqState->DStream);
+    /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */
+    ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);

    seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0);    /* <=  16 bits */
-    if (MEM_32bits() ||
-       (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream);
+    if (MEM_32bits())
+        BIT_reloadDStream(&seqState->DStream);

    {   size_t const pos = seqState->pos + seq.litLength;
-        seq.match = seqState->base + pos - seq.offset;    /* single memory segment */
-        if (seq.offset > pos) seq.match += seqState->gotoDict;   /* separate memory segment */
+        const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart;
+        seq.match = matchBase + pos - seq.offset;  /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
+                                                    * No consequence though : no memory access will occur, overly large offset will be detected in ZSTD_execSequenceLong() */
        seqState->pos = pos + seq.matchLength;
    }

@@ -1202,7 +1244,7 @@ HINT_INLINE
 size_t ZSTD_execSequenceLong(BYTE* op,
                             BYTE* const oend, seq_t sequence,
                             const BYTE** litPtr, const BYTE* const litLimit,
-                             const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
+                             const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd)
 {
    BYTE* const oLitEnd = op + sequence.litLength;
    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
@@ -1212,21 +1254,21 @@ size_t ZSTD_execSequenceLong(BYTE* op,
    const BYTE* match = sequence.match;

    /* check */
-    if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+    if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
-    if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
+    if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd);

    /* copy Literals */
-    ZSTD_copy8(op, *litPtr);
+    ZSTD_copy8(op, *litPtr);  /* note : op <= oLitEnd <= oend_w == oend - 8 */
    if (sequence.litLength > 8)
        ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
    op = oLitEnd;
    *litPtr = iLitEnd;   /* update for next sequence */

    /* copy Match */
-    if (sequence.offset > (size_t)(oLitEnd - base)) {
+    if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
        /* offset beyond prefix */
-        if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
+        if (sequence.offset > (size_t)(oLitEnd - dictStart)) return ERROR(corruption_detected);
        if (match + sequence.matchLength <= dictEnd) {
            memmove(oLitEnd, match, sequence.matchLength);
            return sequenceLength;
@@ -1236,7 +1278,7 @@ size_t ZSTD_execSequenceLong(BYTE* op,
            memmove(oLitEnd, match, length1);
            op = oLitEnd + length1;
            sequence.matchLength -= length1;
-            match = base;
+            match = prefixStart;
            if (op > oend_w || sequence.matchLength < MINMATCH) {
              U32 i;
              for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
@@ -1290,8 +1332,8 @@ static size_t ZSTD_decompressSequencesLong(
    BYTE* op = ostart;
    const BYTE* litPtr = dctx->litPtr;
    const BYTE* const litEnd = litPtr + dctx->litSize;
-    const BYTE* const base = (const BYTE*) (dctx->base);
-    const BYTE* const vBase = (const BYTE*) (dctx->vBase);
+    const BYTE* const prefixStart = (const BYTE*) (dctx->base);
+    const BYTE* const dictStart = (const BYTE*) (dctx->vBase);
    const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
    int nbSeq;

@@ -1312,9 +1354,9 @@ static size_t ZSTD_decompressSequencesLong(
        int seqNb;
        dctx->fseEntropy = 1;
        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
-        seqState.base = base;
-        seqState.pos = (size_t)(op-base);
-        seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
+        seqState.prefixStart = prefixStart;
+        seqState.pos = (size_t)(op-prefixStart);
+        seqState.dictEnd = dictEnd;
        CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
        FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
        FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1329,9 +1371,9 @@ static size_t ZSTD_decompressSequencesLong(
        /* decode and decompress */
        for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
            seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, isLongOffset);
-            size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
+            size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
            if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
-            PREFETCH(sequence.match);
+            PREFETCH(sequence.match);  /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
            sequences[seqNb&STOSEQ_MASK] = sequence;
            op += oneSeqSize;
        }
@@ -1340,7 +1382,7 @@ static size_t ZSTD_decompressSequencesLong(
        /* finish queue */
        seqNb -= seqAdvance;
        for ( ; seqNb<nbSeq ; seqNb++) {
-            size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
+            size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
            if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
            op += oneSeqSize;
        }
@@ -1372,9 +1414,10 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
     * conservative.
     */
    ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)));
-    /* We don't expect window sizes this big. */
-    assert(!frame || dctx->fParams.windowSize <= (1ULL << STREAM_ACCUMULATOR_MIN_64));
-    DEBUGLOG(5, "ZSTD_decompressBlock_internal");
+    /* windowSize could be any value at this point, since it is only validated
+     * in the streaming API.
+     */
+    DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);

    if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);

@@ -1423,7 +1466,7 @@ ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, siz
 }


-size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
+static size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
 {
    if (length > dstCapacity) return ERROR(dstSize_tooSmall);
    memset(dst, byte, length);
@@ -1443,7 +1486,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 #endif
    if ( (srcSize >= ZSTD_skippableHeaderSize)
      && (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
-        return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + 4);
+        return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize);
    } else {
        const BYTE* ip = (const BYTE*)src;
        const BYTE* const ipstart = ip;
@@ -1545,6 +1588,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
        if (blockProperties.lastBlock) break;
    }

+    if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+        if ((U64)(op-ostart) != dctx->fParams.frameContentSize) {
+            return ERROR(corruption_detected);
+    }   }
    if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
        U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
        U32 checkRead;
@@ -1602,13 +1649,15 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
 #endif

        magicNumber = MEM_readLE32(src);
+        DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
+                    (U32)magicNumber, (U32)ZSTD_MAGICNUMBER);
        if (magicNumber != ZSTD_MAGICNUMBER) {
            if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
                size_t skippableSize;
                if (srcSize < ZSTD_skippableHeaderSize)
                    return ERROR(srcSize_wrong);
-                skippableSize = MEM_readLE32((const BYTE *)src + 4) +
-                                ZSTD_skippableHeaderSize;
+                skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize)
+                              + ZSTD_skippableHeaderSize;
                if (srcSize < skippableSize) return ERROR(srcSize_wrong);

                src = (const BYTE *)src + skippableSize;
@@ -1711,33 +1760,31 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 {
    DEBUGLOG(5, "ZSTD_decompressContinue");
    /* Sanity check */
-    if (srcSize != dctx->expected) return ERROR(srcSize_wrong);   /* unauthorized */
+    if (srcSize != dctx->expected) return ERROR(srcSize_wrong);  /* not allowed */
    if (dstCapacity) ZSTD_checkContinuity(dctx, dst);

    switch (dctx->stage)
    {
    case ZSTDds_getFrameHeaderSize :
-        if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);      /* unauthorized */
        assert(src != NULL);
-        if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {        /* skippable frame */
-            memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
-            dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix;  /* magic number + skippable frame length */
-            dctx->stage = ZSTDds_decodeSkippableHeader;
-            return 0;
-        }
-        dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
+        if (dctx->format == ZSTD_f_zstd1) {  /* allows header */
+            assert(srcSize >= ZSTD_frameIdSize);  /* to read skippable magic number */
+            if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {        /* skippable frame */
+                memcpy(dctx->headerBuffer, src, srcSize);
+                dctx->expected = ZSTD_skippableHeaderSize - srcSize;  /* remaining to load to get full skippable frame header */
+                dctx->stage = ZSTDds_decodeSkippableHeader;
+                return 0;
+        }   }
+        dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
        if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
-        memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
-        if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
-            dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
-            dctx->stage = ZSTDds_decodeFrameHeader;
-            return 0;
-        }
-        dctx->expected = 0;   /* not necessary to copy more */
-        /* fall-through */
+        memcpy(dctx->headerBuffer, src, srcSize);
+        dctx->expected = dctx->headerSize - srcSize;
+        dctx->stage = ZSTDds_decodeFrameHeader;
+        return 0;
+
    case ZSTDds_decodeFrameHeader:
        assert(src != NULL);
-        memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
+        memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
        CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
        dctx->expected = ZSTD_blockHeaderSize;
        dctx->stage = ZSTDds_decodeBlockHeader;
@@ -1769,6 +1816,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
            }
            return 0;
        }
+
    case ZSTDds_decompressLastBlock:
    case ZSTDds_decompressBlock:
        DEBUGLOG(5, "case ZSTDds_decompressBlock");
@@ -1814,29 +1862,31 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
            }
            return rSize;
        }
+
    case ZSTDds_checkChecksum:
-        DEBUGLOG(4, "case ZSTDds_checkChecksum");
        assert(srcSize == 4);  /* guaranteed by dctx->expected */
        {   U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
            U32 const check32 = MEM_readLE32(src);
-            DEBUGLOG(4, "calculated %08X :: %08X read", h32, check32);
+            DEBUGLOG(4, "checksum : calculated %08X :: %08X read", h32, check32);
            if (check32 != h32) return ERROR(checksum_wrong);
            dctx->expected = 0;
            dctx->stage = ZSTDds_getFrameHeaderSize;
            return 0;
        }
+
    case ZSTDds_decodeSkippableHeader:
-        {   assert(src != NULL);
-            memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
-            dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
-            dctx->stage = ZSTDds_skipFrame;
-            return 0;
-        }
+        assert(src != NULL);
+        assert(srcSize <= ZSTD_skippableHeaderSize);
+        memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize);   /* complete skippable header */
+        dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_frameIdSize);   /* note : dctx->expected can grow seriously large, beyond local buffer size */
+        dctx->stage = ZSTDds_skipFrame;
+        return 0;
+
    case ZSTDds_skipFrame:
-        {   dctx->expected = 0;
-            dctx->stage = ZSTDds_getFrameHeaderSize;
-            return 0;
-        }
+        dctx->expected = 0;
+        dctx->stage = ZSTDds_getFrameHeaderSize;
+        return 0;
+
    default:
        return ERROR(GENERIC);   /* impossible */
    }
@@ -1917,7 +1967,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
        if (magic != ZSTD_MAGIC_DICTIONARY) {
            return ZSTD_refDictContent(dctx, dict, dictSize);   /* pure content mode */
    }   }
-    dctx->dictID = MEM_readLE32((const char*)dict + 4);
+    dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_frameIdSize);

    /* load entropy tables */
    {   size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
@@ -1931,6 +1981,29 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
    return ZSTD_refDictContent(dctx, dict, dictSize);
 }

+/* Note : this function cannot fail */
+size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
+{
+    assert(dctx != NULL);
+    dctx->expected = ZSTD_startingInputLength(dctx->format);  /* dctx->format must be properly set */
+    dctx->stage = ZSTDds_getFrameHeaderSize;
+    dctx->decodedSize = 0;
+    dctx->previousDstEnd = NULL;
+    dctx->base = NULL;
+    dctx->vBase = NULL;
+    dctx->dictEnd = NULL;
+    dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
+    dctx->litEntropy = dctx->fseEntropy = 0;
+    dctx->dictID = 0;
+    ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
+    memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
+    dctx->LLTptr = dctx->entropy.LLTable;
+    dctx->MLTptr = dctx->entropy.MLTable;
+    dctx->OFTptr = dctx->entropy.OFTable;
+    dctx->HUFptr = dctx->entropy.hufTable;
+    return 0;
+}
+
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
    CHECK_F( ZSTD_decompressBegin(dctx) );
@@ -1997,7 +2070,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
    {   U32 const magic = MEM_readLE32(ddict->dictContent);
        if (magic != ZSTD_MAGIC_DICTIONARY) return 0;   /* pure content mode */
    }
-    ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
+    ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);

    /* load entropy tables */
    CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
@@ -2118,7 +2191,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
 {
    if (dictSize < 8) return 0;
    if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
-    return MEM_readLE32((const char*)dict + 4);
+    return MEM_readLE32((const char*)dict + ZSTD_frameIdSize);
 }

 /*! ZSTD_getDictID_fromDDict() :
@@ -2213,13 +2286,15 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
    return ZSTD_frameHeaderSize_prefix;
 }

+/* note : this variant can't fail */
 size_t ZSTD_initDStream(ZSTD_DStream* zds)
 {
    return ZSTD_initDStream_usingDict(zds, NULL, 0);
 }

 /* ZSTD_initDStream_usingDDict() :
- * ddict will just be referenced, and must outlive decompression session */
+ * ddict will just be referenced, and must outlive decompression session
+ * this function cannot fail */
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict)
 {
    size_t const initResult = ZSTD_initDStream(zds);
@@ -2239,14 +2314,39 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds)
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
                                ZSTD_DStreamParameter_e paramType, unsigned paramValue)
 {
+    ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init);
+    if ((unsigned)zds->streamStage > (unsigned)zdss_loadHeader)
+        return ERROR(stage_wrong);
    switch(paramType)
    {
        default : return ERROR(parameter_unsupported);
-        case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
+        case DStream_p_maxWindowSize :
+            DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10);
+            zds->maxWindowSize = paramValue ? paramValue : (U32)(-1);
+            break;
    }
    return 0;
 }

+size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
+{
+    ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init);
+    if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader)
+        return ERROR(stage_wrong);
+    dctx->maxWindowSize = maxWindowSize;
+    return 0;
+}
+
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
+{
+    DEBUGLOG(4, "ZSTD_DCtx_setFormat : %u", (unsigned)format);
+    ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init);
+    if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader)
+        return ERROR(stage_wrong);
+    dctx->format = format;
+    return 0;
+}
+

 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
 {
@@ -2271,7 +2371,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
    return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
 }

-ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
+size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
 {
    U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;   /* note : should be user-selectable */
    ZSTD_frameHeader zfh;
@@ -2305,7 +2405,18 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
    U32 someMoreWork = 1;

    DEBUGLOG(5, "ZSTD_decompressStream");
+    if (input->pos > input->size) {  /* forbidden */
+        DEBUGLOG(5, "in: pos: %u   vs size: %u",
+                    (U32)input->pos, (U32)input->size);
+        return ERROR(srcSize_wrong);
+    }
+    if (output->pos > output->size) {  /* forbidden */
+        DEBUGLOG(5, "out: pos: %u   vs size: %u",
+                    (U32)output->pos, (U32)output->size);
+        return ERROR(dstSize_tooSmall);
+    }
    DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
+
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
    if (zds->legacyVersion) {
        /* legacy support is incompatible with static dctx */
@@ -2322,7 +2433,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
            /* fall-through */

        case zdss_loadHeader :
-            {   size_t const hSize = ZSTD_getFrameHeader(&zds->fParams, zds->headerBuffer, zds->lhSize);
+            DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
+            {   size_t const hSize = ZSTD_getFrameHeader_internal(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
+                DEBUGLOG(5, "header size : %u", (U32)hSize);
                if (ZSTD_isError(hSize)) {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
                    U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
@@ -2338,14 +2451,16 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                        return ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
                    }
 #endif
-                    return hSize; /* error */
+                    return hSize;   /* error */
                }
                if (hSize != 0) {   /* need more input */
                    size_t const toLoad = hSize - zds->lhSize;   /* if hSize!=0, hSize > zds->lhSize */
-                    if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
-                        if (iend-ip > 0) {
-                            memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
-                            zds->lhSize += iend-ip;
+                    size_t const remainingInput = (size_t)(iend-ip);
+                    assert(iend >= ip);
+                    if (toLoad > remainingInput) {   /* not enough input to load full header */
+                        if (remainingInput > 0) {
+                            memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
+                            zds->lhSize += remainingInput;
                        }
                        input->pos = input->size;
                        return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
@@ -2360,8 +2475,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
                size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
                if (cSize <= (size_t)(iend-istart)) {
+                    /* shortcut : using single-pass mode */
                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
                    if (ZSTD_isError(decompressedSize)) return decompressedSize;
+                    DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
                    ip = istart + cSize;
                    op += decompressedSize;
                    zds->expected = 0;
@@ -2375,7 +2492,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
            CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));

            if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
-                zds->expected = MEM_readLE32(zds->headerBuffer + 4);
+                zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_frameIdSize);
                zds->stage = ZSTDds_skipFrame;
            } else {
                CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
@@ -2384,7 +2501,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
            }

            /* control buffer memory usage */
-            DEBUGLOG(4, "Control max buffer memory usage");
+            DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
+                        (U32)(zds->fParams.windowSize >>10),
+                        (U32)(zds->maxWindowSize >> 10) );
            zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
            if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);

@@ -2442,17 +2561,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
            /* fall-through */
        case zdss_load:
            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
-                size_t const toLoad = neededInSize - zds->inPos;   /* should always be <= remaining space within inBuff */
+                size_t const toLoad = neededInSize - zds->inPos;
+                int const isSkipFrame = ZSTD_isSkipFrame(zds);
                size_t loadedSize;
-                if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
-                loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
+                if (isSkipFrame) {
+                    loadedSize = MIN(toLoad, (size_t)(iend-ip));
+                } else {
+                    if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
+                    loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
+                }
                ip += loadedSize;
                zds->inPos += loadedSize;
                if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */

                /* decode loaded input */
-                {  const int isSkipFrame = ZSTD_isSkipFrame(zds);
-                   size_t const decodedSize = ZSTD_decompressContinue(zds,
+                {   size_t const decodedSize = ZSTD_decompressContinue(zds,
                        zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
                        zds->inBuff, neededInSize);
                    if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -2513,3 +2636,30 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
        return nextSrcSizeHint;
    }
 }
+
+
+size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+    return ZSTD_decompressStream(dctx, output, input);
+}
+
+size_t ZSTD_decompress_generic_simpleArgs (
+                            ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos)
+{
+    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
+    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
+    /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
+    size_t const cErr = ZSTD_decompress_generic(dctx, &output, &input);
+    *dstPos = output.pos;
+    *srcPos = input.pos;
+    return cErr;
+}
+
+void ZSTD_DCtx_reset(ZSTD_DCtx* dctx)
+{
+    (void)ZSTD_initDStream(dctx);
+    dctx->format = ZSTD_f_zstd1;
+    dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
+}
@@ -72,6 +72,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
                                   const void* dict, size_t dictSize,
                                   ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
+    if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;  /* preserve "0 == unknown" behavior */
    return ZSTD_initCStream_advanced(zbc, dict, dictSize, params, pledgedSrcSize);
 }

@@ -711,8 +711,8 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
 * compiled with multithreaded support.
 */
 typedef struct COVER_best_s {
-  pthread_mutex_t mutex;
-  pthread_cond_t cond;
+  ZSTD_pthread_mutex_t mutex;
+  ZSTD_pthread_cond_t cond;
  size_t liveJobs;
  void *dict;
  size_t dictSize;
@@ -725,8 +725,8 @@ typedef struct COVER_best_s {
 */
 static void COVER_best_init(COVER_best_t *best) {
  if (best==NULL) return; /* compatible with init on NULL */
-  (void)pthread_mutex_init(&best->mutex, NULL);
-  (void)pthread_cond_init(&best->cond, NULL);
+  (void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
+  (void)ZSTD_pthread_cond_init(&best->cond, NULL);
  best->liveJobs = 0;
  best->dict = NULL;
  best->dictSize = 0;
@@ -741,11 +741,11 @@ static void COVER_best_wait(COVER_best_t *best) {
  if (!best) {
    return;
  }
-  pthread_mutex_lock(&best->mutex);
+  ZSTD_pthread_mutex_lock(&best->mutex);
  while (best->liveJobs != 0) {
-    pthread_cond_wait(&best->cond, &best->mutex);
+    ZSTD_pthread_cond_wait(&best->cond, &best->mutex);
  }
-  pthread_mutex_unlock(&best->mutex);
+  ZSTD_pthread_mutex_unlock(&best->mutex);
 }

 /**
@@ -759,8 +759,8 @@ static void COVER_best_destroy(COVER_best_t *best) {
  if (best->dict) {
    free(best->dict);
  }
-  pthread_mutex_destroy(&best->mutex);
-  pthread_cond_destroy(&best->cond);
+  ZSTD_pthread_mutex_destroy(&best->mutex);
+  ZSTD_pthread_cond_destroy(&best->cond);
 }

 /**
@@ -771,9 +771,9 @@ static void COVER_best_start(COVER_best_t *best) {
  if (!best) {
    return;
  }
-  pthread_mutex_lock(&best->mutex);
+  ZSTD_pthread_mutex_lock(&best->mutex);
  ++best->liveJobs;
-  pthread_mutex_unlock(&best->mutex);
+  ZSTD_pthread_mutex_unlock(&best->mutex);
 }

 /**
@@ -789,7 +789,7 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
  }
  {
    size_t liveJobs;
-    pthread_mutex_lock(&best->mutex);
+    ZSTD_pthread_mutex_lock(&best->mutex);
    --best->liveJobs;
    liveJobs = best->liveJobs;
    /* If the new dictionary is better */
@@ -812,9 +812,9 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
      best->parameters = parameters;
      best->compressedSize = compressedSize;
    }
-    pthread_mutex_unlock(&best->mutex);
+    ZSTD_pthread_mutex_unlock(&best->mutex);
    if (liveJobs == 0) {
-      pthread_cond_broadcast(&best->cond);
+      ZSTD_pthread_cond_broadcast(&best->cond);
    }
  }
 }
@@ -103,7 +103,7 @@ unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
 /*-********************************************************
 *  Dictionary training functions
 **********************************************************/
-static unsigned ZDICT_NbCommonBytes (register size_t val)
+static unsigned ZDICT_NbCommonBytes (size_t val)
 {
    if (MEM_isLittleEndian()) {
        if (MEM_64bits()) {
@@ -339,7 +339,7 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
 /****************************************************************
 *  Internal functions
 ****************************************************************/
-FORCE_INLINE unsigned FSE_highbit32 (register U32 val)
+FORCE_INLINE unsigned FSE_highbit32 (U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
    unsigned long r;
@@ -330,18 +330,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);


-/*
-* Start by invoking BIT_initDStream().
-* A chunk of the bitStream is then stored into a local register.
-* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-* You can then retrieve bitFields stored into the local register, **in reverse order**.
-* Local register is manually filled from memory by the BIT_reloadDStream() method.
-* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
-* Otherwise, it can be less than that, so proceed accordingly.
-* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
-*/
-
-
 /******************************************
 *  unsafe API
 ******************************************/
@@ -353,7 +341,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 /****************************************************************
 *  Helper functions
 ****************************************************************/
-MEM_STATIC unsigned BIT_highbit32 (register U32 val)
+MEM_STATIC unsigned BIT_highbit32 (U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
    unsigned long r=0;
@@ -427,13 +415,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
    return srcSize;
 }

-/*!BIT_lookBits
- * Provides next n bits from local register
- * local register is not modified (bits are still present for next read/look)
- * On 32-bits, maxNbBits==25
- * On 64-bits, maxNbBits==57
- * @return : value extracted
- */
 MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
 {
    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -453,11 +434,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
    bitD->bitsConsumed += nbBits;
 }

-/*!BIT_readBits
- * Read next n bits from local register.
- * pay attention to not read more than nbBits contained into local register.
- * @return : extracted value.
- */
 MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 {
    size_t value = BIT_lookBits(bitD, nbBits);
@@ -695,55 +671,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi

 static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);

-/*
-Let's now decompose FSE_decompress_usingDTable() into its unitary components.
-You will decode FSE-encoded symbols from the bitStream,
-and also any other bitFields you put in, **in reverse order**.
-
-You will need a few variables to track your bitStream. They are :
-
-BIT_DStream_t DStream;    // Stream context
-FSE_DState_t  DState;     // State context. Multiple ones are possible
-FSE_DTable*   DTablePtr;  // Decoding table, provided by FSE_buildDTable()
-
-The first thing to do is to init the bitStream.
-    errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
-
-You should then retrieve your initial state(s)
-(in reverse flushing order if you have several ones) :
-    errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
-
-You can then decode your data, symbol after symbol.
-For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
-Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
-    unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
-
-You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-Note : maximum allowed nbBits is 25, for 32-bits compatibility
-    size_t bitField = BIT_readBits(&DStream, nbBits);
-
-All above operations only read from local register (which size depends on size_t).
-Refueling the register from memory is manually performed by the reload method.
-    endSignal = FSE_reloadDStream(&DStream);
-
-BIT_reloadDStream() result tells if there is still some more data to read from DStream.
-BIT_DStream_unfinished : there is still some data left into the DStream.
-BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-
-When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-to properly detect the exact end of stream.
-After each decoded symbol, check if DStream is fully consumed using this simple test :
-    BIT_reloadDStream(&DStream) >= BIT_DStream_completed
-
-When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-Checking if DStream has reached its end is performed by :
-    BIT_endOfDStream(&DStream);
-Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
-    FSE_endOfDState(&DState);
-*/
-

 /******************************************
 *  FSE unsafe API
@@ -332,17 +332,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);


-/*
-* Start by invoking BIT_initDStream().
-* A chunk of the bitStream is then stored into a local register.
-* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-* You can then retrieve bitFields stored into the local register, **in reverse order**.
-* Local register is manually filled from memory by the BIT_reloadDStream() method.
-* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
-* Otherwise, it can be less than that, so proceed accordingly.
-* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
-*/
-

 /******************************************
 *  unsafe API
@@ -355,7 +344,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 /****************************************************************
 *  Helper functions
 ****************************************************************/
-MEM_STATIC unsigned BIT_highbit32 (register U32 val)
+MEM_STATIC unsigned BIT_highbit32 (U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
    unsigned long r=0;
@@ -428,14 +417,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si

    return srcSize;
 }
-
-/*!BIT_lookBits
- * Provides next n bits from local register
- * local register is not modified (bits are still present for next read/look)
- * On 32-bits, maxNbBits==25
- * On 64-bits, maxNbBits==57
- * @return : value extracted
- */
 MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
 {
    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -455,11 +436,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
    bitD->bitsConsumed += nbBits;
 }

-/*!BIT_readBits
- * Read next n bits from local register.
- * pay attention to not read more than nbBits contained into local register.
- * @return : extracted value.
- */
 MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 {
    size_t value = BIT_lookBits(bitD, nbBits);
@@ -697,55 +673,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi

 static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);

-/*
-Let's now decompose FSE_decompress_usingDTable() into its unitary components.
-You will decode FSE-encoded symbols from the bitStream,
-and also any other bitFields you put in, **in reverse order**.
-
-You will need a few variables to track your bitStream. They are :
-
-BIT_DStream_t DStream;    // Stream context
-FSE_DState_t  DState;     // State context. Multiple ones are possible
-FSE_DTable*   DTablePtr;  // Decoding table, provided by FSE_buildDTable()
-
-The first thing to do is to init the bitStream.
-    errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
-
-You should then retrieve your initial state(s)
-(in reverse flushing order if you have several ones) :
-    errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
-
-You can then decode your data, symbol after symbol.
-For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
-Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
-    unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
-
-You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-Note : maximum allowed nbBits is 25, for 32-bits compatibility
-    size_t bitField = BIT_readBits(&DStream, nbBits);
-
-All above operations only read from local register (which size depends on size_t).
-Refueling the register from memory is manually performed by the reload method.
-    endSignal = FSE_reloadDStream(&DStream);
-
-BIT_reloadDStream() result tells if there is still some more data to read from DStream.
-BIT_DStream_unfinished : there is still some data left into the DStream.
-BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-
-When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-to properly detect the exact end of stream.
-After each decoded symbol, check if DStream is fully consumed using this simple test :
-    BIT_reloadDStream(&DStream) >= BIT_DStream_completed
-
-When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-Checking if DStream has reached its end is performed by :
-    BIT_endOfDStream(&DStream);
-Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
-    FSE_endOfDState(&DState);
-*/
-

 /******************************************
 *  FSE unsafe API
@@ -738,16 +738,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);


-/*
-* Start by invoking BIT_initDStream().
-* A chunk of the bitStream is then stored into a local register.
-* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-* You can then retrieve bitFields stored into the local register, **in reverse order**.
-* Local register is manually filled from memory by the BIT_reloadDStream() method.
-* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
-* Otherwise, it can be less than that, so proceed accordingly.
-* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
-*/


 /******************************************
@@ -761,7 +751,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 /****************************************************************
 *  Helper functions
 ****************************************************************/
-MEM_STATIC unsigned BIT_highbit32 (register U32 val)
+MEM_STATIC unsigned BIT_highbit32 (U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
    unsigned long r=0;
@@ -834,13 +824,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
    return srcSize;
 }

-/*!BIT_lookBits
- * Provides next n bits from local register
- * local register is not modified (bits are still present for next read/look)
- * On 32-bits, maxNbBits==25
- * On 64-bits, maxNbBits==57
- * @return : value extracted
- */
 MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
 {
    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -860,11 +843,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
    bitD->bitsConsumed += nbBits;
 }

-/*!BIT_readBits
- * Read next n bits from local register.
- * pay attention to not read more than nbBits contained into local register.
- * @return : extracted value.
- */
 MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 {
    size_t value = BIT_lookBits(bitD, nbBits);
@@ -1011,55 +989,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi

 static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);

-/*!
-Let's now decompose FSE_decompress_usingDTable() into its unitary components.
-You will decode FSE-encoded symbols from the bitStream,
-and also any other bitFields you put in, **in reverse order**.
-
-You will need a few variables to track your bitStream. They are :
-
-BIT_DStream_t DStream;    // Stream context
-FSE_DState_t  DState;     // State context. Multiple ones are possible
-FSE_DTable*   DTablePtr;  // Decoding table, provided by FSE_buildDTable()
-
-The first thing to do is to init the bitStream.
-    errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
-
-You should then retrieve your initial state(s)
-(in reverse flushing order if you have several ones) :
-    errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
-
-You can then decode your data, symbol after symbol.
-For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
-Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
-    unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
-
-You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-Note : maximum allowed nbBits is 25, for 32-bits compatibility
-    size_t bitField = BIT_readBits(&DStream, nbBits);
-
-All above operations only read from local register (which size depends on size_t).
-Refueling the register from memory is manually performed by the reload method.
-    endSignal = FSE_reloadDStream(&DStream);
-
-BIT_reloadDStream() result tells if there is still some more data to read from DStream.
-BIT_DStream_unfinished : there is still some data left into the DStream.
-BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-
-When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-to properly detect the exact end of stream.
-After each decoded symbol, check if DStream is fully consumed using this simple test :
-    BIT_reloadDStream(&DStream) >= BIT_DStream_completed
-
-When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-Checking if DStream has reached its end is performed by :
-    BIT_endOfDStream(&DStream);
-Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
-    FSE_endOfDState(&DState);
-*/
-

 /* *****************************************
 *  FSE unsafe API
@@ -736,18 +736,6 @@ MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD);
 MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* bitD);


-/*!
-* Start by invoking BITv05_initDStream().
-* A chunk of the bitStream is then stored into a local register.
-* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-* You can then retrieve bitFields stored into the local register, **in reverse order**.
-* Local register is explicitly reloaded from memory by the BITv05_reloadDStream() method.
-* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BITv05_DStream_unfinished.
-* Otherwise, it can be less than that, so proceed accordingly.
-* Checking if DStream has reached its end can be performed with BITv05_endOfDStream()
-*/
-
-
 /*-****************************************
 *  unsafe API
 ******************************************/
@@ -759,7 +747,7 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits);
 /*-**************************************************************
 *  Helper functions
 ****************************************************************/
-MEM_STATIC unsigned BITv05_highbit32 (register U32 val)
+MEM_STATIC unsigned BITv05_highbit32 (U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
    unsigned long r=0;
@@ -829,13 +817,6 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
    return srcSize;
 }

-/*!BITv05_lookBits
- * Provides next n bits from local register
- * local register is not modified (bits are still present for next read/look)
- * On 32-bits, maxNbBits==25
- * On 64-bits, maxNbBits==57
- * @return : value extracted
- */
 MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits)
 {
    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -855,11 +836,6 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
    bitD->bitsConsumed += nbBits;
 }

-/*!BITv05_readBits
- * Read next n bits from local register.
- * pay attention to not read more than nbBits contained into local register.
- * @return : extracted value.
- */
 MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
 {
    size_t value = BITv05_lookBits(bitD, nbBits);
@@ -995,54 +971,6 @@ static unsigned char FSEv05_decodeSymbol(FSEv05_DState_t* DStatePtr, BITv05_DStr

 static unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr);

-/*!
-Let's now decompose FSEv05_decompress_usingDTable() into its unitary components.
-You will decode FSEv05-encoded symbols from the bitStream,
-and also any other bitFields you put in, **in reverse order**.
-
-You will need a few variables to track your bitStream. They are :
-
-BITv05_DStream_t DStream;    // Stream context
-FSEv05_DState_t  DState;     // State context. Multiple ones are possible
-FSEv05_DTable*   DTablePtr;  // Decoding table, provided by FSEv05_buildDTable()
-
-The first thing to do is to init the bitStream.
-    errorCode = BITv05_initDStream(&DStream, srcBuffer, srcSize);
-
-You should then retrieve your initial state(s)
-(in reverse flushing order if you have several ones) :
-    errorCode = FSEv05_initDState(&DState, &DStream, DTablePtr);
-
-You can then decode your data, symbol after symbol.
-For information the maximum number of bits read by FSEv05_decodeSymbol() is 'tableLog'.
-Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
-    unsigned char symbol = FSEv05_decodeSymbol(&DState, &DStream);
-
-You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-Note : maximum allowed nbBits is 25, for 32-bits compatibility
-    size_t bitField = BITv05_readBits(&DStream, nbBits);
-
-All above operations only read from local register (which size depends on size_t).
-Refueling the register from memory is manually performed by the reload method.
-    endSignal = FSEv05_reloadDStream(&DStream);
-
-BITv05_reloadDStream() result tells if there is still some more data to read from DStream.
-BITv05_DStream_unfinished : there is still some data left into the DStream.
-BITv05_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-BITv05_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-BITv05_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-
-When reaching end of buffer (BITv05_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-to properly detect the exact end of stream.
-After each decoded symbol, check if DStream is fully consumed using this simple test :
-    BITv05_reloadDStream(&DStream) >= BITv05_DStream_completed
-
-When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-Checking if DStream has reached its end is performed by :
-    BITv05_endOfDStream(&DStream);
-Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
-    FSEv05_endOfDState(&DState);
-*/


 /* *****************************************
@@ -839,16 +839,6 @@ MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD);
 MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* bitD);


-/* Start by invoking BITv06_initDStream().
-*  A chunk of the bitStream is then stored into a local register.
-*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-*  You can then retrieve bitFields stored into the local register, **in reverse order**.
-*  Local register is explicitly reloaded from memory by the BITv06_reloadDStream() method.
-*  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv06_DStream_unfinished.
-*  Otherwise, it can be less than that, so proceed accordingly.
-*  Checking if DStream has reached its end can be performed with BITv06_endOfDStream().
-*/
-

 /*-****************************************
 *  unsafe API
@@ -861,7 +851,7 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, unsigned nbBits);
 /*-**************************************************************
 *  Internal functions
 ****************************************************************/
-MEM_STATIC unsigned BITv06_highbit32 (register U32 val)
+MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
    unsigned long r=0;
@@ -929,13 +919,6 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
 }


-/*! BITv06_lookBits() :
- *  Provides next n bits from local register.
- *  local register is not modified.
- *  On 32-bits, maxNbBits==24.
- *  On 64-bits, maxNbBits==56.
- *  @return : value extracted
- */
 MEM_STATIC size_t BITv06_lookBits(const BITv06_DStream_t* bitD, U32 nbBits)
 {
    U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -955,11 +938,6 @@ MEM_STATIC void BITv06_skipBits(BITv06_DStream_t* bitD, U32 nbBits)
    bitD->bitsConsumed += nbBits;
 }

-/*! BITv06_readBits() :
- *  Read (consume) next n bits from local register and update.
- *  Pay attention to not read more than nbBits contained into local register.
- *  @return : extracted value.
- */
 MEM_STATIC size_t BITv06_readBits(BITv06_DStream_t* bitD, U32 nbBits)
 {
    size_t const value = BITv06_lookBits(bitD, nbBits);
@@ -976,11 +954,6 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, U32 nbBits)
    return value;
 }

-/*! BITv06_reloadDStream() :
-*   Refill `BITv06_DStream_t` from src buffer previously defined (see BITv06_initDStream() ).
-*   This function is safe, it guarantees it will not read beyond src buffer.
-*   @return : status of `BITv06_DStream_t` internal register.
-              if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
 MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD)
 {
    if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should never happen */
@@ -1103,55 +1076,6 @@ static void     FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t*

 static unsigned char FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);

-/*!
-Let's now decompose FSEv06_decompress_usingDTable() into its unitary components.
-You will decode FSE-encoded symbols from the bitStream,
-and also any other bitFields you put in, **in reverse order**.
-
-You will need a few variables to track your bitStream. They are :
-
-BITv06_DStream_t DStream;    // Stream context
-FSEv06_DState_t  DState;     // State context. Multiple ones are possible
-FSEv06_DTable*   DTablePtr;  // Decoding table, provided by FSEv06_buildDTable()
-
-The first thing to do is to init the bitStream.
-    errorCode = BITv06_initDStream(&DStream, srcBuffer, srcSize);
-
-You should then retrieve your initial state(s)
-(in reverse flushing order if you have several ones) :
-    errorCode = FSEv06_initDState(&DState, &DStream, DTablePtr);
-
-You can then decode your data, symbol after symbol.
-For information the maximum number of bits read by FSEv06_decodeSymbol() is 'tableLog'.
-Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
-    unsigned char symbol = FSEv06_decodeSymbol(&DState, &DStream);
-
-You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-Note : maximum allowed nbBits is 25, for 32-bits compatibility
-    size_t bitField = BITv06_readBits(&DStream, nbBits);
-
-All above operations only read from local register (which size depends on size_t).
-Refueling the register from memory is manually performed by the reload method.
-    endSignal = FSEv06_reloadDStream(&DStream);
-
-BITv06_reloadDStream() result tells if there is still some more data to read from DStream.
-BITv06_DStream_unfinished : there is still some data left into the DStream.
-BITv06_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-BITv06_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-BITv06_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-
-When reaching end of buffer (BITv06_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-to properly detect the exact end of stream.
-After each decoded symbol, check if DStream is fully consumed using this simple test :
-    BITv06_reloadDStream(&DStream) >= BITv06_DStream_completed
-
-When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-Checking if DStream has reached its end is performed by :
-    BITv06_endOfDStream(&DStream);
-Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
-    FSEv06_endOfDState(&DState);
-*/
-

 /* *****************************************
 *  FSE unsafe API
@@ -511,16 +511,6 @@ MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD);
 MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* bitD);


-/* Start by invoking BITv07_initDStream().
-*  A chunk of the bitStream is then stored into a local register.
-*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-*  You can then retrieve bitFields stored into the local register, **in reverse order**.
-*  Local register is explicitly reloaded from memory by the BITv07_reloadDStream() method.
-*  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv07_DStream_unfinished.
-*  Otherwise, it can be less than that, so proceed accordingly.
-*  Checking if DStream has reached its end can be performed with BITv07_endOfDStream().
-*/
-

 /*-****************************************
 *  unsafe API
@@ -533,7 +523,7 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, unsigned nbBits);
 /*-**************************************************************
 *  Internal functions
 ****************************************************************/
-MEM_STATIC unsigned BITv07_highbit32 (register U32 val)
+MEM_STATIC unsigned BITv07_highbit32 (U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
    unsigned long r=0;
@@ -599,13 +589,6 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
 }


-/*! BITv07_lookBits() :
- *  Provides next n bits from local register.
- *  local register is not modified.
- *  On 32-bits, maxNbBits==24.
- *  On 64-bits, maxNbBits==56.
- *  @return : value extracted
- */
 MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits)
 {
    U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -625,11 +608,6 @@ MEM_STATIC void BITv07_skipBits(BITv07_DStream_t* bitD, U32 nbBits)
    bitD->bitsConsumed += nbBits;
 }

-/*! BITv07_readBits() :
- *  Read (consume) next n bits from local register and update.
- *  Pay attention to not read more than nbBits contained into local register.
- *  @return : extracted value.
- */
 MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, U32 nbBits)
 {
    size_t const value = BITv07_lookBits(bitD, nbBits);
@@ -646,11 +624,6 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits)
    return value;
 }

-/*! BITv07_reloadDStream() :
-*   Refill `BITv07_DStream_t` from src buffer previously defined (see BITv07_initDStream() ).
-*   This function is safe, it guarantees it will not read beyond src buffer.
-*   @return : status of `BITv07_DStream_t` internal register.
-              if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
 MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD)
 {
    if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should not happen => corruption detected */
@@ -874,55 +847,6 @@ static void     FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t*
 static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD);


-/**<
-Let's now decompose FSEv07_decompress_usingDTable() into its unitary components.
-You will decode FSE-encoded symbols from the bitStream,
-and also any other bitFields you put in, **in reverse order**.
-
-You will need a few variables to track your bitStream. They are :
-
-BITv07_DStream_t DStream;    // Stream context
-FSEv07_DState_t  DState;     // State context. Multiple ones are possible
-FSEv07_DTable*   DTablePtr;  // Decoding table, provided by FSEv07_buildDTable()
-
-The first thing to do is to init the bitStream.
-    errorCode = BITv07_initDStream(&DStream, srcBuffer, srcSize);
-
-You should then retrieve your initial state(s)
-(in reverse flushing order if you have several ones) :
-    errorCode = FSEv07_initDState(&DState, &DStream, DTablePtr);
-
-You can then decode your data, symbol after symbol.
-For information the maximum number of bits read by FSEv07_decodeSymbol() is 'tableLog'.
-Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
-    unsigned char symbol = FSEv07_decodeSymbol(&DState, &DStream);
-
-You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-Note : maximum allowed nbBits is 25, for 32-bits compatibility
-    size_t bitField = BITv07_readBits(&DStream, nbBits);
-
-All above operations only read from local register (which size depends on size_t).
-Refueling the register from memory is manually performed by the reload method.
-    endSignal = FSEv07_reloadDStream(&DStream);
-
-BITv07_reloadDStream() result tells if there is still some more data to read from DStream.
-BITv07_DStream_unfinished : there is still some data left into the DStream.
-BITv07_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-BITv07_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-BITv07_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-
-When reaching end of buffer (BITv07_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-to properly detect the exact end of stream.
-After each decoded symbol, check if DStream is fully consumed using this simple test :
-    BITv07_reloadDStream(&DStream) >= BITv07_DStream_completed
-
-When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-Checking if DStream has reached its end is performed by :
-    BITv07_endOfDStream(&DStream);
-Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
-    FSEv07_endOfDState(&DState);
-*/
-

 /* *****************************************
 *  FSE unsafe API
@@ -59,7 +59,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    3
-#define ZSTD_VERSION_RELEASE  1
+#define ZSTD_VERSION_RELEASE  3

 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< useful to check dll version */
@@ -131,10 +131,11 @@ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t


 /*======  Helper functions  ======*/
-ZSTDLIB_API int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
+#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
 ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
 ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
+ZSTDLIB_API int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */


 /***************************************
@@ -376,30 +377,31 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
 #define ZSTD_MAGIC_DICTIONARY       0xEC30A437   /* v0.7+ */

-#define ZSTD_WINDOWLOG_MAX_32  27
-#define ZSTD_WINDOWLOG_MAX_64  27
+#define ZSTD_WINDOWLOG_MAX_32   30
+#define ZSTD_WINDOWLOG_MAX_64   31
 #define ZSTD_WINDOWLOG_MAX    ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
-#define ZSTD_WINDOWLOG_MIN     10
-#define ZSTD_HASHLOG_MAX       ZSTD_WINDOWLOG_MAX
-#define ZSTD_HASHLOG_MIN        6
-#define ZSTD_CHAINLOG_MAX     (ZSTD_WINDOWLOG_MAX+1)
-#define ZSTD_CHAINLOG_MIN      ZSTD_HASHLOG_MIN
-#define ZSTD_HASHLOG3_MAX      17
-#define ZSTD_SEARCHLOG_MAX    (ZSTD_WINDOWLOG_MAX-1)
-#define ZSTD_SEARCHLOG_MIN      1
-#define ZSTD_SEARCHLENGTH_MAX   7   /* only for ZSTD_fast, other strategies are limited to 6 */
-#define ZSTD_SEARCHLENGTH_MIN   3   /* only for ZSTD_btopt, other strategies are limited to 4 */
-#define ZSTD_TARGETLENGTH_MIN   4
-#define ZSTD_TARGETLENGTH_MAX 999
-#define ZSTD_LDM_MINMATCH_MIN 4
+#define ZSTD_WINDOWLOG_MIN      10
+#define ZSTD_HASHLOG_MAX        MIN(ZSTD_WINDOWLOG_MAX, 30)
+#define ZSTD_HASHLOG_MIN         6
+#define ZSTD_CHAINLOG_MAX       MIN(ZSTD_WINDOWLOG_MAX+1, 30)
+#define ZSTD_CHAINLOG_MIN       ZSTD_HASHLOG_MIN
+#define ZSTD_HASHLOG3_MAX       17
+#define ZSTD_SEARCHLOG_MAX     (ZSTD_WINDOWLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN       1
+#define ZSTD_SEARCHLENGTH_MAX    7   /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_SEARCHLENGTH_MIN    3   /* only for ZSTD_btopt, other strategies are limited to 4 */
+#define ZSTD_TARGETLENGTH_MIN    4   /* only useful for btopt */
+#define ZSTD_TARGETLENGTH_MAX  999   /* only useful for btopt */
+#define ZSTD_LDM_MINMATCH_MIN    4
 #define ZSTD_LDM_MINMATCH_MAX 4096
 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8

-#define ZSTD_FRAMEHEADERSIZE_MAX 18    /* for static allocation */
-#define ZSTD_FRAMEHEADERSIZE_MIN  6
-static const size_t ZSTD_frameHeaderSize_prefix = 5;  /* minimum input size to know frame header size */
-static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
+#define ZSTD_FRAMEHEADERSIZE_PREFIX 5   /* minimum input size to know frame header size */
+#define ZSTD_FRAMEHEADERSIZE_MIN    6
+#define ZSTD_FRAMEHEADERSIZE_MAX   18   /* for static allocation */
+static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX;
 static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
+static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
 static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable frame length */


@@ -430,12 +432,12 @@ typedef struct {

 typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;

-/*= Custom memory allocation functions */
+/*--- Custom memory allocation functions ---*/
 typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
 /* use this constant to defer to stdlib's functions */
-static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
+static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };


 /***************************************
@@ -444,7 +446,7 @@ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };

 /*! ZSTD_findFrameCompressedSize() :
 *  `src` should point to the start of a ZSTD encoded frame or skippable frame
- *  `srcSize` must be at least as large as the frame
+ *  `srcSize` must be >= first frame size
 *  @return : the compressed size of the first frame starting at `src`,
 *            suitable to pass to `ZSTD_decompress` or similar,
 *            or an error code if input is invalid */
@@ -486,7 +488,7 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);

 /*! ZSTD_sizeof_*() :
 *  These functions give the current memory usage of selected object.
- *  Object memory usage can evolve if it's re-used multiple times. */
+ *  Object memory usage can evolve when re-used multiple times. */
 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
 ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
@@ -499,21 +501,21 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 *  of a future {D,C}Ctx, before its creation.
 *  ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
 *  It will also consider src size to be arbitrarily "large", which is worst case.
- *  If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced_usingCParams() can provide a tighter estimation.
- *  ZSTD_estimateCCtxSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- *  ZSTD_estimateCCtxSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1.
+ *  If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
+ *  ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ *  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1.
 *  Note : CCtx estimation is only correct for single-threaded compression */
 ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
-ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced_usingCParams(ZSTD_compressionParameters cParams);
-ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);

 /*! ZSTD_estimateCStreamSize() :
 *  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
 *  It will also consider src size to be arbitrarily "large", which is worst case.
- *  If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced_usingCParams() can provide a tighter estimation.
- *  ZSTD_estimateCStreamSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- *  ZSTD_estimateCStreamSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1.
+ *  If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
+ *  ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1.
 *  Note : CStream estimation is only correct for single-threaded compression.
 *  ZSTD_DStream memory budget depends on window Size.
 *  This information can be passed manually, using ZSTD_estimateDStreamSize,
@@ -522,14 +524,14 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 *         an internal ?Dict will be created, which additional size is not estimated here.
 *         In this case, get total size by adding ZSTD_estimate?DictSize */
 ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
-ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced_usingCParams(ZSTD_compressionParameters cParams);
-ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);

 typedef enum {
-    ZSTD_dlm_byCopy = 0,      /* Copy dictionary content internally. */
-    ZSTD_dlm_byRef,           /* Reference dictionary content -- the dictionary buffer must outlives its users. */
+    ZSTD_dlm_byCopy = 0,     /**< Copy dictionary content internally */
+    ZSTD_dlm_byRef,          /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
 } ZSTD_dictLoadMethod_e;

 /*! ZSTD_estimate?DictSize() :
@@ -555,7 +557,8 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 *             It must outlive context usage.
 *  workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
 *                 to determine how large workspace must be to support scenario.
- * @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
+ * @return : pointer to ZSTD_CCtx* (same address as workspace, but different type),
+ *           or NULL if error (typically size too small)
 *  Note : zstd will never resize nor malloc() when using a static cctx.
 *         If it needs more memory than available, it will simply error out.
 *  Note 2 : there is no corresponding "free" function.
@@ -585,7 +588,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
                                                  ZSTD_compressionParameters cParams,
                                                  ZSTD_customMem customMem);

-/*! ZSTD_initStaticCDict_advanced() :
+/*! ZSTD_initStaticCDict() :
 *  Generate a digested dictionary in provided memory area.
 *  workspace: The memory area to emplace the dictionary into.
 *             Provided pointer must 8-bytes aligned.
@@ -594,7 +597,8 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
 *                 to determine how large workspace must be.
 *  cParams : use ZSTD_getCParams() to transform a compression level
 *            into its relevants cParams.
- * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ * @return : pointer to ZSTD_CDict* (same address as workspace, but different type),
+ *           or NULL if error (typically, size too small).
 *  Note : there is no corresponding "free" function.
 *         Since workspace was allocated externally, it must be freed externally.
 */
@@ -611,7 +615,7 @@ ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, uns

 /*! ZSTD_getParams() :
 *   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
-*   All fields of `ZSTD_frameParameters` are set to default (0) */
+*   All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
 ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);

 /*! ZSTD_checkCParams() :
@@ -658,7 +662,8 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
 *             It must outlive context usage.
 *  workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
 *                 to determine how large workspace must be to support scenario.
- * @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
+ * @return : pointer to ZSTD_DCtx* (same address as workspace, but different type),
+ *           or NULL if error (typically size too small)
 *  Note : zstd will never resize nor malloc() when using a static dctx.
 *         If it needs more memory than available, it will simply error out.
 *  Note 2 : static dctx is incompatible with legacy support
@@ -729,30 +734,32 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 /*=====   Advanced Streaming compression functions  =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticCCtx() */
-ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */

 /*! ZSTD_resetCStream() :
 *  start a new compression job, using same parameters from previous job.
 *  This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
 *  Note that zcs must be init at least once before using ZSTD_resetCStream().
- *  pledgedSrcSize==0 means "srcSize unknown".
+ *  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
 *  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
- *  @return : 0, or an error code (which can be tested using ZSTD_isError()) */
+ *  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ *  but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);


 /*=====   Advanced Streaming decompression functions  =====*/
-typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticDCtx() */
-ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
-ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
-ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
+typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
+ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);   /* obsolete : this API will be removed in a future version */
+ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
+ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict is referenced, it must outlive decompression session */
 ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */


@@ -760,8 +767,8 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompress
 *  Buffer-less and synchronous inner streaming functions
 *
 *  This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
-*  But it's also a complex one, with many restrictions (documented below).
-*  Prefer using normal streaming API for an easier experience
+*  But it's also a complex one, with several restrictions, documented below.
+*  Prefer normal streaming API for an easier experience.
 ********************************************************************* */

 /**
@@ -778,8 +785,8 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompress

  Then, consume your input using ZSTD_compressContinue().
  There are some important considerations to keep in mind when using this advanced function :
-  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
-  - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks.
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
+  - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
    Worst case evaluation is provided by ZSTD_compressBound().
    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
@@ -790,18 +797,18 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompress

  Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
-  Without last block mark, frames will be considered unfinished (corrupted) by decoders.
+  Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.

-  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
+  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
 */

 /*=====   Buffer-less streaming compression functions  =====*/
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
 ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
+ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */

 ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -908,40 +915,64 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);



-/*===   New advanced API (experimental, and compression only)  ===*/
+/* ============================================ */
+/**       New advanced API (experimental)       */
+/* ============================================ */

 /* notes on API design :
- *   In this proposal, parameters are pushed one by one into an existing CCtx,
+ *   In this proposal, parameters are pushed one by one into an existing context,
 *   and then applied on all subsequent compression jobs.
 *   When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
 *
 *   This API is intended to replace all others experimental API.
 *   It can basically do all other use cases, and even new ones.
- *   It stands a good chance to become "stable",
- *   after a reasonable testing period.
+ *   In constrast with _advanced() variants, it stands a reasonable chance to become "stable",
+ *   after a good testing period.
 */

 /* note on naming convention :
 *   Initially, the API favored names like ZSTD_setCCtxParameter() .
 *   In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
 *   The main driver is that it identifies more clearly the target object type.
- *   It feels clearer in light of potential variants :
+ *   It feels clearer when considering multiple targets :
 *   ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
- *   ZSTD_DCtx_setParameter()  (rather than ZSTD_setDCtxParameter() )
- *   Left variant feels easier to distinguish.
+ *   ZSTD_CCtxParams_setParameter()  (rather than ZSTD_setCCtxParamsParameter() )
+ *   etc...
 */

 /* note on enum design :
- * All enum will be manually set to explicit values before reaching "stable API" status */
+ * All enum will be pinned to explicit values before reaching "stable API" status */

 typedef enum {
+    /* Question : should we have a format ZSTD_f_auto ?
+     * For the time being, it would mean exactly the same as ZSTD_f_zstd1.
+     * But, in the future, should several formats be supported,
+     * on the compression side, it would mean "default format".
+     * On the decompression side, it would mean "multi format",
+     * and ZSTD_f_zstd1 could be reserved to mean "accept *only* zstd frames".
+     * Since meaning is a little different, another option could be to define different enums for compression and decompression.
+     * This question could be kept for later, when there are actually multiple formats to support,
+     * but there is also the question of pinning enum values, and pinning value `0` is especially important */
+    ZSTD_f_zstd1 = 0,        /* zstd frame format, specified in zstd_compression_format.md (default) */
+    ZSTD_f_zstd1_magicless,  /* Variant of zstd frame format, without initial 4-bytes magic number.
+                              * Useful to save 4 bytes per generated frame.
+                              * Decoder cannot recognise automatically this format, requiring instructions. */
+} ZSTD_format_e;
+
+typedef enum {
+    /* compression format */
+    ZSTD_p_format = 10,      /* See ZSTD_format_e enum definition.
+                              * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
+
    /* compression parameters */
    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
                              * Default level is ZSTD_CLEVEL_DEFAULT==3.
                              * Special: value 0 means "do not change cLevel". */
    ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * Special: value 0 means "do not change windowLog". */
+                              * Special: value 0 means "do not change windowLog".
+                              * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
+                              * requires setting the maximum window size at least as large during decompression. */
    ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
                              * Resulting table size is (1 << (hashLog+2)).
                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -974,18 +1005,19 @@ typedef enum {
                              * Special: value 0 means "do not change strategy". */

    /* frame parameters */
-    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1)
-                              * note that content size must be known at the beginning,
-                              * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
+    ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+                              * Content size must be known at the beginning of compression,
+                              * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
    ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
-    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+    ZSTD_p_dictIDFlag,       /* When applicable, dictionary's ID is written into frame header (default:1) */

    /* multi-threading parameters */
    ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
                              * More threads improve speed, but also increase memory usage.
                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
                              * Special: value 0 means "do not change nbThreads" */
-    ZSTD_p_jobSize,          /* Size of a compression job. Each compression job is completed in parallel.
+    ZSTD_p_jobSize,          /* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.
+                              * Each compression job is completed in parallel, so indirectly controls the nb of active threads.
                              * 0 means default, which is dynamically determined based on compression parameters.
                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                              * The minimum size is automatically and transparently enforced */
@@ -1009,7 +1041,7 @@ typedef enum {
                          * Larger values increase memory usage and compression ratio, but decrease
                          * compression speed.
                          * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
-                          * (default: 20). */
+                          * (default: windowlog - 7). */
    ZSTD_p_ldmMinMatch,  /* Minimum size of searched matches for long distance matcher.
                          * Larger/too small values usually decrease compression ratio.
                          * Must be clamped between ZSTD_LDM_MINMATCH_MIN
@@ -1031,7 +1063,8 @@ typedef enum {
 /*! ZSTD_CCtx_setParameter() :
 *  Set one compression parameter, selected by enum ZSTD_cParameter.
 *  Note : when `value` is an enum, cast it to unsigned for proper type checking.
- *  @result : 0, or an error code (which can be tested with ZSTD_isError()). */
+ *  @result : informational value (typically, the one being set, possibly corrected),
+ *            or an error code (which can be tested with ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);

 /*! ZSTD_CCtx_setPledgedSrcSize() :
@@ -1040,7 +1073,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
 *  Note 1 : 0 means zero, empty.
 *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- *           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+ *           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
 *  Note 2 : If all data is provided and consumed in a single round,
 *           this value is overriden by srcSize instead. */
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
@@ -1103,22 +1136,28 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
 typedef enum {
    ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
-    ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
+    ZSTD_e_end         /* flush any remaining data and close current frame. Any additional data starts a new frame. */
 } ZSTD_EndDirective;

 /*! ZSTD_compress_generic() :
 *  Behave about the same as ZSTD_compressStream. To note :
 *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
 *  - Compression parameters cannot be changed once compression is started.
- *  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
- *  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
- *  - @return provides the minimum amount of data still to flush from internal buffers
+ *  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
+ *  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ *  - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
+ *  - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
+ *                                                     and then immediately returns, just indicating that there is some data remaining to be flushed.
+ *                                                     The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ *  - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
+ *  - @return provides the minimum amount of data remaining to be flushed from internal buffers
 *            or an error code, which can be tested using ZSTD_isError().
- *            if @return != 0, flush is not fully completed, there is some data left within internal buffers.
- *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
+ *            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ *            This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
+ *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
 *            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
- *            It is necessary to fully flush internal buffers
- *            before starting a new compression job, or changing compression parameters.
+ *            Before starting a new compression job, or changing compression parameters,
+ *            it is required to fully flush internal buffers.
 */
 ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
                                          ZSTD_outBuffer* output,
@@ -1130,6 +1169,7 @@ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 *  Useful after an error, or to interrupt an ongoing compression job and start a new one.
 *  Any internal data not yet flushed is cancelled.
 *  Dictionary (if any) is dropped.
+ *  All parameters are back to default values.
 *  It's possible to modify compression parameters after a reset.
 */
 ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
@@ -1138,30 +1178,33 @@ ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
 /*! ZSTD_compress_generic_simpleArgs() :
 *  Same as ZSTD_compress_generic(),
 *  but using only integral types as arguments.
- *  Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
+ *  Argument list is larger than ZSTD_{in,out}Buffer,
 *  but can be helpful for binders from dynamic languages
 *  which have troubles handling structures containing memory pointers.
 */
-size_t ZSTD_compress_generic_simpleArgs (
+ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs (
                            ZSTD_CCtx* cctx,
                            void* dst, size_t dstCapacity, size_t* dstPos,
                      const void* src, size_t srcSize, size_t* srcPos,
                            ZSTD_EndDirective endOp);


-/** ZSTD_CCtx_params
- *
+/*! ZSTD_CCtx_params :
+ *  Quick howto :
 *  - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
- *  - ZSTD_CCtxParam_setParameter() : Push parameters one by one into an
- *  existing ZSTD_CCtx_params structure. This is similar to
- *  ZSTD_CCtx_setParameter().
- *  - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to an existing CCtx. These
- *  parameters will be applied to all subsequent compression jobs.
+ *  - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
+ *                                    an existing ZSTD_CCtx_params structure.
+ *                                    This is similar to
+ *                                    ZSTD_CCtx_setParameter().
+ *  - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+ *                                    an existing CCtx.
+ *                                    These parameters will be applied to
+ *                                    all subsequent compression jobs.
 *  - ZSTD_compress_generic() : Do compression using the CCtx.
 *  - ZSTD_freeCCtxParams() : Free the memory.
 *
- *  This can be used with ZSTD_estimateCCtxSize_opaque() for static allocation
- *  for single-threaded compression.
+ *  This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+ *  for static allocation for single-threaded compression.
 */
 ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);

@@ -1189,22 +1232,133 @@ ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 *  Set one compression parameter, selected by enum ZSTD_cParameter.
 *  Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
 *  Note : when `value` is an enum, cast it to unsigned for proper type checking.
- *  @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
 */
 ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);

 /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
- * Apply a set of ZSTD_CCtx_params to the compression context.
- * This must be done before the dictionary is loaded.
- * The pledgedSrcSize is treated as unknown.
- * Multithreading parameters are applied only if nbThreads > 1.
+ *  Apply a set of ZSTD_CCtx_params to the compression context.
+ *  This must be done before the dictionary is loaded.
+ *  The pledgedSrcSize is treated as unknown.
+ *  Multithreading parameters are applied only if nbThreads > 1.
 */
 ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
        ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);

-/**
-    Block functions

+/*===   Advanced parameters for decompression API  ===*/
+
+/* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object,
+ * but before starting decompression of a frame.
+ */
+
+/*! ZSTD_DCtx_loadDictionary() :
+ *  Create an internal DDict from dict buffer,
+ *  to be used to decompress next frames.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ *            meaning "return to no-dictionary mode".
+ *  Note 1 : `dict` content will be copied internally.
+ *            Use ZSTD_DCtx_loadDictionary_byReference()
+ *            to reference dictionary content instead.
+ *            In which case, the dictionary buffer must outlive its users.
+ *  Note 2 : Loading a dictionary involves building tables,
+ *           which has a non-negligible impact on CPU usage and latency.
+ *  Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
+ *           how dictionary content will be interpreted and loaded.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);   /* not implemented */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);   /* not implemented */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);   /* not implemented */
+
+
+/*! ZSTD_DCtx_refDDict() :
+ *  Reference a prepared dictionary, to be used to decompress next frames.
+ *  The dictionary remains active for decompression of future frames using same DCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : Currently, only one dictionary can be managed.
+ *           Referencing a new dictionary effectively "discards" any previous one.
+ *  Special : adding a NULL DDict means "return to no-dictionary mode".
+ *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);   /* not implemented */
+
+
+/*! ZSTD_DCtx_refPrefix() :
+ *  Reference a prefix (single-usage dictionary) for next compression job.
+ *  Prefix is **only used once**. It must be explicitly referenced before each frame.
+ *  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ *  Note 2 : Prefix buffer is referenced. It must outlive compression job.
+ *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+ *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
+ *  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize);   /* not implemented */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);   /* not implemented */
+
+
+/*! ZSTD_DCtx_setMaxWindowSize() :
+ *  Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+ *  This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
+ *  This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
+ *  By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+
+
+/*! ZSTD_DCtx_setFormat() :
+ *  Instruct the decoder context about what kind of data to decode next.
+ *  This instruction is mandatory to decode data without a fully-formed header,
+ *  such ZSTD_f_zstd1_magicless for example.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+
+
+/*! ZSTD_decompress_generic() :
+ *  Behave the same as ZSTD_decompressStream.
+ *  Decompression parameters cannot be changed once decompression is started.
+ * @return : an error code, which can be tested using ZSTD_isError()
+ *           if >0, a hint, nb of expected input bytes for next invocation.
+ *           `0` means : a frame has just been fully decoded and flushed.
+ */
+ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
+                                           ZSTD_outBuffer* output,
+                                           ZSTD_inBuffer* input);
+
+
+/*! ZSTD_decompress_generic_simpleArgs() :
+ *  Same as ZSTD_decompress_generic(),
+ *  but using only integral types as arguments.
+ *  Argument list is larger than ZSTD_{in,out}Buffer,
+ *  but can be helpful for binders from dynamic languages
+ *  which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs (
+                            ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos);
+
+
+/*! ZSTD_DCtx_reset() :
+ *  Return a DCtx to clean state.
+ *  If a decompression was ongoing, any internal data not yet flushed is cancelled.
+ *  All parameters are back to default values, including sticky ones.
+ *  Dictionary (if any) is dropped.
+ *  Parameters can be modified again after a reset.
+ */
+ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
+
+
+
+/* ============================ */
+/**       Block level API       */
+/* ============================ */
+
+/*!
    Block functions produce and decode raw zstd blocks, without frame metadata.
    Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
    User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -1216,7 +1370,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
      + compression : any ZSTD_compressBegin*() variant, including with dictionary
      + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
      + copyCCtx() and copyDCtx() can be used too
-    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
+    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
      + If input is larger than a block size, it's necessary to split input data into multiple blocks
      + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
        Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -10,38 +10,19 @@ cxx_binary(
        '//lib:mem',
        '//lib:xxhash',
    ],
-)
-
-cxx_binary(
-    name='zstdmt',
-    headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
-    srcs=glob(['*.c'], excludes=['datagen.c']),
-    deps=[
-        ':datagen',
-        ':util',
-        '//lib:zstd',
-        '//lib:zdict',
-        '//lib:mem',
-        '//lib:xxhash',
+    preprocessor_flags=[
+        '-DZSTD_GZCOMPRESS',
+        '-DZSTD_GZDECOMPRESS',
+        '-DZSTD_LZMACOMPRESS',
+        '-DZSTD_LZMADECOMPRES',
+        '-DZSTD_LZ4COMPRESS',
+        '-DZSTD_LZ4DECOMPRES',
    ],
-    preprocessor_flags=['-DZSTD_MULTITHREAD'],
-    linker_flags=['-lpthread'],
-)
-
-cxx_binary(
-    name='gzstd',
-    headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
-    srcs=glob(['*.c'], excludes=['datagen.c']),
-    deps=[
-        ':datagen',
-        ':util',
-        '//lib:zstd',
-        '//lib:zdict',
-        '//lib:mem',
-        '//lib:xxhash',
+    linker_flags=[
+        '-lz',
+        '-llzma',
+        '-llz4',
    ],
-    preprocessor_flags=['-DZSTD_GZDECOMPRESS'],
-    linker_flags=['-lz'],
 )

 cxx_library(
@@ -37,10 +37,9 @@ endif

 CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
           -I$(ZSTDDIR)/dictBuilder \
-           -DZSTD_NEWAPI \
-           -DXXH_NAMESPACE=ZSTD_   # because xxhash.o already compiled with this macro from library
+           -DXXH_NAMESPACE=ZSTD_
 CFLAGS  ?= -O3
-DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
@@ -66,6 +65,7 @@ endif
 else
 endif

+# Sort files in alphabetical order for reproducible builds
 ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES)))

 # Define *.exe as extension for Windows systems
@@ -140,13 +140,10 @@ allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy

 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)

-zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP)
-zstd zstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD)
-zstd4 : CPPFLAGS += $(LZ4CPP)
-zstd4 : LDFLAGS += $(LZ4LD)
-zstd : LZ4_MSG := - lz4 support is disabled
-zstd zstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd zstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
+zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
+zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD)
+zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
 	@echo "$(THREAD_MSG)"
 	@echo "$(ZLIB_MSG)"
 	@echo "$(LZMA_MSG)"
@@ -229,8 +226,8 @@ clean:
 MD2ROFF = ronn
 MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"

-zstd.1: zstd.1.md
-	cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+zstd.1: zstd.1.md ../lib/zstd.h
+	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@

 .PHONY: man
 man: zstd.1
@@ -266,9 +263,10 @@ mandir      ?= $(datarootdir)/man
 man1dir     ?= $(mandir)/man1

 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
-MANDIR  ?= $(PREFIX)/man/man1
+MANDIR  ?= $(PREFIX)/man
+MAN1DIR ?= $(MANDIR)/man1
 else
-MANDIR  ?= $(man1dir)
+MAN1DIR ?= $(man1dir)
 endif

 ifneq (,$(filter $(shell uname),SunOS))
@@ -285,7 +283,7 @@ INSTALL_MAN     ?= $(INSTALL_DATA)
 .PHONY: install
 install: zstd
 	@echo Installing binaries
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/
 	@$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
@@ -293,9 +291,9 @@ install: zstd
 	@$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
 	@$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
 	@echo Installing man pages
-	@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
-	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
-	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
+	@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
+	@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
+	@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
 	@echo zstd installation completed

 .PHONY: uninstall
@@ -305,9 +303,9 @@ uninstall:
 	@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
 	@$(RM) $(DESTDIR)$(BINDIR)/unzstd
 	@$(RM) $(DESTDIR)$(BINDIR)/zstd
-	@$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
-	@$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
-	@$(RM) $(DESTDIR)$(MANDIR)/zstd.1
+	@$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
+	@$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
+	@$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
 	@echo zstd programs successfully uninstalled

 endif
@@ -40,6 +40,16 @@ There are however other Makefile targets that create different variations of CLI
  In which case, linking stage will fail if `lzma` library cannot be found.
  This might be useful to prevent silent feature disabling.

+- __HAVE_LZ4__ : `zstd` can compress and decompress files in `.lz4` formats.
+  This is ordered through commands `--format=lz4`.
+  Alternatively, symlinks named `lz4`, or `unlz4` will mimic intended behavior.
+  `.lz4` support is automatically enabled when `lz4` library is detected at build time.
+  It's possible to disable `.lz4` support, by setting HAVE_LZ4=0 .
+  Example : make zstd HAVE_LZ4=0
+  It's also possible to force compilation with lz4 support, using HAVE_LZ4=1.
+  In which case, linking stage will fail if `lz4` library cannot be found.
+  This might be useful to prevent silent feature disabling.
+
 - __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`.
  Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible.
  But older versions (< v0.8.0) produced different, incompatible, frames.
@@ -34,14 +34,12 @@
 #include <stdlib.h>      /* malloc, free */
 #include <string.h>      /* memset */
 #include <stdio.h>       /* fprintf, fopen */
-#include <time.h>        /* clock_t, clock, CLOCKS_PER_SEC */

 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
 #include "datagen.h"     /* RDG_genBuffer */
 #include "xxhash.h"
-#include "zstdmt_compress.h"


 /* *************************************
@@ -73,12 +71,13 @@ static U32 g_compressibilityDefault = 50;
 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static int g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */

-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
-            { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stderr); } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_time = 0;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
+#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stderr); } } }


 /* *************************************
@@ -130,6 +129,17 @@ void BMK_setNbThreads(unsigned nbThreads) {
 #endif
    g_nbThreads = nbThreads;
 }
+
+static U32 g_realTime = 0;
+void BMK_setRealTime(unsigned priority) {
+    g_realTime = (priority>0);
+}
+
+static U32 g_separateFiles = 0;
+void BMK_setSeparateFiles(unsigned separate) {
+    g_separateFiles = (separate>0);
+}
+
 static U32 g_ldmFlag = 0;
 void BMK_setLdmFlag(unsigned ldmFlag) {
    g_ldmFlag = ldmFlag;
@@ -181,7 +191,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                        const char* displayName, int cLevel,
                        const size_t* fileSizes, U32 nbFiles,
                        const void* dictBuffer, size_t dictBufferSize,
-                        const ZSTD_compressionParameters* comprParams)
+                        const ZSTD_compressionParameters* const comprParams)
 {
    size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
    U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
@@ -189,7 +199,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
    size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
    void* const compressedBuffer = malloc(maxCompressedSize);
    void* resultBuffer = malloc(srcSize);
-    ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
    ZSTD_CCtx* const ctx = ZSTD_createCCtx();
    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
    size_t const loadedCompressedSize = srcSize;
@@ -286,8 +295,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                if (!cCompleted) {   /* still some time to do compression tests */
                    U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
                    U32 nbLoops = 0;
-                    ZSTD_CDict* cdict = NULL;
-#ifdef ZSTD_NEWAPI
                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
@@ -306,84 +313,64 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
                    ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
-#else
-                    size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
-                    ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
-                    ZSTD_customMem const cmem = { NULL, NULL, NULL };
-                    if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
-                    if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
-                    if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
-                    if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
-                    if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
-                    if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
-                    if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
-                    cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem);
-                    if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
-#endif
                    do {
                        U32 blockNb;
                        for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                            size_t rSize;
-#ifdef ZSTD_NEWAPI
-                            ZSTD_outBuffer out = { blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom, 0 };
-                            ZSTD_inBuffer in = { blockTable[blockNb].srcPtr,  blockTable[blockNb].srcSize, 0 };
-                            size_t cError = 1;
-                            while (cError) {
-                                cError = ZSTD_compress_generic(ctx,
+#if 0   /* direct compression function, for occasional comparison */
+                            ZSTD_parameters const params = ZSTD_getParams(cLevel, blockTable[blockNb].srcSize, dictBufferSize);
+                            blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
+                                                            blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+                                                            blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
+                                                            dictBuffer, dictBufferSize,
+                                                            params);
+#else
+                            size_t moreToFlush = 1;
+                            ZSTD_outBuffer out;
+                            ZSTD_inBuffer in;
+                            in.src = blockTable[blockNb].srcPtr;
+                            in.size = blockTable[blockNb].srcSize;
+                            in.pos = 0;
+                            out.dst = blockTable[blockNb].cPtr;
+                            out.size = blockTable[blockNb].cRoom;
+                            out.pos = 0;
+                            while (moreToFlush) {
+                                moreToFlush = ZSTD_compress_generic(ctx,
                                                    &out, &in, ZSTD_e_end);
-                                if (ZSTD_isError(cError))
+                                if (ZSTD_isError(moreToFlush))
                                    EXM_THROW(1, "ZSTD_compress_generic() error : %s",
-                                                ZSTD_getErrorName(cError));
+                                                ZSTD_getErrorName(moreToFlush));
                            }
-                            rSize = out.pos;
-#else  /* ! ZSTD_NEWAPI */
-                            if (dictBufferSize) {
-                                rSize = ZSTD_compress_usingCDict(ctx,
-                                                blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
-                                                cdict);
-                            } else {
-#  ifdef ZSTD_MULTITHREAD       /* note : limitation : MT single-pass does not support compression with dictionary */
-                                rSize = ZSTDMT_compressCCtx(mtctx,
-                                                blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
-                                                cLevel);
-#  else
-                                rSize = ZSTD_compress_advanced (ctx,
-                                                blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
-                                                NULL, 0, zparams);
-#  endif
-                            }
-                            if (ZSTD_isError(rSize))
-                                EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
-                                            ZSTD_getErrorName(rSize));
-#endif  /* ZSTD_NEWAPI */
-                            blockTable[blockNb].cSize = rSize;
+                            blockTable[blockNb].cSize = out.pos;
+#endif
                        }
                        nbLoops++;
                    } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
-                    ZSTD_freeCDict(cdict);
-                    {   U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
-                        if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops;
-                        totalCTime += clockSpanMicro;
-                        cCompleted = (totalCTime >= maxTime);
+                    {   U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
+                        if (loopDuration < fastestC*nbLoops)
+                            fastestC = loopDuration / nbLoops;
+                        totalCTime += loopDuration;
+                        cCompleted = (totalCTime >= maxTime);  /* end compression tests */
                }   }

                cSize = 0;
                { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
                ratio = (double)srcSize / (double)cSize;
                markNb = (markNb+1) % NB_MARKS;
-                DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
-                        marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
-                        (double)srcSize / fastestC );
+                {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                    double const compressionSpeed = (double)srcSize / fastestC;
+                    int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+                    DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
+                            marks[markNb], displayName, (U32)srcSize, (U32)cSize,
+                            ratioAccuracy, ratio,
+                            cSpeedAccuracy, compressionSpeed );
+                }
            } else {   /* g_decodeOnly */
                memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
            }

 #if 0       /* disable decompression test */
            dCompleted=1;
-            (void)totalDTime; (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
+            (void)totalDTime; (void)fastestD; (void)crcOrig;   /* unused when decompression disabled */
 #else
            /* Decompression */
            if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
@@ -413,17 +400,24 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                    nbLoops++;
                } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
                ZSTD_freeDDict(ddict);
-                {   U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
-                    if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops;
-                    totalDTime += clockSpanMicro;
+                {   U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
+                    if (loopDuration < fastestD*nbLoops)
+                        fastestD = loopDuration / nbLoops;
+                    totalDTime += loopDuration;
                    dCompleted = (totalDTime >= maxTime);
            }   }

            markNb = (markNb+1) % NB_MARKS;
-            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
-                    marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
-                    (double)srcSize / fastestC,
-                    (double)srcSize / fastestD );
+            {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                double const compressionSpeed = (double)srcSize / fastestC;
+                int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+                double const decompressionSpeed = (double)srcSize / fastestD;
+                DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
+                        marks[markNb], displayName, (U32)srcSize, (U32)cSize,
+                        ratioAccuracy, ratio,
+                        cSpeedAccuracy, compressionSpeed,
+                        decompressionSpeed);
+            }

            /* CRC Checking */
            {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -444,10 +438,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                            DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
                            if (u>5) {
                                int n;
+                                DISPLAY("origin: ");
                                for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
                                DISPLAY(" :%02X:  ", ((const BYTE*)srcBuffer)[u]);
                                for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
                                DISPLAY(" \n");
+                                DISPLAY("decode: ");
                                for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
                                DISPLAY(" :%02X:  ", ((const BYTE*)resultBuffer)[u]);
                                for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
@@ -463,7 +459,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
 #endif
        }   /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */

-        if (g_displayLevel == 1) {
+        if (g_displayLevel == 1) {   /* hidden display mode -q, used by python speed benchmark */
            double cSpeed = (double)srcSize / fastestC;
            double dSpeed = (double)srcSize / fastestD;
            if (g_additionalParam)
@@ -478,7 +474,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
    free(blockTable);
    free(compressedBuffer);
    free(resultBuffer);
-    ZSTDMT_freeCCtx(mtctx);
    ZSTD_freeCCtx(ctx);
    ZSTD_freeDCtx(dctx);
    return 0;
@@ -503,11 +498,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
    return (size_t)(requiredMem);
 }

-static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
                            const char* displayName, int cLevel, int cLevelLast,
                            const size_t* fileSizes, unsigned nbFiles,
                            const void* dictBuffer, size_t dictBufferSize,
-                            ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
+                            const ZSTD_compressionParameters* const compressionParams)
 {
    int l;

@@ -515,8 +510,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
    if (!pch) pch = strrchr(displayName, '/'); /* Linux */
    if (pch) displayName = pch+1;

-    if (setRealTimePrio) {
-        DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
+    if (g_realTime) {
+        DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
        SET_REALTIME_PRIORITY;
    }

@@ -539,7 +534,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
    At most, fills `buffer` entirely */
 static void BMK_loadFiles(void* buffer, size_t bufferSize,
                          size_t* fileSizes,
-                          const char** fileNamesTable, unsigned nbFiles)
+                          const char* const * const fileNamesTable, unsigned nbFiles)
 {
    size_t pos = 0, totalSize = 0;
    unsigned n;
@@ -551,6 +546,11 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
            fileSizes[n] = 0;
            continue;
        }
+        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAYLEVEL(2, "Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
+            fileSizes[n] = 0;
+            continue;
+        }
        f = fopen(fileNamesTable[n], "rb");
        if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
        DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
@@ -566,26 +566,30 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
    if (totalSize == 0) EXM_THROW(12, "no data to bench");
 }

-static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
-                               int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
+static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
+                               const char* const dictFileName,
+                               int const cLevel, int const cLevelLast,
+                               const ZSTD_compressionParameters* const compressionParams)
 {
    void* srcBuffer;
    size_t benchedSize;
    void* dictBuffer = NULL;
    size_t dictBufferSize = 0;
-    size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+    size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
-    char mfName[20] = {0};

    if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");

    /* Load dictionary */
    if (dictFileName != NULL) {
-        U64 dictFileSize = UTIL_getFileSize(dictFileName);
-        if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
+        U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+        if (dictFileSize > 64 MB)
+            EXM_THROW(10, "dictionary file %s too large", dictFileName);
        dictBufferSize = (size_t)dictFileSize;
        dictBuffer = malloc(dictBufferSize);
-        if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
+        if (dictBuffer==NULL)
+            EXM_THROW(11, "not enough memory for dictionary (%u bytes)",
+                            (U32)dictBufferSize);
        BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
    }

@@ -601,13 +605,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
    BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);

    /* Bench */
-    snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
-    {   const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
-        BMK_benchCLevel(srcBuffer, benchedSize,
-                        displayName, cLevel, cLevelLast,
-                        fileSizes, nbFiles,
-                        dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
-    }
+    if (g_separateFiles) {
+        const BYTE* srcPtr = (const BYTE*)srcBuffer;
+        U32 fileNb;
+        for (fileNb=0; fileNb<nbFiles; fileNb++) {
+            size_t const fileSize = fileSizes[fileNb];
+            BMK_benchCLevel(srcPtr, fileSize,
+                            fileNamesTable[fileNb], cLevel, cLevelLast,
+                            fileSizes+fileNb, 1,
+                            dictBuffer, dictBufferSize, compressionParams);
+            srcPtr += fileSize;
+        }
+    } else {
+        char mfName[20] = {0};
+        snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+        {   const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+            BMK_benchCLevel(srcBuffer, benchedSize,
+                            displayName, cLevel, cLevelLast,
+                            fileSizes, nbFiles,
+                            dictBuffer, dictBufferSize, compressionParams);
+    }   }

    /* clean up */
    free(srcBuffer);
@@ -616,7 +633,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
 }


-static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams)
 {
    char name[20] = {0};
    size_t benchedSize = 10000000;
@@ -630,15 +647,17 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility

    /* Bench */
    snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
-    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
+    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);

    /* clean up */
    free(srcBuffer);
 }


-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
-                   int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+                   const char* dictFileName,
+                   int cLevel, int cLevelLast,
+                   const ZSTD_compressionParameters* compressionParams)
 {
    double const compressibility = (double)g_compressibilityDefault / 100;

@@ -646,11 +665,12 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
    if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
    if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
    if (cLevelLast < cLevel) cLevelLast = cLevel;
-    if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+    if (cLevelLast > cLevel)
+        DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);

    if (nbFiles == 0)
-        BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
+        BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
    else
-        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
+        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
    return 0;
 }
@@ -16,14 +16,16 @@
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
 #include "zstd.h"     /* ZSTD_compressionParameters */

-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
-                   int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio);
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
+                   int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams);

 /* Set Parameters */
 void BMK_setNbSeconds(unsigned nbLoops);
 void BMK_setBlockSize(size_t blockSize);
 void BMK_setNbThreads(unsigned nbThreads);
+void BMK_setRealTime(unsigned priority);
 void BMK_setNotificationLevel(unsigned level);
+void BMK_setSeparateFiles(unsigned separate);
 void BMK_setAdditionalParam(int additionalParam);
 void BMK_setDecodeOnlyMode(unsigned decodeFlag);
 void BMK_setLdmFlag(unsigned ldmFlag);
@@ -26,7 +26,6 @@
 #include <stdlib.h>         /* malloc, free */
 #include <string.h>         /* memset */
 #include <stdio.h>          /* fprintf, fopen, ftello64 */
-#include <time.h>           /* clock_t, clock, CLOCKS_PER_SEC */
 #include <errno.h>          /* errno */

 #include "mem.h"            /* read */
@@ -55,15 +54,13 @@ static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((siz
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }

-#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
-            if ((DIB_clockSpan(g_time) > refreshRate) || (displayLevel>=4)) \
-            { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (displayLevel>=4) fflush(stderr); } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10;
-static clock_t g_time = 0;
-
-static clock_t DIB_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

+#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (displayLevel>=4) fflush(stderr); } } }

 /*-*************************************
 *  Exceptions
@@ -117,7 +114,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
    for (fileIndex=0; fileIndex<nbFiles; fileIndex++) {
        const char* const fileName = fileNamesTable[fileIndex];
        unsigned long long const fs64 = UTIL_getFileSize(fileName);
-        unsigned long long remainingToLoad = fs64;
+        unsigned long long remainingToLoad = (fs64 == UTIL_FILESIZE_UNKNOWN) ? 0 : fs64;
        U32 const nbChunks = targetChunkSize ? (U32)((fs64 + (targetChunkSize-1)) / targetChunkSize) : 1;
        U64 const chunkSize = targetChunkSize ? MIN(targetChunkSize, fs64) : fs64;
        size_t const maxChunkSize = (size_t)MIN(chunkSize, SAMPLESIZE_MAX);
@@ -245,8 +242,9 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si
    memset(&fs, 0, sizeof(fs));
    for (n=0; n<nbFiles; n++) {
        U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
-        U32 const nbSamples = (U32)(chunkSize ? (fileSize + (chunkSize-1)) / chunkSize : 1);
-        U64 const chunkToLoad = chunkSize ? MIN(chunkSize, fileSize) : fileSize;
+        U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? 0 : fileSize;
+        U32 const nbSamples = (U32)(chunkSize ? (srcSize + (chunkSize-1)) / chunkSize : 1);
+        U64 const chunkToLoad = chunkSize ? MIN(chunkSize, srcSize) : srcSize;
        size_t const cappedChunkSize = (size_t)MIN(chunkToLoad, SAMPLESIZE_MAX);
        fs.totalSizeToLoad += cappedChunkSize * nbSamples;
        fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX);
@@ -25,11 +25,10 @@
 *  Includes
 ***************************************/
 #include "platform.h"   /* Large Files support, SET_BINARY_MODE */
-#include "util.h"       /* UTIL_getFileSize */
+#include "util.h"       /* UTIL_getFileSize, UTIL_isRegularFile */
 #include <stdio.h>      /* fprintf, fopen, fread, _fileno, stdin, stdout */
 #include <stdlib.h>     /* malloc, free */
 #include <string.h>     /* strcmp, strlen */
-#include <time.h>       /* clock */
 #include <errno.h>      /* errno */

 #if defined (_MSC_VER)
@@ -37,13 +36,12 @@
 #  include <io.h>
 #endif

+#include "bitstream.h"
 #include "mem.h"
 #include "fileio.h"
+#include "util.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
 #include "zstd.h"
-#ifdef ZSTD_MULTITHREAD
-#  include "zstdmt_compress.h"
-#endif
 #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
 #  include <zlib.h>
 #  if !defined(z_const)
@@ -56,6 +54,7 @@

 #define LZ4_MAGICNUMBER 0x184D2204
 #if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
+#  define LZ4F_ENABLE_OBSOLETE_ENUMS
 #  include <lz4frame.h>
 #  include <lz4.h>
 #endif
@@ -68,18 +67,6 @@
 #define MB *(1<<20)
 #define GB *(1U<<30)

-#define _1BIT  0x01
-#define _2BITS 0x03
-#define _3BITS 0x07
-#define _4BITS 0x0F
-#define _6BITS 0x3F
-#define _8BITS 0xFF
-
-#define BLOCKSIZE      (128 KB)
-#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
-
-#define FIO_FRAMEHEADERSIZE  5    /* as a define, because needed to allocated table on stack */
-
 #define DICTSIZE_MAX (32 MB)   /* protection against large input (attack scenario) */

 #define FNSPACE 30
@@ -94,12 +81,13 @@
 static int g_displayLevel = 2;   /* 0 : no display;  1: errors;  2: + result + interaction + warnings;  3: + progression;  4: + information */
 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }

+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
 #define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
-            if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
-            { g_time = clock(); DISPLAY(__VA_ARGS__); \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
            if (g_displayLevel>=4) fflush(stderr); } } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_time = 0;

 #undef MIN  /* in case it would be already defined */
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
@@ -120,22 +108,56 @@ static clock_t g_time = 0;
 #  define ZSTD_DEBUG 0
 #endif
 #define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...)                                              \
-{                                                                          \
-    DISPLAYLEVEL(1, "zstd: ");                                             \
-    DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__);  \
-    DISPLAYLEVEL(1, "error %i : ", error);                                 \
-    DISPLAYLEVEL(1, __VA_ARGS__);                                          \
-    DISPLAYLEVEL(1, " \n");                                                \
-    exit(error);                                                           \
+#define EXM_THROW(error, ...)                                             \
+{                                                                         \
+    DISPLAYLEVEL(1, "zstd: ");                                            \
+    DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+    DISPLAYLEVEL(1, "error %i : ", error);                                \
+    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
+    DISPLAYLEVEL(1, " \n");                                               \
+    exit(error);                                                          \
 }

-#define CHECK(f) {                                   \
-    size_t const err = f;                            \
-    if (ZSTD_isError(err)) {                         \
+#define CHECK_V(v, f)                                \
+    v = f;                                           \
+    if (ZSTD_isError(v)) {                           \
        DEBUGLOG(1, "%s \n", #f);                    \
-        EXM_THROW(11, "%s", ZSTD_getErrorName(err)); \
-}   }
+        EXM_THROW(11, "%s", ZSTD_getErrorName(v));   \
+    }
+#define CHECK(f) { size_t err; CHECK_V(err, f); }
+
+
+/*-************************************
+*  Signal (Ctrl-C trapping)
+**************************************/
+#include  <signal.h>
+
+static const char* g_artefact = NULL;
+static void INThandler(int sig)
+{
+    assert(sig==SIGINT); (void)sig;
+#if !defined(_MSC_VER)
+    signal(sig, SIG_IGN);  /* this invocation generates a buggy warning in Visual Studio */
+#endif
+    if (g_artefact) remove(g_artefact);
+    DISPLAY("\n");
+    exit(2);
+}
+static void addHandler(char const* dstFileName)
+{
+    if (UTIL_isRegularFile(dstFileName)) {
+        g_artefact = dstFileName;
+        signal(SIGINT, INThandler);
+    } else {
+        g_artefact = NULL;
+    }
+}
+/* Idempotent */
+static void clearHandler(void)
+{
+    if (g_artefact) signal(SIGINT, SIG_DFL);
+    g_artefact = NULL;
+}


 /* ************************************************************
@@ -198,10 +220,6 @@ static U32 g_blockSize = 0;
 void FIO_setBlockSize(unsigned blockSize) {
    if (blockSize && g_nbThreads==1)
        DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
-#ifdef ZSTD_MULTITHREAD
-    if (blockSize-1 < ZSTDMT_SECTION_SIZE_MIN-1)   /* intentional underflow */
-        DISPLAYLEVEL(2, "Note : minimum block size is %u KB \n", (ZSTDMT_SECTION_SIZE_MIN>>10));
-#endif
    g_blockSize = blockSize;
 }
 #define FIO_OVERLAP_LOG_NOTSET 9999
@@ -244,6 +262,10 @@ void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
 * @result : Unlink `fileName`, even if it's read-only */
 static int FIO_remove(const char* path)
 {
+    if (!UTIL_isRegularFile(path)) {
+        DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
+        return 0;
+    }
 #if defined(_WIN32) || defined(WIN32)
    /* windows doesn't allow remove read-only files,
     * so try to make it writable first */
@@ -253,86 +275,92 @@ static int FIO_remove(const char* path)
 }

 /** FIO_openSrcFile() :
- * condition : `dstFileName` must be non-NULL.
- * @result : FILE* to `dstFileName`, or NULL if it fails */
+ *  condition : `srcFileName` must be non-NULL.
+ * @result : FILE* to `srcFileName`, or NULL if it fails */
 static FILE* FIO_openSrcFile(const char* srcFileName)
 {
-    FILE* f;
-
+    assert(srcFileName != NULL);
    if (!strcmp (srcFileName, stdinmark)) {
        DISPLAYLEVEL(4,"Using stdin for input\n");
-        f = stdin;
        SET_BINARY_MODE(stdin);
-    } else {
-        if (!UTIL_isRegularFile(srcFileName)) {
-            DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
-                            srcFileName);
-            return NULL;
-        }
-        f = fopen(srcFileName, "rb");
-        if ( f==NULL )
-            DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
+        return stdin;
    }

-    return f;
+    if (!UTIL_isRegularFile(srcFileName)) {
+        DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
+                        srcFileName);
+        return NULL;
+    }
+
+    {   FILE* const f = fopen(srcFileName, "rb");
+        if (f == NULL)
+            DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
+        return f;
+    }
 }

 /** FIO_openDstFile() :
- * condition : `dstFileName` must be non-NULL.
+ *  condition : `dstFileName` must be non-NULL.
 * @result : FILE* to `dstFileName`, or NULL if it fails */
 static FILE* FIO_openDstFile(const char* dstFileName)
 {
-    FILE* f;
-
+    assert(dstFileName != NULL);
    if (!strcmp (dstFileName, stdoutmark)) {
        DISPLAYLEVEL(4,"Using stdout for output\n");
-        f = stdout;
        SET_BINARY_MODE(stdout);
        if (g_sparseFileSupport==1) {
            g_sparseFileSupport = 0;
            DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
        }
-    } else {
-        if (g_sparseFileSupport == 1) {
-            g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
-        }
-        if (strcmp (dstFileName, nulmark)) {
-            /* Check if destination file already exists */
-            f = fopen( dstFileName, "rb" );
-            if (f != 0) {  /* dst file exists, prompt for overwrite authorization */
-                fclose(f);
-                if (!g_overwrite) {
-                    if (g_displayLevel <= 1) {
-                        /* No interaction possible */
-                        DISPLAY("zstd: %s already exists; not overwritten  \n",
-                                dstFileName);
-                        return NULL;
-                    }
-                    DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ",
-                            dstFileName);
-                    {   int ch = getchar();
-                        if ((ch!='Y') && (ch!='y')) {
-                            DISPLAY("    not overwritten  \n");
-                            return NULL;
-                        }
-                        /* flush rest of input line */
-                        while ((ch!=EOF) && (ch!='\n')) ch = getchar();
-                }   }
-                /* need to unlink */
-                FIO_remove(dstFileName);
-        }   }
-        f = fopen( dstFileName, "wb" );
-        if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
+        return stdout;
    }

-    return f;
+    if (g_sparseFileSupport == 1) {
+        g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
+    }
+
+    if (UTIL_isRegularFile(dstFileName)) {
+        FILE* fCheck;
+        if (!strcmp(dstFileName, nulmark)) {
+            EXM_THROW(40, "%s is unexpectedly a regular file", dstFileName);
+        }
+        /* Check if destination file already exists */
+        fCheck = fopen( dstFileName, "rb" );
+        if (fCheck != NULL) {  /* dst file exists, authorization prompt */
+            fclose(fCheck);
+            if (!g_overwrite) {
+                if (g_displayLevel <= 1) {
+                    /* No interaction possible */
+                    DISPLAY("zstd: %s already exists; not overwritten  \n",
+                            dstFileName);
+                    return NULL;
+                }
+                DISPLAY("zstd: %s already exists; overwrite (y/N) ? ",
+                        dstFileName);
+                {   int ch = getchar();
+                    if ((ch!='Y') && (ch!='y')) {
+                        DISPLAY("    not overwritten  \n");
+                        return NULL;
+                    }
+                    /* flush rest of input line */
+                    while ((ch!=EOF) && (ch!='\n')) ch = getchar();
+            }   }
+            /* need to unlink */
+            FIO_remove(dstFileName);
+    }   }
+
+    {   FILE* const f = fopen( dstFileName, "wb" );
+        if (f == NULL)
+            DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
+        return f;
+    }
 }


 /*! FIO_createDictBuffer() :
 *  creates a buffer, pointed by `*bufferPtr`,
 *  loads `filename` content into it, up to DICTSIZE_MAX bytes.
- *  @return : loaded size
+ * @return : loaded size
 *  if fileName==NULL, returns 0 and a NULL pointer
 */
 static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
@@ -340,20 +368,24 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
    FILE* fileHandle;
    U64 fileSize;

+    assert(bufferPtr != NULL);
    *bufferPtr = NULL;
    if (fileName == NULL) return 0;

    DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
    fileHandle = fopen(fileName, "rb");
-    if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
+    if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
    fileSize = UTIL_getFileSize(fileName);
-    if (fileSize > DICTSIZE_MAX)
+    if (fileSize > DICTSIZE_MAX) {
        EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
                        fileName, DICTSIZE_MAX >> 20);   /* avoid extreme cases */
+    }
    *bufferPtr = malloc((size_t)fileSize);
    if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
-    { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
-      if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
+    {   size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
+        if (readSize!=fileSize)
+            EXM_THROW(35, "Error reading dictionary file %s", fileName);
+    }
    fclose(fileHandle);
    return (size_t)fileSize;
 }
@@ -370,37 +402,18 @@ typedef struct {
    size_t srcBufferSize;
    void*  dstBuffer;
    size_t dstBufferSize;
-#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
-    ZSTDMT_CCtx* cctx;
-#else
    ZSTD_CStream* cctx;
-#endif
 } cRess_t;

 static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
-                                    U64 srcSize, int srcIsRegularFile,
+                                    U64 srcSize,
                                    ZSTD_compressionParameters* comprParams) {
    cRess_t ress;
    memset(&ress, 0, sizeof(ress));

-#ifdef ZSTD_NEWAPI
    ress.cctx = ZSTD_createCCtx();
    if (ress.cctx == NULL)
        EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
-#elif defined(ZSTD_MULTITHREAD)
-    ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
-    if (ress.cctx == NULL)
-        EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
-    if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
-        /* use complete window for overlap */
-        ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
-    if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
-        ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
-#else
-    ress.cctx = ZSTD_createCStream();
-    if (ress.cctx == NULL)
-        EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
-#endif
    ress.srcBufferSize = ZSTD_CStreamInSize();
    ress.srcBuffer = malloc(ress.srcBufferSize);
    ress.dstBufferSize = ZSTD_CStreamOutSize();
@@ -408,74 +421,44 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
    if (!ress.srcBuffer || !ress.dstBuffer)
        EXM_THROW(31, "allocation error : not enough memory");

-    /* dictionary */
+    /* Advances parameters, including dictionary */
    {   void* dictBuffer;
        size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);   /* works with dictFileName==NULL */
        if (dictFileName && (dictBuffer==NULL))
            EXM_THROW(32, "allocation error : can't create dictBuffer");

-#ifdef ZSTD_NEWAPI
-        {   /* frame parameters */
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, srcIsRegularFile) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
-            CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );
-            /* compression level */
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
-            /* long distance matching */
-            CHECK( ZSTD_CCtx_setParameter(
-                          ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
-            if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
-                CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
-            }
-            if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
-                CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
-            }
-            /* compression parameters */
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
-            /* multi-threading */
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
-            /* dictionary */
-            CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );  /* always enable content size when available (note: supposed to be default) */
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
+        /* compression level */
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
+        /* long distance matching */
+        CHECK( ZSTD_CCtx_setParameter(
+                      ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
+        if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
        }
-#elif defined(ZSTD_MULTITHREAD)
-        {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
-            params.fParams.contentSizeFlag = srcIsRegularFile;
-            params.fParams.checksumFlag = g_checksumFlag;
-            params.fParams.noDictIDFlag = !g_dictIDFlag;
-            if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
-            if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
-            if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
-            if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
-            if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
-            if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
-            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
-            CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
-            ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
+        if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
        }
-#else
-        {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
-            params.fParams.contentSizeFlag = srcIsRegularFile;
-            params.fParams.checksumFlag = g_checksumFlag;
-            params.fParams.noDictIDFlag = !g_dictIDFlag;
-            if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
-            if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
-            if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
-            if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
-            if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
-            if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
-            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
-            CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
-        }
-#endif
+        /* compression parameters */
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
+        /* multi-threading */
+        DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
+        /* dictionary */
+        CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );  /* just for dictionary loading, for compression parameters adaptation */
+        CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
+        CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) );  /* reset */
+
        free(dictBuffer);
    }

@@ -486,11 +469,7 @@ static void FIO_freeCResources(cRess_t ress)
 {
    free(ress.srcBuffer);
    free(ress.dstBuffer);
-#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
-    ZSTDMT_freeCCtx(ress.cctx);
-#else
    ZSTD_freeCStream(ress.cctx);   /* never fails */
-#endif
 }


@@ -541,7 +520,7 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress,
                strm.avail_out = (uInt)ress->dstBufferSize;
            }
        }
-        if (!srcFileSize)
+        if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
                            (U32)(inFileSize>>20),
                            (double)outFileSize/inFileSize*100)
@@ -628,7 +607,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
                strm.next_out = (BYTE*)ress->dstBuffer;
                strm.avail_out = ress->dstBufferSize;
        }   }
-        if (!srcFileSize)
+        if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
                            (U32)(inFileSize>>20),
                            (double)outFileSize/inFileSize*100)
@@ -647,11 +626,16 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
 #endif

 #ifdef ZSTD_LZ4COMPRESS
+#if LZ4_VERSION_NUMBER <= 10600
+#define LZ4F_blockLinked blockLinked
+#define LZ4F_max64KB max64KB
+#endif
 static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
 static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
                            const char* srcFileName, U64 const srcFileSize,
                            int compressionLevel, U64* readsize)
 {
+    const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
    unsigned long long inFileSize = 0, outFileSize = 0;

    LZ4F_preferences_t prefs;
@@ -663,29 +647,26 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,

    memset(&prefs, 0, sizeof(prefs));

-#if LZ4_VERSION_NUMBER <= 10600
-#define LZ4F_blockIndependent blockIndependent
-#define LZ4F_max4MB max4MB
-#endif
+    assert(blockSize <= ress->srcBufferSize);

    prefs.autoFlush = 1;
    prefs.compressionLevel = compressionLevel;
-    prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* stick to defaults for lz4 cli */
-    prefs.frameInfo.blockSizeID = LZ4F_max4MB;
+    prefs.frameInfo.blockMode = LZ4F_blockLinked;
+    prefs.frameInfo.blockSizeID = LZ4F_max64KB;
    prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag;
 #if LZ4_VERSION_NUMBER >= 10600
-    prefs.frameInfo.contentSize = srcFileSize;
+    prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
 #endif
+    assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize);

    {
-        size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max4MB);
        size_t readSize;
        size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs);
        if (LZ4F_isError(headerSize))
            EXM_THROW(33, "File header generation failed : %s",
                            LZ4F_getErrorName(headerSize));
-        { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
-          if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
+        if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize)
+            EXM_THROW(34, "Write error : cannot write header");
        outFileSize += headerSize;

        /* Read first block */
@@ -702,7 +683,7 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
                EXM_THROW(35, "zstd: %s: lz4 compression failed : %s",
                            srcFileName, LZ4F_getErrorName(outSize));
            outFileSize += outSize;
-            if (!srcFileSize)
+            if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
                DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
                                (U32)(inFileSize>>20),
                                (double)outFileSize/inFileSize*100)
@@ -753,6 +734,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
    U64 readsize = 0;
    U64 compressedfilesize = 0;
    U64 const fileSize = UTIL_getFileSize(srcFileName);
+    ZSTD_EndDirective directive = ZSTD_e_continue;
+    DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);

    switch (g_compressionType) {
        case FIO_zstdCompression:
@@ -791,79 +774,52 @@ static int FIO_compressFilename_internal(cRess_t ress,
    }

    /* init */
-#ifdef ZSTD_NEWAPI
-    /* nothing, reset is implied */
-#elif defined(ZSTD_MULTITHREAD)
-    CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) );
-#else
-    CHECK( ZSTD_resetCStream(ress.cctx, fileSize) );
-#endif
+    if (fileSize != UTIL_FILESIZE_UNKNOWN)
+        ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);

    /* Main compression loop */
-    while (1) {
+    do {
+        size_t result;
        /* Fill input Buffer */
        size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
        ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
-        if (inSize==0) break;
        readsize += inSize;

-        while (inBuff.pos != inBuff.size) {
+        if (inSize == 0 || (fileSize != UTIL_FILESIZE_UNKNOWN && readsize == fileSize))
+            directive = ZSTD_e_end;
+
+        result = 1;
+        while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) {
            ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
-#ifdef ZSTD_NEWAPI
-            CHECK( ZSTD_compress_generic(ress.cctx,
-                        &outBuff, &inBuff, ZSTD_e_continue) );
-#elif defined(ZSTD_MULTITHREAD)
-            CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
-#else
-            CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
-#endif
+            CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));

            /* Write compressed stream */
+            DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_continue: generated %u bytes \n",
+                            (U32)outBuff.pos);
            if (outBuff.pos) {
                size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
                if (sizeCheck!=outBuff.pos)
                    EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
                compressedfilesize += outBuff.pos;
-        }   }
+            }
+        }
        if (g_nbThreads > 1) {
-            if (!fileSize)
+            if (fileSize == UTIL_FILESIZE_UNKNOWN)
                DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
            else
                DISPLAYUPDATE(2, "\rRead : %u / %u MB",
                                    (U32)(readsize>>20), (U32)(fileSize>>20));
        } else {
-            if (!fileSize)
+            if (fileSize == UTIL_FILESIZE_UNKNOWN)
                DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
                                (U32)(readsize>>20),
                                (double)compressedfilesize/readsize*100)
            else
-            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
+                DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
                                (U32)(readsize>>20), (U32)(fileSize>>20),
                                (double)compressedfilesize/readsize*100);
        }
-    }
-
-    /* End of Frame */
-    {   size_t result = 1;
-        while (result!=0) {   /* note : is there any possibility of endless loop ? */
-            ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
-#ifdef ZSTD_NEWAPI
-            ZSTD_inBuffer inBuff = { NULL, 0, 0};
-            result = ZSTD_compress_generic(ress.cctx,
-                        &outBuff, &inBuff, ZSTD_e_end);
-#elif defined(ZSTD_MULTITHREAD)
-            result = ZSTDMT_endStream(ress.cctx, &outBuff);
-#else
-            result = ZSTD_endStream(ress.cctx, &outBuff);
-#endif
-            if (ZSTD_isError(result))
-                EXM_THROW(26, "Compression error during frame end : %s",
-                            ZSTD_getErrorName(result));
-            { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
-              if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
-            compressedfilesize += outBuff.pos;
-        }
-    }
+    } while (directive != ZSTD_e_end);

 finish:
    /* Status */
@@ -901,6 +857,10 @@ static int FIO_compressFilename_srcFile(cRess_t ress,

    fclose(ress.srcFile);
    if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) {
+        /* We must clear the handler, since after this point calling it would
+         * delete both the source and destination files.
+         */
+        clearHandler();
        if (remove(srcFileName))
            EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
    }
@@ -923,10 +883,16 @@ static int FIO_compressFilename_dstFile(cRess_t ress,

    ress.dstFile = FIO_openDstFile(dstFileName);
    if (ress.dstFile==NULL) return 1;  /* could not open dstFileName */
+    /* Must ony be added after FIO_openDstFile() succeeds.
+     * Otherwise we may delete the destination file if at already exists, and
+     * the user presses Ctrl-C when asked if they wish to overwrite.
+     */
+    addHandler(dstFileName);

    if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf))
        stat_result = 1;
    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel);
+    clearHandler();

    if (fclose(ress.dstFile)) { /* error closing dstFile */
        DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
@@ -938,6 +904,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
    }
    else if (strcmp (dstFileName, stdoutmark) && stat_result)
        UTIL_setFileStat(dstFileName, &statbuf);
+
    return result;
 }

@@ -946,10 +913,10 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
                         const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
 {
    clock_t const start = clock();
-    U64 const srcSize = UTIL_getFileSize(srcFileName);
-    int const isRegularFile = UTIL_isRegularFile(srcFileName);
+    U64 const fileSize = UTIL_getFileSize(srcFileName);
+    U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize;

-    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
+    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
    int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);

    double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
@@ -961,7 +928,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,


 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
-                                  const char* suffix,
+                                  const char* outFileName, const char* suffix,
                                  const char* dictFileName, int compressionLevel,
                                  ZSTD_compressionParameters* comprParams)
 {
@@ -969,23 +936,23 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
    size_t dfnSize = FNSPACE;
    char*  dstFileName = (char*)malloc(FNSPACE);
    size_t const suffixSize = suffix ? strlen(suffix) : 0;
-    U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
-    int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);
-    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
+    U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]);
+    U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize;
+    U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ;
+    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);

    /* init */
    if (dstFileName==NULL)
        EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
-    if (suffix == NULL)
+    if (outFileName == NULL && suffix == NULL)
        EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown");  /* should never happen */

    /* loop on each file */
-    if (!strcmp(suffix, stdoutmark)) {
+    if (outFileName != NULL) {
        unsigned u;
-        ress.dstFile = stdout;
-        SET_BINARY_MODE(stdout);
+        ress.dstFile = FIO_openDstFile(outFileName);
        for (u=0; u<nbFiles; u++)
-            missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u], compressionLevel);
+            missed_files += FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel);
        if (fclose(ress.dstFile))
            EXM_THROW(29, "Write error : cannot properly close stdout");
    } else {
@@ -996,9 +963,9 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
                free(dstFileName);
                dfnSize = ifnSize + 20;
                dstFileName = (char*)malloc(dfnSize);
-                if (!dstFileName)
+                if (!dstFileName) {
                    EXM_THROW(30, "zstd: %s", strerror(errno));
-            }
+            }   }
            strcpy(dstFileName, inFileNamesTable[u]);
            strcat(dstFileName, suffix);
            missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
@@ -1036,7 +1003,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
    /* Allocation */
    ress.dctx = ZSTD_createDStream();
    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
-    ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit);
+    CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) );
    ress.srcBufferSize = ZSTD_DStreamInSize();
    ress.srcBuffer = malloc(ress.srcBufferSize);
    ress.dstBufferSize = ZSTD_DStreamOutSize();
@@ -1167,6 +1134,35 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
    return 0;
 }

+static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName)
+{
+    ZSTD_frameHeader header;
+    /* No special help for these errors */
+    if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge)
+        return;
+    /* Try to decode the frame header */
+    ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
+    if (ret == 0) {
+        U32 const windowSize = (U32)header.windowSize;
+        U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
+        U32 const windowMB = (windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0);
+        assert(header.windowSize <= (U64)((U32)-1));
+        assert(g_memLimit > 0);
+        DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
+                        srcFileName, header.windowSize, g_memLimit);
+        if (windowLog <= ZSTD_WINDOWLOG_MAX) {
+            DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
+                            srcFileName, windowLog, windowMB);
+            return;
+        }
+    } else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) {
+        DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n",
+                        srcFileName, ZSTD_getErrorName(ret));
+        return;
+    }
+    DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n",
+                    srcFileName, ZSTD_WINDOWLOG_MAX);
+}

 /** FIO_decompressFrame() :
 *  @return : size of decoded zstd frame, or an error code
@@ -1180,14 +1176,18 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
    U64 frameSize = 0;
    U32 storedSkips = 0;

-    ZSTD_resetDStream(ress->dctx);
-    if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20;   /* display last 20 characters */
+    size_t const srcFileLength = strlen(srcFileName);
+    if (srcFileLength>20) srcFileName += srcFileLength-20;  /* display last 20 characters only */

-    /* Header loading (optional, saves one loop) */
-    {   size_t const toRead = 9;
-        if (ress->srcBufferLoaded < toRead)
-            ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
-    }
+    ZSTD_resetDStream(ress->dctx);
+
+    /* Header loading : ensures ZSTD_getFrameHeader() will succeed */
+    {   size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX;
+        if (ress->srcBufferLoaded < toDecode) {
+            size_t const toRead = toDecode - ress->srcBufferLoaded;
+            void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded;
+            ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput);
+    }   }

    /* Main decompression Loop */
    while (1) {
@@ -1197,6 +1197,7 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
        if (ZSTD_isError(readSizeHint)) {
            DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
                            srcFileName, ZSTD_getErrorName(readSizeHint));
+            FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
            return FIO_ERROR_FRAME_DECODING;
        }

@@ -1219,14 +1220,17 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
        }

        /* Fill input buffer */
-        {   size_t const toRead = MIN(readSizeHint, ress->srcBufferSize);  /* support large skippable frames */
-            if (ress->srcBufferLoaded < toRead)
-                ress->srcBufferLoaded += fread((char*)ress->srcBuffer + ress->srcBufferLoaded,
-                                               1, toRead - ress->srcBufferLoaded, finput);
-            if (ress->srcBufferLoaded < toRead) {
-                DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
-                                srcFileName);
-                return FIO_ERROR_FRAME_DECODING;
+        {   size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize);  /* support large skippable frames */
+            if (ress->srcBufferLoaded < toDecode) {
+                size_t const toRead = toDecode - ress->srcBufferLoaded;   /* > 0 */
+                void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded;
+                size_t const readSize = fread(startPosition, 1, toRead, finput);
+                if (readSize==0) {
+                    DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
+                                    srcFileName);
+                    return FIO_ERROR_FRAME_DECODING;
+                }
+                ress->srcBufferLoaded += readSize;
    }   }   }

    FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
@@ -1563,6 +1567,10 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
    if ( g_removeSrcFile /* --rm */
      && (result==0)     /* decompression successful */
      && strcmp(srcFileName, stdinmark) ) /* not stdin */ {
+        /* We must clear the handler, since after this point calling it would
+         * delete both the source and destination files.
+         */
+        clearHandler();
        if (remove(srcFileName)) {
            /* failed to remove src file */
            DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
@@ -1586,11 +1594,17 @@ static int FIO_decompressDstFile(dRess_t ress,

    ress.dstFile = FIO_openDstFile(dstFileName);
    if (ress.dstFile==0) return 1;
+    /* Must ony be added after FIO_openDstFile() succeeds.
+     * Otherwise we may delete the destination file if at already exists, and
+     * the user presses Ctrl-C when asked if they wish to overwrite.
+     */
+    addHandler(dstFileName);

    if ( strcmp(srcFileName, stdinmark)
      && UTIL_getFileStat(srcFileName, &statbuf) )
        stat_result = 1;
    result = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
+    clearHandler();

    if (fclose(ress.dstFile)) {
        DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
@@ -1607,6 +1621,9 @@ static int FIO_decompressDstFile(dRess_t ress,
          && stat_result )                   /* file permissions correctly extracted from src */
            UTIL_setFileStat(dstFileName, &statbuf);  /* transfer file permissions from src into dst */
    }
+
+    signal(SIGINT, SIG_DFL);
+
    return result;
 }

@@ -1625,24 +1642,21 @@ int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,

 #define MAXSUFFIXSIZE 8
 int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
-                                    const char* suffix,
+                                    const char* outFileName,
                                    const char* dictFileName)
 {
    int skippedFiles = 0;
    int missingFiles = 0;
    dRess_t ress = FIO_createDResources(dictFileName);

-    if (suffix==NULL)
-        EXM_THROW(70, "zstd: decompression: unknown dst");   /* should never happen */
-
-    if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) {  /* special cases : -c or -t */
+    if (outFileName) {
        unsigned u;
-        ress.dstFile = FIO_openDstFile(suffix);
-        if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
+        ress.dstFile = FIO_openDstFile(outFileName);
+        if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
        for (u=0; u<nbFiles; u++)
-            missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]);
+            missingFiles += FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]);
        if (fclose(ress.dstFile))
-            EXM_THROW(72, "Write error : cannot properly close stdout");
+            EXM_THROW(72, "Write error : cannot properly close output file");
    } else {
        size_t suffixSize;
        size_t dfnSize = FNSPACE;
@@ -1698,12 +1712,14 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
 ***************************************************************************/

 typedef struct {
+    U64 decompressedSize;
+    U64 compressedSize;
+    U64 windowSize;
    int numActualFrames;
    int numSkippableFrames;
-    unsigned long long decompressedSize;
    int decompUnavailable;
-    unsigned long long compressedSize;
    int usesCheck;
+    U32 nbFiles;
 } fileInfo_t;

 /** getFileInfo() :
@@ -1713,21 +1729,24 @@ typedef struct {
 *           2 for file not compressed with zstd
 *           3 for cases in which file could not be opened.
 */
-static int getFileInfo(fileInfo_t* info, const char* inFileName){
+static int getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName){
    int detectError = 0;
    FILE* const srcFile = FIO_openSrcFile(inFileName);
    if (srcFile == NULL) {
        DISPLAY("Error: could not open source file %s\n", inFileName);
        return 3;
    }
-    info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
+    info->compressedSize = UTIL_getFileSize(inFileName);

    /* begin analyzing frame */
    for ( ; ; ) {
        BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
        size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
        if (numBytesRead < ZSTD_frameHeaderSize_min) {
-            if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
+            if ( feof(srcFile)
+              && (numBytesRead == 0)
+              && (info->compressedSize > 0)
+              && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
                break;
            }
            else if (feof(srcFile)) {
@@ -1744,12 +1763,19 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
        {   U32 const magicNumber = MEM_readLE32(headerBuffer);
            /* Zstandard frame */
            if (magicNumber == ZSTD_MAGICNUMBER) {
+                ZSTD_frameHeader header;
                U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
                if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
                    info->decompUnavailable = 1;
                } else {
                    info->decompressedSize += frameContentSize;
                }
+                if (ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0) {
+                    DISPLAY("Error: could not decode frame header\n");
+                    detectError = 1;
+                    break;
+                }
+                info->windowSize = header.windowSize;
                /* move to the end of the frame header */
                {   size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
                    if (ZSTD_isError(headerSize)) {
@@ -1829,37 +1855,58 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
        }
    }  /* end analyzing frame */
    fclose(srcFile);
+    info->nbFiles = 1;
    return detectError;
 }

-static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
+static int getFileInfo(fileInfo_t* info, const char* srcFileName)
+{
+    int const isAFile = UTIL_isRegularFile(srcFileName);
+    if (!isAFile) {
+        DISPLAY("Error : %s is not a file", srcFileName);
+        return 3;
+    }
+    return getFileInfo_fileConfirmed(info, srcFileName);
+}
+
+
+static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){
    unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
    const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
+    double const windowSizeUnit = (double)info->windowSize / unit;
    double const compressedSizeUnit = (double)info->compressedSize / unit;
    double const decompressedSizeUnit = (double)info->decompressedSize / unit;
    double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
    const char* const checkString = (info->usesCheck ? "XXH64" : "None");
    if (displayLevel <= 2) {
        if (!info->decompUnavailable) {
-            DISPLAYOUT("Skippable  Non-Skippable  Compressed  Uncompressed  Ratio  Check  Filename\n");
-            DISPLAYOUT("%9d  %13d  %7.2f %2s  %9.2f %2s  %5.3f  %5s  %s\n",
-                    info->numSkippableFrames, info->numActualFrames,
+            DISPLAYOUT("%6d  %5d  %7.2f %2s  %9.2f %2s  %5.3f  %5s  %s\n",
+                    info->numSkippableFrames + info->numActualFrames,
+                    info->numSkippableFrames,
                    compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
                    ratio, checkString, inFileName);
        } else {
-            DISPLAYOUT("Skippable  Non-Skippable  Compressed  Check  Filename\n");
-            DISPLAYOUT("%9d  %13d  %7.2f MB  %5s  %s\n",
-                    info->numSkippableFrames, info->numActualFrames,
-                    compressedSizeUnit, checkString, inFileName);
+            DISPLAYOUT("%6d  %5d  %7.2f %2s                       %5s  %s\n",
+                    info->numSkippableFrames + info->numActualFrames,
+                    info->numSkippableFrames,
+                    compressedSizeUnit, unitStr,
+                    checkString, inFileName);
        }
    } else {
+        DISPLAYOUT("%s \n", inFileName);
        DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
-        DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
+        if (info->numSkippableFrames)
+            DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
+        DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n",
+                   windowSizeUnit, unitStr,
+                   (unsigned long long)info->windowSize);
        DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
-                    compressedSizeUnit, unitStr, info->compressedSize);
+                    compressedSizeUnit, unitStr,
+                    (unsigned long long)info->compressedSize);
        if (!info->decompUnavailable) {
            DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
-                    decompressedSizeUnit, unitStr, info->decompressedSize);
+                    decompressedSizeUnit, unitStr,
+                    (unsigned long long)info->decompressedSize);
            DISPLAYOUT("Ratio: %.4f\n", ratio);
        }
        DISPLAYOUT("Check: %s\n", checkString);
@@ -1867,33 +1914,39 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
    }
 }

+static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
+{
+    fileInfo_t total;
+    total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames;
+    total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames;
+    total.compressedSize = fi1.compressedSize + fi2.compressedSize;
+    total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize;
+    total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable;
+    total.usesCheck = fi1.usesCheck & fi2.usesCheck;
+    total.nbFiles = fi1.nbFiles + fi2.nbFiles;
+    return total;
+}

-static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
-    /* initialize info to avoid warnings */
+static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
    fileInfo_t info;
    memset(&info, 0, sizeof(info));
-    DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
-    {
-        int const error = getFileInfo(&info, inFileName);
+    {   int const error = getFileInfo(&info, inFileName);
        if (error == 1) {
            /* display error, but provide output */
-            DISPLAY("An error occurred with getting file info\n");
+            DISPLAY("An error occurred while getting file info \n");
        }
        else if (error == 2) {
-            DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
-            if (displayLevel > 2) {
-                DISPLAYOUT("\n");
-            }
+            DISPLAYOUT("File %s not compressed by zstd \n", inFileName);
+            if (displayLevel > 2) DISPLAYOUT("\n");
            return 1;
        }
        else if (error == 3) {
-            /* error occurred with opening the file */
-            if (displayLevel > 2) {
-                DISPLAYOUT("\n");
-            }
+            /* error occurred while opening the file */
+            if (displayLevel > 2) DISPLAYOUT("\n");
            return 1;
        }
        displayInfo(inFileName, &info, displayLevel);
+        *total = FIO_addFInfo(*total, info);
        return error;
    }
 }
@@ -1903,16 +1956,38 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
        DISPLAYOUT("No files given\n");
        return 0;
    }
-    DISPLAYOUT("===========================================\n");
-    DISPLAYOUT("Printing information about compressed files\n");
-    DISPLAYOUT("===========================================\n");
-    DISPLAYOUT("Number of files listed: %u\n", numFiles);
-    {
-        int error = 0;
+    if (displayLevel <= 2) {
+        DISPLAYOUT("Frames  Skips  Compressed  Uncompressed  Ratio  Check  Filename\n");
+    }
+    {   int error = 0;
        unsigned u;
+        fileInfo_t total;
+        memset(&total, 0, sizeof(total));
+        total.usesCheck = 1;
        for (u=0; u<numFiles;u++) {
-            error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
+            error |= FIO_listFile(&total, filenameTable[u], displayLevel);
        }
+        if (numFiles > 1 && displayLevel <= 2) {   /* display total */
+            unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
+            const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
+            double const compressedSizeUnit = (double)total.compressedSize / unit;
+            double const decompressedSizeUnit = (double)total.decompressedSize / unit;
+            double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize;
+            const char* const checkString = (total.usesCheck ? "XXH64" : "");
+            DISPLAYOUT("----------------------------------------------------------------- \n");
+            if (total.decompUnavailable) {
+                DISPLAYOUT("%6d  %5d  %7.2f %2s                       %5s  %u files\n",
+                        total.numSkippableFrames + total.numActualFrames,
+                        total.numSkippableFrames,
+                        compressedSizeUnit, unitStr,
+                        checkString, total.nbFiles);
+            } else {
+                DISPLAYOUT("%6d  %5d  %7.2f %2s  %9.2f %2s  %5.3f  %5s  %u files\n",
+                        total.numSkippableFrames + total.numActualFrames,
+                        total.numSkippableFrames,
+                        compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
+                        ratio, checkString, total.nbFiles);
+        }   }
        return error;
    }
 }
@@ -84,14 +84,14 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
 /** FIO_compressMultipleFilenames() :
    @return : nb of missing files */
 int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
-                                  const char* suffix,
+                                  const char* outFileName, const char* suffix,
                                  const char* dictFileName, int compressionLevel,
                                  ZSTD_compressionParameters* comprParams);

 /** FIO_decompressMultipleFilenames() :
    @return : nb of missing or skipped files */
 int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
-                                    const char* suffix,
+                                    const char* outFileName,
                                    const char* dictFileName);


@@ -21,10 +21,10 @@ extern "C" {
 *  Compiler Options
 ****************************************/
 #if defined(_MSC_VER)
-#  define _CRT_SECURE_NO_WARNINGS   /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
-#  define _CRT_SECURE_NO_DEPRECATE  /* VS2005 - must be declared before <io.h> and <windows.h> */
-#  if (_MSC_VER <= 1800)            /* (1800 = Visual Studio 2013) */
-#    define snprintf sprintf_s      /* snprintf unsupported by Visual <= 2013 */
+#  define _CRT_SECURE_NO_WARNINGS    /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
+#  if (_MSC_VER <= 1800)             /* 1800 == Visual Studio 2013 */
+#    define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
+#    define snprintf sprintf_s       /* snprintf unsupported by Visual <= 2013 */
 #  endif
 #endif

@@ -117,7 +117,7 @@ static __inline int IS_CONSOLE(FILE* stdStream)


 /******************************
-*  OS-specific Includes
+*  OS-specific IO behaviors
 ******************************/
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
 #  include <fcntl.h>   /* _O_BINARY */
@@ -125,7 +125,7 @@ static __inline int IS_CONSOLE(FILE* stdStream)
 #  if !defined(__DJGPP__)
 #    include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
 #    include <winioctl.h> /* FSCTL_SET_SPARSE */
-#    define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
+#    define SET_BINARY_MODE(file) { int const unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
 #    define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
 #  else
 #    define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
@@ -34,7 +34,7 @@ extern "C" {
 #  include <unistd.h>     /* chown, stat */
 #  include <utime.h>      /* utime */
 #endif
-#include <time.h>         /* time */
+#include <time.h>         /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
 #include <errno.h>
 #include "mem.h"          /* U32, U64 */

@@ -64,7 +64,6 @@ extern "C" {
 #elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
 #  include <unistd.h>
 #  include <sys/resource.h> /* setpriority */
-#  include <time.h>         /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
 #  if defined(PRIO_PROCESS)
 #    define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
 #  else
@@ -118,6 +117,7 @@ static int g_utilDisplayLevel;
 *  Time functions
 ******************************************/
 #if defined(_WIN32)   /* Windows */
+    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
    typedef LARGE_INTEGER UTIL_time_t;
    UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
@@ -144,6 +144,7 @@ static int g_utilDisplayLevel;
    }
 #elif defined(__APPLE__) && defined(__MACH__)
    #include <mach/mach_time.h>
+    #define UTIL_TIME_INITIALIZER 0
    typedef U64 UTIL_time_t;
    UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
@@ -166,8 +167,8 @@ static int g_utilDisplayLevel;
        }
        return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
    }
-#elif (PLATFORM_POSIX_VERSION >= 200112L)
-    #include <time.h>
+#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2))
+    #define UTIL_TIME_INITIALIZER { 0, 0 }
    typedef struct timespec UTIL_freq_t;
    typedef struct timespec UTIL_time_t;
    UTIL_STATIC UTIL_time_t UTIL_getTime(void)
@@ -207,11 +208,13 @@ static int g_utilDisplayLevel;
    }
 #else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
    typedef clock_t UTIL_time_t;
+    #define UTIL_TIME_INITIALIZER 0
    UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
    UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
 #endif

+#define SEC_TO_MICRO 1000000

 /* returns time span in microseconds */
 UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart )
@@ -313,33 +316,40 @@ UTIL_STATIC U32 UTIL_isLink(const char* infilename)
 }


+#define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
 UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
 {
-    int r;
+    if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
+    {   int r;
 #if defined(_MSC_VER)
-    struct __stat64 statbuf;
-    r = _stat64(infilename, &statbuf);
-    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
+        struct __stat64 statbuf;
+        r = _stat64(infilename, &statbuf);
+        if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
 #elif defined(__MINGW32__) && defined (__MSVCRT__)
-    struct _stati64 statbuf;
-    r = _stati64(infilename, &statbuf);
-    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
+        struct _stati64 statbuf;
+        r = _stati64(infilename, &statbuf);
+        if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
 #else
-    struct stat statbuf;
-    r = stat(infilename, &statbuf);
-    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+        struct stat statbuf;
+        r = stat(infilename, &statbuf);
+        if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
 #endif
-    return (U64)statbuf.st_size;
+        return (U64)statbuf.st_size;
+    }
 }


-UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
+UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
 {
    U64 total = 0;
+    int error = 0;
    unsigned n;
-    for (n=0; n<nbFiles; n++)
-        total += UTIL_getFileSize(fileNamesTable[n]);
-    return total;
+    for (n=0; n<nbFiles; n++) {
+        U64 const size = UTIL_getFileSize(fileNamesTable[n]);
+        error |= (size == UTIL_FILESIZE_UNKNOWN);
+        total += size;
+    }
+    return error ? UTIL_FILESIZE_UNKNOWN : total;
 }


@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "September 2017" "zstd 1.3.1" "User Commands"
+.TH "ZSTD" "1" "December 2017" "zstd 1.3.3" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -104,8 +104,11 @@ Display information related to a zstd compressed file, such as size, ratio, and
 unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
 .
 .TP
-\fB\-\-long\fR
-enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
+\fB\-\-long[=#]\fR
+enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
+.
+.IP
+Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
 .
 .TP
 \fB\-T#\fR, \fB\-\-threads=#\fR
@@ -148,6 +151,10 @@ keep source file(s) after successful compression or decompression\. This is the
 operate recursively on dictionaries
 .
 .TP
+\fB\-\-format=FORMAT\fR
+compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\.
+.
+.TP
 \fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR
 display help/long help and exit
 .
@@ -190,6 +197,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\.
 Limit dictionary to specified size (default: 112640)\.
 .
 .TP
+\fB\-B#\fR
+Split input files in blocks of size # (default: no split)
+.
+.TP
 \fB\-\-dictID=#\fR
 A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
 .
@@ -250,6 +261,12 @@ cut file(s) into independent blocks of size # (default: no block)
 \fB\-\-priority=rt\fR
 set process priority to real\-time
 .
+.P
+\fBOutput Format:\fR CompressionLevel#Filename : IntputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+.
+.P
+\fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
+.
 .SH "ADVANCED COMPRESSION OPTIONS"
 .
 .SS "\-\-zstd[=options]:"
@@ -267,7 +284,10 @@ There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fas
 Specify the maximum number of bits for a match distance\.
 .
 .IP
-The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
+The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
+.
+.IP
+Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
 .
 .TP
 \fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
@@ -340,7 +360,7 @@ Bigger hash tables usually improve compression ratio at the expense of more memo
 The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
 .
 .TP
-\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR
+\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
 Specify the minimum searched length of a match for long distance matching\.
 .
 .IP
@@ -105,12 +105,16 @@ the last one takes effect.
 * `--ultra`:
    unlocks high compression levels 20+ (maximum 22), using a lot more memory.
    Note that decompression will also require more memory when using these levels.
-* `--long`:
-    enables long distance matching.
+* `--long[=#]`:
+    enables long distance matching with `#` `windowLog`, if not `#` is not
+    present it defaults to `27`.
    This increases the window size (`windowLog`) and memory usage for both the
-    compressor and decompressor. This setting is designed to improve the
-    compression ratio for files with long matches at a large distance
-    (up to the maximum window size, 128 MiB).
+    compressor and decompressor.
+    This setting is designed to improve the compression ratio for files with
+    long matches at a large distance.
+
+    Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
+    `--memory=windowSize` needs to be passed to the decompressor.
 * `-T#`, `--threads=#`:
    Compress using `#` threads (default: 1).
    If `#` is 0, attempt to detect and use the number of physical CPU cores.
@@ -143,6 +147,10 @@ the last one takes effect.
    This is the default behavior.
 * `-r`:
    operate recursively on dictionaries
+* `--format=FORMAT`:
+    compress and decompress in other formats. If compiled with
+    support, zstd can compress to or decompress from other compression algorithm
+    formats. Possibly available options are `gzip`, `xz`, `lzma`, and `lz4`.
 * `-h`/`-H`, `--help`:
    display help/long help and exit
 * `-V`, `--version`:
@@ -250,6 +258,9 @@ BENCHMARK
 * `--priority=rt`:
    set process priority to real-time

+**Output Format:** CompressionLevel#Filename : IntputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+
+**Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.

 ADVANCED COMPRESSION OPTIONS
 ----------------------------
@@ -275,7 +286,11 @@ The list of available _options_:
    The higher number of increases the chance to find a match which usually
    improves compression ratio.
    It also increases memory requirements for the compressor and decompressor.
-    The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
+    The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
+    platforms and 31 (2 GiB) on 64-bit platforms.
+
+    Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
+    `--memory=windowSize` needs to be passed to the decompressor.

 - `hashLog`=_hlog_, `hlog`=_hlog_:
    Specify the maximum number of bits for a hash table.
@@ -61,6 +61,8 @@
 #define ZSTD_UNLZMA "unlzma"
 #define ZSTD_XZ "xz"
 #define ZSTD_UNXZ "unxz"
+#define ZSTD_LZ4 "lz4"
+#define ZSTD_UNLZ4 "unlz4"

 #define KB *(1 <<10)
 #define MB *(1 <<20)
@@ -72,6 +74,7 @@ static const char*    g_defaultDictName = "dictionary";
 static const unsigned g_defaultMaxDictSize = 110 KB;
 static const int      g_defaultDictCLevel = 3;
 static const unsigned g_defaultSelectivityLevel = 9;
+static const unsigned g_defaultMaxWindowLog = 27;
 #define OVERLAP_LOG_DEFAULT 9999
 #define LDM_PARAM_DEFAULT 9999  /* Default for parameters where 0 is valid */
 static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
@@ -129,7 +132,7 @@ static int usage_advanced(const char* programName)
    DISPLAY( " -l     : print information about zstd compressed files \n");
 #ifndef ZSTD_NOCOMPRESS
    DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
-    DISPLAY( "--long  : enable long distance matching (requires more memory)\n");
+    DISPLAY( "--long[=#]  : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog);
 #ifdef ZSTD_MULTITHREAD
    DISPLAY( " -T#    : use # threads for compression (default:1) \n");
    DISPLAY( " -B#    : select size of each job (default:0==automatic) \n");
@@ -374,6 +377,7 @@ int main(int argCount, const char* argv[])
        lastCommand = 0,
        nbThreads = 1,
        setRealTimePrio = 0,
+        separateFiles = 0,
        ldmFlag = 0;
    unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
    size_t blockSize = 0;
@@ -403,6 +407,7 @@ int main(int argCount, const char* argv[])
    int cover = 1;
 #endif

+
    /* init */
    (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
    (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
@@ -425,6 +430,8 @@ int main(int argCount, const char* argv[])
    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }    /* behave like unlzma */
    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }    /* behave like xz */
    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }    /* behave like unxz */
+    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); }    /* behave like xz */
+    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); }    /* behave like unxz */
    memset(&compressionParams, 0, sizeof(compressionParams));

    /* command switches */
@@ -459,7 +466,6 @@ int main(int argCount, const char* argv[])
                    if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
                    if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
                    if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
-                    if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
                    if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
                    if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
                    if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
@@ -514,6 +520,22 @@ int main(int argCount, const char* argv[])
                    if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
                    if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
                    if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
+                    if (longCommandWArg(&argument, "--long")) {
+                        unsigned ldmWindowLog = 0;
+                        ldmFlag = 1;
+                        /* Parse optional window log */
+                        if (*argument == '=') {
+                            ++argument;
+                            ldmWindowLog = readU32FromChar(&argument);
+                        } else if (*argument != 0) {
+                            /* Invalid character following --long */
+                            CLEAN_RETURN(badusage(programName));
+                        }
+                        /* Only set windowLog if not already set by --zstd */
+                        if (compressionParams.windowLog == 0)
+                            compressionParams.windowLog = ldmWindowLog;
+                        continue;
+                    }
                    /* fall-through, will trigger bad_usage() later on */
                }

@@ -612,6 +634,12 @@ int main(int argCount, const char* argv[])
                        blockSize = readU32FromChar(&argument);
                        break;

+                        /* benchmark files separately (hidden option) */
+                    case 'S':
+                        argument++;
+                        separateFiles = 1;
+                        break;
+
 #endif   /* ZSTD_NOBENCH */

                        /* nb of threads (hidden option) */
@@ -730,8 +758,10 @@ int main(int argCount, const char* argv[])
    if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
        BMK_setNotificationLevel(g_displayLevel);
+        BMK_setSeparateFiles(separateFiles);
        BMK_setBlockSize(blockSize);
        BMK_setNbThreads(nbThreads);
+        BMK_setRealTime(setRealTimePrio);
        BMK_setNbSeconds(bench_nbSeconds);
        BMK_setLdmFlag(ldmFlag);
        BMK_setLdmMinMatch(g_ldmMinMatch);
@@ -742,9 +772,10 @@ int main(int argCount, const char* argv[])
        if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
            BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
        }
-        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
+        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
+#else
+        (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
 #endif
-        (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio;
        goto _end;
    }

@@ -784,12 +815,6 @@ int main(int argCount, const char* argv[])
    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !strcmp(filenameTable[0], stdinmark) && !forceStdout && operation!=zom_decompress)
        CLEAN_RETURN(badusage(programName));

-    /* user-selected output filename, only possible with a single file */
-    if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) {
-        DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
-        CLEAN_RETURN(filenameIdx);
-    }
-
 #ifndef ZSTD_NOCOMPRESS
    /* check compression level limits */
    {   int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
@@ -823,18 +848,25 @@ int main(int argCount, const char* argv[])
        if ((filenameIdx==1) && outFileName)
          operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
        else
-          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : suffix, dictFileName, cLevel, &compressionParams);
+          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams);
 #else
        (void)suffix;
        DISPLAY("Compression not supported\n");
 #endif
    } else {  /* decompression or test */
 #ifndef ZSTD_NODECOMPRESS
+        if (memLimit == 0) {
+            if (compressionParams.windowLog == 0)
+                memLimit = (U32)1 << g_defaultMaxWindowLog;
+            else {
+                memLimit = (U32)1 << (compressionParams.windowLog & 31);
+            }
+        }
        FIO_setMemLimit(memLimit);
        if (filenameIdx==1 && outFileName)
            operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
        else
-            operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName);
+            operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
 #else
        DISPLAY("Decompression not supported\n");
 #endif
@@ -23,18 +23,18 @@ PRGDIR  = ../programs
 PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest

-DEBUGLEVEL= 1
-DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL)
-CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-            -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
-CFLAGS   ?= -O3
-CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
-            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wformat-security                   \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
-            -Wredundant-decls
-CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+DEBUGLEVEL ?= 1
+DEBUGFLAGS  = -g -DZSTD_DEBUG=$(DEBUGLEVEL)
+CPPFLAGS   += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+              -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CFLAGS     ?= -O3
+CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+              -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+              -Wstrict-prototypes -Wundef -Wformat-security                   \
+              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+              -Wredundant-decls
+CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)


 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
@@ -58,8 +58,8 @@ endif
 MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)

 VOID = /dev/null
-ZSTREAM_TESTTIME ?= -T2mn
-FUZZERTEST ?= -T5mn
+ZSTREAM_TESTTIME ?= -T90s
+FUZZERTEST ?= -T200s
 ZSTDRTTEST = --test-large-data
 DECODECORPUS_TESTTIME ?= -T30

@@ -76,16 +76,16 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
 dll: fuzzer-dll zstreamtest-dll

 zstd:
-	$(MAKE) -C $(PRGDIR) $@
+	$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"

 zstd32:
-	$(MAKE) -C $(PRGDIR) $@
+	$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"

 zstd-nolegacy:
-	$(MAKE) -C $(PRGDIR) $@
+	$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"

 gzstd:
-	$(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1
+	$(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 MOREFLAGS="$(DEBUGFLAGS)"

 fullbench32: CPPFLAGS += -m32
 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
@@ -130,15 +130,12 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)

-ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-zstreamtest : CPPFLAGS += $(MULTITHREAD_CPP)
-zstreamtest : LDFLAGS += $(MULTITHREAD_LD)
-zstreamtest : $(ZSTREAMFILES)
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
+ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c seqgen.c zstreamtest.c
 zstreamtest32 : CFLAGS += -m32
-zstreamtest32 : $(ZSTREAMFILES)
-	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP)
+zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD)
+zstreamtest zstreamtest32 : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $^ -o $@$(EXT)

 zstreamtest_asan : CFLAGS += -fsanitize=address
 zstreamtest_asan : $(ZSTREAMFILES)
@@ -208,7 +205,7 @@ clean:
        zstreamtest$(EXT) zstreamtest32$(EXT) \
        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
        symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
-	decodecorpus$(EXT)
+        decodecorpus$(EXT)
 	@echo Cleaning completed


@@ -241,7 +238,7 @@ endif


 #-----------------------------------------------------------------------------
-#make tests validated only for MSYS, Linux, OSX, BSD, Hurd and Solaris targets
+# make tests validated only for below targets
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))

@@ -303,10 +300,10 @@ test-fullbench32: fullbench32 datagen
 	$(QEMU_SYS) ./fullbench32 -i1 -P0

 test-fuzzer: fuzzer
-	$(QEMU_SYS) ./fuzzer $(FUZZERTEST) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)

 test-fuzzer32: fuzzer32
-	$(QEMU_SYS) ./fuzzer32 $(FUZZERTEST) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)

 test-zbuff: zbufftest
 	$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
@@ -315,10 +312,10 @@ test-zbuff32: zbufftest32
 	$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)

 test-zstream: zstreamtest
-	$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
-	$(QEMU_SYS) ./zstreamtest --mt $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
-	$(QEMU_SYS) ./zstreamtest --newapi $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
-	$(QEMU_SYS) ./zstreamtest --opaqueapi $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest --mt -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest --opaqueapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)

 test-zstream32: zstreamtest32
 	$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
@@ -374,4 +371,39 @@ test-decodecorpus-cli: decodecorpus
 test-pool: poolTests
 	$(QEMU_SYS) ./poolTests

+test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
+test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
+test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
+test-lz4: zstd decodecorpus datagen
+	[ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
+	[ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4
+
+	./decodecorpus -ptmp
+	# lz4 -> zstd
+	lz4 < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+	lz4 < tmp | \
+	$(ZSTD_UNLZ4) | \
+	cmp - tmp
+	# zstd -> lz4
+	$(ZSTD) --format=lz4 < tmp | \
+	lz4 -d | \
+	cmp - tmp
+	$(ZSTD_LZ4) < tmp | \
+	lz4 -d | \
+	cmp - tmp
+	# zstd -> zstd
+	$(ZSTD) --format=lz4 < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+	# zstd -> zstd
+	$(ZSTD) < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+
+	./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null
+
+	rm tmp lz4 unlz4
+
 endif
@@ -14,8 +14,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <time.h>

+#include "util.h"
 #include "zstd.h"
 #include "zstd_internal.h"
 #include "mem.h"
@@ -45,24 +45,20 @@
 **************************************/
 #define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-static U32 g_displayLevel = 0;
+static U32 g_displayLevel = 2;

 #define DISPLAYUPDATE(...)                                                     \
    do {                                                                       \
-        if ((clockSpan(g_displayClock) > g_refreshRate) ||                     \
+        if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) ||           \
            (g_displayLevel >= 4)) {                                           \
-            g_displayClock = clock();                                          \
+            g_displayClock = UTIL_getTime();                                   \
            DISPLAY(__VA_ARGS__);                                              \
            if (g_displayLevel >= 4) fflush(stderr);                           \
        }                                                                      \
    } while (0)
-static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
-static clock_t g_displayClock = 0;

-static clock_t clockSpan(clock_t cStart)
-{
-    return clock() - cStart;   /* works even when overflow; max span ~ 30mn */
-}
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

 #define CHECKERR(code)                                                         \
    do {                                                                       \
@@ -174,21 +170,19 @@ const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"};
 #define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG)

 #define MAX_WINDOW_LOG 22 /* Recommended support is 8MB, so limit to 4MB + mantissa */
-#define MAX_BLOCK_SIZE_LOG 17
-#define MAX_BLOCK_SIZE (1ULL << MAX_BLOCK_SIZE_LOG) /* 128 KB */

 #define MIN_SEQ_LEN (3)
-#define MAX_NB_SEQ ((MAX_BLOCK_SIZE + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
+#define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)

 BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
 BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
-BYTE LITERAL_BUFFER[MAX_BLOCK_SIZE];
+BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];

 seqDef SEQUENCE_BUFFER[MAX_NB_SEQ];
-BYTE SEQUENCE_LITERAL_BUFFER[MAX_BLOCK_SIZE]; /* storeSeq expects a place to copy literals to */
-BYTE SEQUENCE_LLCODE[MAX_BLOCK_SIZE];
-BYTE SEQUENCE_MLCODE[MAX_BLOCK_SIZE];
-BYTE SEQUENCE_OFCODE[MAX_BLOCK_SIZE];
+BYTE SEQUENCE_LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; /* storeSeq expects a place to copy literals to */
+BYTE SEQUENCE_LLCODE[ZSTD_BLOCKSIZE_MAX];
+BYTE SEQUENCE_MLCODE[ZSTD_BLOCKSIZE_MAX];
+BYTE SEQUENCE_OFCODE[ZSTD_BLOCKSIZE_MAX];

 unsigned WKSP[1024];

@@ -249,7 +243,7 @@ typedef enum {
 *  Global variables (set from command line)
 *********************************************************/
 U32 g_maxDecompressedSizeLog = MAX_DECOMPRESSED_SIZE_LOG;  /* <= 20 */
-U32 g_maxBlockSize = MAX_BLOCK_SIZE;                       /* <= 128 KB */
+U32 g_maxBlockSize = ZSTD_BLOCKSIZE_MAX;                       /* <= 128 KB */

 /*-*******************************************************
 *  Generator Functions
@@ -356,10 +350,10 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info)
        }
    }

-    DISPLAYLEVEL(2, " frame content size:\t%u\n", (U32)fh.contentSize);
-    DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize);
-    DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag);
-    DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment);
+    DISPLAYLEVEL(3, " frame content size:\t%u\n", (U32)fh.contentSize);
+    DISPLAYLEVEL(3, " frame window size:\t%u\n", fh.windowSize);
+    DISPLAYLEVEL(3, " content size flag:\t%d\n", contentSizeFlag);
+    DISPLAYLEVEL(3, " single segment flag:\t%d\n", singleSegment);

    frame->data = op + pos;
    frame->header = fh;
@@ -881,7 +875,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
                                  frame->stats.offsetSymbolSet, 28)) {
            Offtype = set_repeat;
        } else if (!(RAND(seed) & 3)) {
-            FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
+            FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, DefaultMaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
            Offtype = set_basic;
        } else {
            size_t nbSeq_1 = nbSeq;
@@ -1034,9 +1028,9 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
    BYTE *const header = (BYTE*)frame->data;
    BYTE *op = header + 3;

-    DISPLAYLEVEL(3, " block:\n");
-    DISPLAYLEVEL(3, "  block content size: %u\n", (U32)contentSize);
-    DISPLAYLEVEL(3, "  last block: %s\n", lastBlock ? "yes" : "no");
+    DISPLAYLEVEL(4, " block:\n");
+    DISPLAYLEVEL(4, "  block content size: %u\n", (U32)contentSize);
+    DISPLAYLEVEL(4, "  last block: %s\n", lastBlock ? "yes" : "no");

    if (blockTypeDesc == 0) {
        /* Raw data frame */
@@ -1066,7 +1060,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,

        frame->data = op;
        compressedSize = writeCompressedBlock(seed, frame, contentSize, info);
-        if (compressedSize > contentSize) {
+        if (compressedSize >= contentSize) {   /* compressed block must be strictly smaller than uncompressed one */
            blockType = 0;
            memcpy(op, frame->src, contentSize);

@@ -1082,8 +1076,8 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
    }
    frame->src = (BYTE*)frame->src + contentSize;

-    DISPLAYLEVEL(3, "  block type: %s\n", BLOCK_TYPES[blockType]);
-    DISPLAYLEVEL(3, "  block size field: %u\n", (U32)blockSize);
+    DISPLAYLEVEL(4, "  block type: %s\n", BLOCK_TYPES[blockType]);
+    DISPLAYLEVEL(4, "  block size field: %u\n", (U32)blockSize);

    header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff);
    MEM_writeLE16(header + 1, (U16) (blockSize >> 5));
@@ -1103,13 +1097,13 @@ static void writeBlocks(U32* seed, frame_t* frame, dictInfo info)
            blockContentSize = contentLeft;
        } else {
            if (contentLeft > 0 && (RAND(seed) & 7)) {
-                /* some variable size blocks */
+                /* some variable size block */
                blockContentSize = RAND(seed) % (MIN(maxBlockSize, contentLeft)+1);
            } else if (contentLeft > maxBlockSize && (RAND(seed) & 1)) {
-                /* some full size blocks */
+                /* some full size block */
                blockContentSize = maxBlockSize;
            } else {
-                /* some empty blocks */
+                /* some empty block */
                blockContentSize = 0;
            }
        }
@@ -1125,7 +1119,7 @@ static void writeChecksum(frame_t* frame)
 {
    /* write checksum so implementations can verify their output */
    U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0);
-    DISPLAYLEVEL(2, "  checksum: %08x\n", (U32)digest);
+    DISPLAYLEVEL(3, "  checksum: %08x\n", (U32)digest);
    MEM_writeLE32(frame->data, (U32)digest);
    frame->data = (BYTE*)frame->data + 4;
 }
@@ -1146,8 +1140,7 @@ static void outputBuffer(const void* buf, size_t size, const char* const path)
        exit(1);
    }

-    {
-        size_t fsize = size;
+    {   size_t fsize = size;
        size_t written = 0;
        while (written < fsize) {
            written += fwrite(ip + written, 1, fsize - written, out);
@@ -1187,23 +1180,21 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
    size_t blockContentSize;
    int blockWritten = 0;
    BYTE* op;
-    DISPLAYLEVEL(1, "block seed: %u\n", seed);
+    DISPLAYLEVEL(4, "block seed: %u\n", seed);
    initFrame(frame);
    op = (BYTE*)frame->data;

    while (!blockWritten) {
        size_t cSize;
        /* generate window size */
-        {
-            int const exponent = RAND(&seed) % (MAX_WINDOW_LOG - 10);
+        {   int const exponent = RAND(&seed) % (MAX_WINDOW_LOG - 10);
            int const mantissa = RAND(&seed) % 8;
            frame->header.windowSize = (1U << (exponent + 10));
            frame->header.windowSize += (frame->header.windowSize / 8) * mantissa;
        }

        /* generate content size */
-        {
-            size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize);
+        {   size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize);
            if (RAND(&seed) & 15) {
                /* some full size blocks */
                blockContentSize = maxBlockSize;
@@ -1220,13 +1211,13 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
        frame->oldStats = frame->stats;
        frame->data = op;
        cSize = writeCompressedBlock(&seed, frame, blockContentSize, info);
-        if (cSize > blockContentSize) {
+        if (cSize >= blockContentSize) {  /* compressed size must be strictly smaller than decompressed size : https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks */
            /* data doesn't compress -- try again */
            frame->stats = frame->oldStats; /* don't update the stats */
-            DISPLAYLEVEL(3, "   can't compress block\n");
+            DISPLAYLEVEL(5, "   can't compress block : try again \n");
        } else {
            blockWritten = 1;
-            DISPLAYLEVEL(3, "   block size: %u\n", (U32)cSize);
+            DISPLAYLEVEL(4, "   block size: %u \n", (U32)cSize);
            frame->src = (BYTE*)frame->src + blockContentSize;
        }
    }
@@ -1237,7 +1228,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
 static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info)
 {
    /* generate a complete frame */
-    DISPLAYLEVEL(1, "frame seed: %u\n", seed);
+    DISPLAYLEVEL(3, "frame seed: %u\n", seed);
    initFrame(fr);

    writeFrameHeader(&seed, fr, info);
@@ -1251,7 +1242,8 @@ static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info)
 *  Dictionary Helper Functions
 *********************************************************/
 /* returns 0 if successful, otherwise returns 1 upon error */
-static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){
+static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict)
+{
    /* allocate space for samples */
    int ret = 0;
    unsigned const numSamples = 4;
@@ -1263,27 +1255,20 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){
    }

    /* generate samples */
-    {
-        unsigned literalValue = 1;
+    {   unsigned literalValue = 1;
        unsigned samplesPos = 0;
        size_t currSize = 1;
        while (literalValue <= 4) {
            sampleSizes[literalValue - 1] = currSize;
-            {
-                size_t k;
+            {   size_t k;
                for (k = 0; k < currSize; k++) {
                    *(samples + (samplesPos++)) = (BYTE)literalValue;
-                }
-            }
+            }   }
            literalValue++;
            currSize *= 16;
-        }
-    }
+    }   }

-
-    {
-        /* create variables */
-        size_t dictWriteSize = 0;
+    {   size_t dictWriteSize = 0;
        ZDICT_params_t zdictParams;
        size_t const headerSize = MAX(dictSize/4, 256);
        size_t const dictContentSize = dictSize - headerSize;
@@ -1403,24 +1388,19 @@ static size_t testDecodeWithDict(U32 seed, genType_e genType)
    }

    /* generate random dictionary */
-    {
-        int const ret = genRandomDict(dictID, seed, dictSize, fullDict);
-        if (ret != 0) {
-            errorDetected = ERROR(GENERIC);
-            goto dictTestCleanup;
-        }
+    if (genRandomDict(dictID, seed, dictSize, fullDict)) {  /* return 0 on success */
+        errorDetected = ERROR(GENERIC);
+        goto dictTestCleanup;
    }


-    {
-        frame_t fr;
+    {   frame_t fr;
        dictInfo info;
        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
        size_t ret;

        /* get dict info */
-        {
-            size_t const headerSize = MAX(dictSize/4, 256);
+        {   size_t const headerSize = MAX(dictSize/4, 256);
            size_t const dictContentSize = dictSize-headerSize;
            BYTE* const dictContent = fullDict+headerSize;
            info = initDictInfo(1, dictContentSize, dictContent, dictID);
@@ -1429,13 +1409,13 @@ static size_t testDecodeWithDict(U32 seed, genType_e genType)
        /* manually decompress and check difference */
        if (genType == gt_frame) {
            /* Test frame */
-            seed = generateFrame(seed, &fr, info);
+            generateFrame(seed, &fr, info);
            ret = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE,
                                            fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart,
                                            fullDict, dictSize);
        } else {
            /* Test block */
-            seed = generateCompressedBlock(seed, &fr, info);
+            generateCompressedBlock(seed, &fr, info);
            ret = ZSTD_decompressBegin_usingDict(dctx, fullDict, dictSize);
            if (ZSTD_isError(ret)) {
                errorDetected = ret;
@@ -1488,8 +1468,7 @@ static int runBlockTest(U32* seed)
 {
    frame_t fr;
    U32 const seedCopy = *seed;
-    {
-        dictInfo const info = initDictInfo(0, 0, NULL, 0);
+    {   dictInfo const info = initDictInfo(0, 0, NULL, 0);
        *seed = generateCompressedBlock(*seed, &fr, info);
    }

@@ -1501,8 +1480,7 @@ static int runBlockTest(U32* seed)
        }
    }

-    {
-        size_t const r = testDecodeWithDict(*seed, gt_block);
+    {   size_t const r = testDecodeWithDict(*seed, gt_block);
        if (ZSTD_isError(r)) {
            DISPLAY("Error in block mode with dictionary on test seed %u: %s\n",
                    seedCopy, ZSTD_getErrorName(r));
@@ -1516,30 +1494,28 @@ static int runFrameTest(U32* seed)
 {
    frame_t fr;
    U32 const seedCopy = *seed;
-    {
-        dictInfo const info = initDictInfo(0, 0, NULL, 0);
+    {   dictInfo const info = initDictInfo(0, 0, NULL, 0);
        *seed = generateFrame(*seed, &fr, info);
    }

    {   size_t const r = testDecodeSimple(&fr);
        if (ZSTD_isError(r)) {
-            DISPLAY("Error in simple mode on test seed %u: %s\n", seedCopy,
-                    ZSTD_getErrorName(r));
+            DISPLAY("Error in simple mode on test seed %u: %s\n",
+                    seedCopy, ZSTD_getErrorName(r));
            return 1;
        }
    }
    {   size_t const r = testDecodeStreaming(&fr);
        if (ZSTD_isError(r)) {
-            DISPLAY("Error in streaming mode on test seed %u: %s\n", seedCopy,
-                    ZSTD_getErrorName(r));
+            DISPLAY("Error in streaming mode on test seed %u: %s\n",
+                    seedCopy, ZSTD_getErrorName(r));
            return 1;
        }
    }
-    {
-        /* don't create a dictionary that is too big */
-        size_t const r = testDecodeWithDict(*seed, gt_frame);
+    {   size_t const r = testDecodeWithDict(*seed, gt_frame);  /* avoid big dictionaries */
        if (ZSTD_isError(r)) {
-            DISPLAY("Error in dictionary mode on test seed %u: %s\n", seedCopy, ZSTD_getErrorName(r));
+            DISPLAY("Error in dictionary mode on test seed %u: %s\n",
+                    seedCopy, ZSTD_getErrorName(r));
            return 1;
        }
    }
@@ -1551,29 +1527,23 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS
 {
    unsigned fnum;

-    clock_t const startClock = clock();
-    clock_t const maxClockSpan = testDurationS * CLOCKS_PER_SEC;
+    UTIL_time_t const startClock = UTIL_getTime();
+    U64 const maxClockSpan = testDurationS * SEC_TO_MICRO;

    if (numFiles == 0 && !testDurationS) numFiles = 1;

    DISPLAY("seed: %u\n", seed);

-    for (fnum = 0; fnum < numFiles || clockSpan(startClock) < maxClockSpan; fnum++) {
+    for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) {
        if (fnum < numFiles)
            DISPLAYUPDATE("\r%u/%u        ", fnum, numFiles);
        else
            DISPLAYUPDATE("\r%u           ", fnum);
-        {
-          int ret;
-          if (genType == gt_frame) {
-              ret = runFrameTest(&seed);
-          } else {
-              ret = runBlockTest(&seed);
-          }

-          if (ret) {
-              return ret;
-          }
+        {   int const ret = (genType == gt_frame) ?
+                            runFrameTest(&seed) :
+                            runBlockTest(&seed);
+            if (ret) return ret;
        }
    }

@@ -1594,8 +1564,7 @@ static int generateFile(U32 seed, const char* const path,

    DISPLAY("seed: %u\n", seed);

-    {
-        dictInfo const info = initDictInfo(0, 0, NULL, 0);
+    {   dictInfo const info = initDictInfo(0, 0, NULL, 0);
        if (genType == gt_frame) {
            generateFrame(seed, &fr, info);
        } else {
@@ -1622,8 +1591,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,

        DISPLAYUPDATE("\r%u/%u        ", fnum, numFiles);

-        {
-            dictInfo const info = initDictInfo(0, 0, NULL, 0);
+        {   dictInfo const info = initDictInfo(0, 0, NULL, 0);
            if (genType == gt_frame) {
                seed = generateFrame(seed, &fr, info);
            } else {
@@ -1673,8 +1641,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const
    }

    /* randomly generate the dictionary */
-    {
-        int const ret = genRandomDict(dictID, seed, dictSize, fullDict);
+    {   int const ret = genRandomDict(dictID, seed, dictSize, fullDict);
        if (ret != 0) {
            errorDetected = ret;
            goto dictCleanup;
@@ -1695,8 +1662,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const
    }

    /* generate random compressed/decompressed files */
-    {
-        unsigned fnum;
+    {   unsigned fnum;
        for (fnum = 0; fnum < MAX(numFiles, 1); fnum++) {
            frame_t fr;
            DISPLAYUPDATE("\r%u/%u        ", fnum, numFiles);
@@ -1907,7 +1873,7 @@ int main(int argc, char** argv)
                        genType = gt_block;
                    } else if (longCommandWArg(&argument, "max-block-size-log=")) {
                        U32 value = readU32FromChar(&argument);
-                        if (value >= 2 && value <= MAX_BLOCK_SIZE) {
+                        if (value >= 2 && value <= ZSTD_BLOCKSIZE_MAX) {
                            g_maxBlockSize = 1U << value;
                        }
                    } else if (longCommandWArg(&argument, "max-content-size-log=")) {
@@ -233,7 +233,7 @@ static ZSTD_CCtx* g_zcc = NULL;
 size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
 {
    (void)buff2;
-    ZSTD_compressBegin(g_zcc, 1);
+    ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
    return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
 }

@@ -376,6 +376,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
            skippedSize = frameHeaderSize + ZSTD_blockHeaderSize;
            memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize);
            srcSize = srcSize > 128 KB ? 128 KB : srcSize;    /* speed relative to block */
+            ZSTD_decompressBegin(g_zdc);
            break;
        }
    case 32:   /* ZSTD_decodeSeqHeaders */
@@ -483,8 +484,8 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
    /* Loop for each file */
    int fileIdx;
    for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
-        const char* inFileName = fileNamesTable[fileIdx];
-        FILE* inFile = fopen( inFileName, "rb" );
+        const char* const inFileName = fileNamesTable[fileIdx];
+        FILE* const inFile = fopen( inFileName, "rb" );
        U64   inFileSize;
        size_t benchedSize;
        void* origBuff;
@@ -494,6 +495,11 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN

        /* Memory allocation & restrictions */
        inFileSize = UTIL_getFileSize(inFileName);
+        if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAY( "Cannot measure size of %s\n", inFileName);
+            fclose(inFile);
+            return 11;
+        }
        benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
        if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
        if (benchedSize < inFileSize)
@@ -0,0 +1,5 @@
+# test artefacts
+corpora
+block_decompress
+block_round_trip
+simple_round_trip
@@ -14,6 +14,13 @@ CPPFLAGS ?=
 LDFLAGS ?=
 ARFLAGS ?=
 LIB_FUZZING_ENGINE ?= libregression.a
+PYTHON ?= python
+ifeq ($(shell uname), Darwin)
+	DOWNLOAD?=curl -L -o
+else
+	DOWNLOAD?=wget -O
+endif
+CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/

 ZSTDDIR = ../../lib
 PRGDIR = ../../programs
@@ -48,18 +55,20 @@ FUZZ_SRC       := \
 FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC)))


-.PHONY: default all clean
+.PHONY: default all clean cleanall

 default: all

-all: \
+FUZZ_TARGETS :=       \
 	simple_round_trip \
 	stream_round_trip \
-	block_round_trip \
+	block_round_trip  \
 	simple_decompress \
 	stream_decompress \
 	block_decompress

+all: $(FUZZ_TARGETS)
+
 %.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@

@@ -93,7 +102,26 @@ libFuzzer:
 	@git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer
 	@cd Fuzzer && ./build.sh

+corpora/%_seed_corpus.zip:
+	@mkdir -p corpora
+	$(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip
+
+corpora/%: corpora/%_seed_corpus.zip
+	unzip -q $^ -d $@
+
+.PHONY: corpora
+corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS))
+
+regressiontest: corpora
+	CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all
+	$(PYTHON) ./fuzz.py regression all
+
 clean:
 	@$(MAKE) -C $(ZSTDDIR) clean
-	@$(RM) -f *.a *.o
-	@$(RM) -f simple_round_trip stream_round_trip simple_decompress stream_decompress
+	@$(RM) *.a *.o
+	@$(RM) simple_round_trip stream_round_trip simple_decompress \
+           stream_decompress block_decompress block_round_trip
+
+cleanall:
+	@$(RM) -r Fuzzer
+	@$(RM) -r corpora
@@ -1,6 +1,14 @@
 # Fuzzing

 Each fuzzing target can be built with multiple engines.
+Zstd provides a fuzz corpus for each target that can be downloaded with
+the command:
+
+```
+make corpora
+```
+
+It will download each corpus into `./corpora/TARGET`.

 ## fuzz.py

@@ -34,27 +34,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
                            void *compressed, size_t compressedCapacity,
                            const void *src, size_t srcSize)
 {
-  int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
-  size_t ret = ZSTD_compressBegin(cctx, cLevel);
+    int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
+    ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0);
+    size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize);
+    FUZZ_ZASSERT(ret);

-  if (ZSTD_isError(ret)) {
-      fprintf(stderr, "ZSTD_compressBegin() error: %s\n",
-              ZSTD_getErrorName(ret));
-      return ret;
-  }
-
-  ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize);
-  if (ZSTD_isError(ret)) {
-    fprintf(stderr, "ZSTD_compressBlock() error: %s\n", ZSTD_getErrorName(ret));
-    return ret;
-  }
-  if (ret == 0) {
+    ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize);
+    FUZZ_ZASSERT(ret);
+    if (ret == 0) {
        FUZZ_ASSERT(resultCapacity >= srcSize);
        memcpy(result, src, srcSize);
        return srcSize;
-  }
-  ZSTD_decompressBegin(dctx);
-  return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret);
+    }
+    ZSTD_decompressBegin(dctx);
+    return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret);
 }

 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
@@ -87,7 +80,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
    {
        size_t const result =
            roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
-        FUZZ_ASSERT_MSG(!ZSTD_isError(result), ZSTD_getErrorName(result));
+        FUZZ_ZASSERT(result);
        FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
        FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
    }
@@ -0,0 +1,2 @@
+[libfuzzer]
+max_len = 8192
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#!/usr/bin/env python

 # ################################################################
 # Copyright (c) 2016-present, Facebook, Inc.
@@ -82,11 +82,39 @@ def tmpdir():
        shutil.rmtree(dirpath, ignore_errors=True)


+def parse_targets(in_targets):
+    targets = set()
+    for target in in_targets:
+        if not target:
+            continue
+        if target == 'all':
+            targets = targets.union(TARGETS)
+        elif target in TARGETS:
+            targets.add(target)
+        else:
+            raise RuntimeError('{} is not a valid target'.format(target))
+    return list(targets)
+
+
+def targets_parser(args, description):
+    parser = argparse.ArgumentParser(prog=args.pop(0), description=description)
+    parser.add_argument(
+        'TARGET',
+        nargs='*',
+        type=str,
+        help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)))
+    args, extra = parser.parse_known_args(args)
+    args.extra = extra
+
+    args.TARGET = parse_targets(args.TARGET)
+
+    return args
+
+
 def parse_env_flags(args, flags):
    """
    Look for flags set by environment variables.
    """
-    flags = ' '.join(flags)
    san_flags = ','.join(re.findall('-fsanitize=((?:[a-z]+,?)+)', flags))
    nosan_flags = ','.join(re.findall('-fno-sanitize=((?:[a-z]+,?)+)', flags))

@@ -112,6 +140,34 @@ def parse_env_flags(args, flags):
    return args


+def compiler_version(cc, cxx):
+    """
+    Determines the compiler and version.
+    Only works for clang and gcc.
+    """
+    cc_version_bytes = subprocess.check_output([cc, "--version"])
+    cxx_version_bytes = subprocess.check_output([cxx, "--version"])
+    if cc_version_bytes.startswith(b'clang'):
+        assert(cxx_version_bytes.startswith(b'clang'))
+        compiler = 'clang'
+    if cc_version_bytes.startswith(b'gcc'):
+        assert(cxx_version_bytes.startswith(b'g++'))
+        compiler = 'gcc'
+    version_regex = b'([0-9])+\.([0-9])+\.([0-9])+'
+    version_match = re.search(version_regex, cc_version_bytes)
+    version = tuple(int(version_match.group(i)) for i in range(1, 4))
+    return compiler, version
+
+
+def overflow_ubsan_flags(cc, cxx):
+    compiler, version = compiler_version(cc, cxx)
+    if compiler == 'gcc':
+        return ['-fno-sanitize=signed-integer-overflow']
+    if compiler == 'clang' and version >= (5, 0, 0):
+        return ['-fno-sanitize=pointer-overflow']
+    return []
+
+
 def build_parser(args):
    description = """
    Cleans the repository and builds a fuzz target (or all).
@@ -336,7 +392,7 @@ def build(args):
    if args.ubsan:
        ubsan_flags = ['-fsanitize=undefined']
        if not args.ubsan_pointer_overflow:
-            ubsan_flags += ['-fno-sanitize=pointer-overflow']
+            ubsan_flags += overflow_ubsan_flags(cc, cxx)
        common_flags += ubsan_flags

    if args.stateful_fuzzing:
@@ -424,36 +480,42 @@ def libfuzzer_parser(args):
    if args.TARGET and args.TARGET not in TARGETS:
        raise RuntimeError('{} is not a valid target'.format(args.TARGET))

-    if not args.corpora:
-        args.corpora = abs_join(CORPORA_DIR, args.TARGET)
-    if not args.artifact:
-        args.artifact = abs_join(CORPORA_DIR, '{}-crash'.format(args.TARGET))
-    if not args.seed:
-        args.seed = abs_join(CORPORA_DIR, '{}-seed'.format(args.TARGET))
-
    return args


-def libfuzzer(args):
-    try:
-        args = libfuzzer_parser(args)
-    except Exception as e:
-        print(e)
-        return 1
-    target = abs_join(FUZZ_DIR, args.TARGET)
+def libfuzzer(target, corpora=None, artifact=None, seed=None, extra_args=None):
+    if corpora is None:
+        corpora = abs_join(CORPORA_DIR, target)
+    if artifact is None:
+        artifact = abs_join(CORPORA_DIR, '{}-crash'.format(target))
+    if seed is None:
+        seed = abs_join(CORPORA_DIR, '{}-seed'.format(target))
+    if extra_args is None:
+        extra_args = []

-    corpora = [create(args.corpora)]
-    artifact = create(args.artifact)
-    seed = check(args.seed)
+    target = abs_join(FUZZ_DIR, target)
+
+    corpora = [create(corpora)]
+    artifact = create(artifact)
+    seed = check(seed)

    corpora += [artifact]
    if seed is not None:
        corpora += [seed]

    cmd = [target, '-artifact_prefix={}/'.format(artifact)]
-    cmd += corpora + args.extra
+    cmd += corpora + extra_args
    print(' '.join(cmd))
-    subprocess.call(cmd)
+    subprocess.check_call(cmd)
+
+
+def libfuzzer_cmd(args):
+    try:
+        args = libfuzzer_parser(args)
+    except Exception as e:
+        print(e)
+        return 1
+    libfuzzer(args.TARGET, args.corpora, args.artifact, args.seed, args.extra)
    return 0


@@ -518,39 +580,15 @@ def afl(args):
    return 0


-def regression_parser(args):
-    description = """
-    Runs one or more regression tests.
-    The fuzzer should have been built with with
-    LIB_FUZZING_ENGINE='libregression.a'.
-    Takes input from CORPORA.
-    """
-    parser = argparse.ArgumentParser(prog=args.pop(0), description=description)
-    parser.add_argument(
-        'TARGET',
-        nargs='*',
-        type=str,
-        help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)))
-    args = parser.parse_args(args)
-
-    targets = set()
-    for target in args.TARGET:
-        if not target:
-            continue
-        if target == 'all':
-            targets = targets.union(TARGETS)
-        elif target in TARGETS:
-            targets.add(target)
-        else:
-            raise RuntimeError('{} is not a valid target'.format(target))
-    args.TARGET = list(targets)
-
-    return args
-
-
 def regression(args):
    try:
-        args = regression_parser(args)
+        description = """
+        Runs one or more regression tests.
+        The fuzzer should have been built with with
+        LIB_FUZZING_ENGINE='libregression.a'.
+        Takes input from CORPORA.
+        """
+        args = targets_parser(args, description)
    except Exception as e:
        print(e)
        return 1
@@ -673,6 +711,56 @@ def gen(args):
    return 0


+def minimize(args):
+    try:
+        description = """
+        Runs a libfuzzer fuzzer with -merge=1 to build a minimal corpus in
+        TARGET_seed_corpus. All extra args are passed to libfuzzer.
+        """
+        args = targets_parser(args, description)
+    except Exception as e:
+        print(e)
+        return 1
+
+    for target in args.TARGET:
+        # Merge the corpus + anything else into the seed_corpus
+        corpus = abs_join(CORPORA_DIR, target)
+        seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target))
+        extra_args = [corpus, "-merge=1"] + args.extra
+        libfuzzer(target, corpora=seed_corpus, extra_args=extra_args)
+        seeds = set(os.listdir(seed_corpus))
+        # Copy all crashes directly into the seed_corpus if not already present
+        crashes = abs_join(CORPORA_DIR, '{}-crash'.format(target))
+        for crash in os.listdir(crashes):
+            if crash not in seeds:
+                shutil.copy(abs_join(crashes, crash), seed_corpus)
+                seeds.add(crash)
+
+
+def zip_cmd(args):
+    try:
+        description = """
+        Zips up the seed corpus.
+        """
+        args = targets_parser(args, description)
+    except Exception as e:
+        print(e)
+        return 1
+
+    for target in args.TARGET:
+        # Zip the seed_corpus
+        seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target))
+        seeds = [abs_join(seed_corpus, f) for f in os.listdir(seed_corpus)]
+        zip_file = "{}.zip".format(seed_corpus)
+        cmd = ["zip", "-q", "-j", "-9", zip_file]
+        print(' '.join(cmd + [abs_join(seed_corpus, '*')]))
+        subprocess.check_call(cmd + seeds)
+
+
+def list_cmd(args):
+    print("\n".join(TARGETS))
+
+
 def short_help(args):
    name = args[0]
    print("Usage: {} [OPTIONS] COMMAND [ARGS]...\n".format(name))
@@ -690,6 +778,9 @@ def help(args):
    print("\tafl\t\tRun an AFL fuzzer")
    print("\tregression\tRun a regression test")
    print("\tgen\t\tGenerate a seed corpus for a fuzzer")
+    print("\tminimize\tMinimize the test corpora")
+    print("\tzip\t\tZip the minimized corpora up")
+    print("\tlist\t\tList the available targets")


 def main():
@@ -705,13 +796,19 @@ def main():
    if command == "build":
        return build(args)
    if command == "libfuzzer":
-        return libfuzzer(args)
+        return libfuzzer_cmd(args)
    if command == "regression":
        return regression(args)
    if command == "afl":
        return afl(args)
    if command == "gen":
        return gen(args)
+    if command == "minimize":
+        return minimize(args)
+    if command == "zip":
+        return zip_cmd(args)
+    if command == "list":
+        return list_cmd(args)
    short_help(args)
    print("Error: No such command {} (pass -h for help)".format(command))
    return 1
@@ -38,23 +38,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
    if (FUZZ_rand(&seed) & 1) {
        ZSTD_inBuffer in = {src, srcSize, 0};
        ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
+        size_t err;

        ZSTD_CCtx_reset(cctx);
-        FUZZ_setRandomParameters(cctx, &seed);
-        size_t const err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
-        if (err != 0) {
-            return err;
-        }
+        FUZZ_setRandomParameters(cctx, srcSize, &seed);
+        err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+        FUZZ_ZASSERT(err);
+        FUZZ_ASSERT(err == 0);
        cSize = out.pos;
    } else {
        int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
        cSize = ZSTD_compressCCtx(
            cctx, compressed, compressedCapacity, src, srcSize, cLevel);
    }
-    if (ZSTD_isError(cSize)) {
-        fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize));
-        return cSize;
-    }
+    FUZZ_ZASSERT(cSize);
    return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
 }

@@ -86,7 +83,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
    {
        size_t const result =
            roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
-        FUZZ_ASSERT_MSG(!ZSTD_isError(result), ZSTD_getErrorName(result));
+        FUZZ_ZASSERT(result);
        FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
        FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
    }
@@ -57,7 +57,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
 {
    size_t dstSize = 0;
    ZSTD_CCtx_reset(cctx);
-    FUZZ_setRandomParameters(cctx, &seed);
+    FUZZ_setRandomParameters(cctx, srcSize, &seed);

    while (srcSize > 0) {
        ZSTD_inBuffer in = makeInBuffer(&src, &srcSize);
@@ -85,7 +85,10 @@ static size_t compress(uint8_t *dst, size_t capacity,
                    /* Reset the compressor when the frame is finished */
                    if (ret == 0) {
                        ZSTD_CCtx_reset(cctx);
-                        FUZZ_setRandomParameters(cctx, &seed);
+                        if ((FUZZ_rand(&seed) & 7) == 0) {
+                            size_t const remaining = in.size - in.pos;
+                            FUZZ_setRandomParameters(cctx, remaining, &seed);
+                        }
                        mode = -1;
                    }
                    break;
@@ -146,7 +149,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
        size_t const cSize = compress(cBuf, neededBufSize, src, size);
        size_t const rSize =
            ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize);
-        FUZZ_ASSERT_MSG(!ZSTD_isError(rSize), ZSTD_getErrorName(rSize));
+        FUZZ_ZASSERT(rSize);
        FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
        FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
    }
@@ -13,30 +13,68 @@
 #include "fuzz_helpers.h"
 #include "zstd.h"

-static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
-                    unsigned max, uint32_t *state) {
-  unsigned const value = FUZZ_rand32(state, min, max);
-  FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
+static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value)
+{
+    FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
 }

-void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state)
+static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
+                    unsigned max, uint32_t *state) {
+    unsigned const value = FUZZ_rand32(state, min, max);
+    set(cctx, param, value);
+}
+
+ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
 {
-    setRand(cctx, ZSTD_p_windowLog, ZSTD_WINDOWLOG_MIN, 23, state);
-    setRand(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN, 23, state);
-    setRand(cctx, ZSTD_p_chainLog, ZSTD_CHAINLOG_MIN, 24, state);
-    setRand(cctx, ZSTD_p_searchLog, ZSTD_SEARCHLOG_MIN, 9, state);
-    setRand(cctx, ZSTD_p_minMatch, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX,
-            state);
-    setRand(cctx, ZSTD_p_targetLength, ZSTD_TARGETLENGTH_MIN,
-            ZSTD_TARGETLENGTH_MAX, state);
-    setRand(cctx, ZSTD_p_compressionStrategy, ZSTD_fast, ZSTD_btultra, state);
+    /* Select compression parameters */
+    ZSTD_compressionParameters cParams;
+    cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15);
+    cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
+    cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
+    cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
+    cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
+                                              ZSTD_SEARCHLENGTH_MAX);
+    cParams.targetLength = FUZZ_rand32(state, ZSTD_TARGETLENGTH_MIN,
+                                              ZSTD_TARGETLENGTH_MAX);
+    cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
+    return ZSTD_adjustCParams(cParams, srcSize, 0);
+}
+
+ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state)
+{
+    /* Select frame parameters */
+    ZSTD_frameParameters fParams;
+    fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1);
+    fParams.checksumFlag = FUZZ_rand32(state, 0, 1);
+    fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1);
+    return fParams;
+}
+
+ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
+{
+    ZSTD_parameters params;
+    params.cParams = FUZZ_randomCParams(srcSize, state);
+    params.fParams = FUZZ_randomFParams(state);
+    return params;
+}
+
+void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
+{
+    ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
+    set(cctx, ZSTD_p_windowLog, cParams.windowLog);
+    set(cctx, ZSTD_p_hashLog, cParams.hashLog);
+    set(cctx, ZSTD_p_chainLog, cParams.chainLog);
+    set(cctx, ZSTD_p_searchLog, cParams.searchLog);
+    set(cctx, ZSTD_p_minMatch, cParams.searchLength);
+    set(cctx, ZSTD_p_targetLength, cParams.targetLength);
+    set(cctx, ZSTD_p_compressionStrategy, cParams.strategy);
    /* Select frame parameters */
    setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
    setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
    setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
    /* Select long distance matchig parameters */
    setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
-    setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 24, state);
+    setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
    setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
            ZSTD_LDM_MINMATCH_MAX, state);
    setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
@@ -21,7 +21,11 @@
 extern "C" {
 #endif

-void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state);
+void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state);
+
+ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state);
+ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state);
+ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state);


 #ifdef __cplusplus
@@ -25,7 +25,7 @@
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
 #include <string.h>       /* strcmp */
-#include <time.h>         /* clock_t */
+#include <assert.h>
 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "zstd.h"         /* ZSTD_VERSION_STRING */
 #include "zstd_errors.h"  /* ZSTD_getErrorCode */
@@ -36,6 +36,7 @@
 #include "mem.h"
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"       /* XXH64 */
+#include "util.h"


 /*-************************************
@@ -56,25 +57,22 @@ static const U32 nbTestsDefault = 30000;
 #define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static U32 g_displayLevel = 2;

-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
-            { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
-static clock_t g_displayClock = 0;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stderr); } }

 /*-*******************************************************
 *  Fuzzer functions
 *********************************************************/
+#undef MIN
+#undef MAX
 #define MIN(a,b) ((a)<(b)?(a):(b))
 #define MAX(a,b) ((a)>(b)?(a):(b))

-static clock_t FUZ_clockSpan(clock_t cStart)
-{
-    return clock() - cStart;   /* works even when overflow; max span ~ 30mn */
-}
-
 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
 static unsigned FUZ_rand(unsigned* src)
 {
@@ -97,6 +95,22 @@ static unsigned FUZ_highbit32(U32 v32)
 }


+/*=============================================
+*   Test macros
+=============================================*/
+#define CHECK_Z(f) {                               \
+    size_t const err = f;                          \
+    if (ZSTD_isError(err)) {                       \
+        DISPLAY("Error => %s : %s ",               \
+                #f, ZSTD_getErrorName(err));       \
+        exit(1);                                   \
+}   }
+
+#define CHECK_V(var, fn)  size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
+#define CHECK(fn)  { CHECK_V(err, fn); }
+#define CHECKPLUS(var, fn, more)  { CHECK_V(var, fn); more; }
+
+
 /*=============================================
 *   Memory Tests
 =============================================*/
@@ -146,14 +160,6 @@ static void FUZ_displayMallocStats(mallocCounter_t count)
        (U32)(count.totalMalloc >> 10));
 }

-#define CHECK_Z(f) {                               \
-    size_t const err = f;                          \
-    if (ZSTD_isError(err)) {                       \
-        DISPLAY("Error => %s : %s ",               \
-                #f, ZSTD_getErrorName(err));       \
-        exit(1);                                   \
-}   }
-
 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 {
    size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
@@ -259,10 +265,6 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 *   Unit tests
 =============================================*/

-#define CHECK_V(var, fn)  size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
-#define CHECK(fn)  { CHECK_V(err, fn); }
-#define CHECKPLUS(var, fn, more)  { CHECK_V(var, fn); more; }
-
 static int basicUnitTests(U32 seed, double compressibility)
 {
    size_t const CNBuffSize = 5 MB;
@@ -359,6 +361,31 @@ static int basicUnitTests(U32 seed, double compressibility)
      if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
    DISPLAYLEVEL(4, "OK \n");

+    DISPLAYLEVEL(4, "test%di : check CCtx size after compressing empty input : ", testNb++);
+    {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
+        if (ZSTD_isError(r)) goto _output_error;
+        if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(4, "OK \n");
+
+    DISPLAYLEVEL(4, "test%di : re-use CCtx with expanding block size : ", testNb++);
+    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+        assert(params.fParams.contentSizeFlag == 1);  /* block size will be adapted if pledgedSrcSize is enabled */
+        CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
+        CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
+
+        CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) );  /* re-use same parameters */
+        {   size_t const inSize = 2* 128 KB;
+            size_t const outSize = ZSTD_compressBound(inSize);
+            CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
+            /* will fail if blockSize is not resized */
+        }
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(4, "OK \n");

    /* Static CCtx tests */
 #define STATIC_CCTX_LEVEL 3
@@ -510,15 +537,14 @@ static int basicUnitTests(U32 seed, double compressibility)
        /* only use the first half so we don't push against size limit of compressedBuffer */
        size_t const segSize = (CNBuffSize / 2) / segs;
        for (i = 0; i < segs; i++) {
-            CHECK_V(r,
-                    ZSTD_compress(
+            CHECK_V(r, ZSTD_compress(
                            (BYTE *)compressedBuffer + off, CNBuffSize - off,
                            (BYTE *)CNBuffer + segSize * i,
                            segSize, 5));
            off += r;
            if (i == segs/2) {
                /* insert skippable frame */
-                const U32 skipLen = 128 KB;
+                const U32 skipLen = 129 KB;
                MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
                MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
                off += skipLen + ZSTD_skippableHeaderSize;
@@ -811,7 +837,7 @@ static int basicUnitTests(U32 seed, double compressibility)
        }
        DISPLAYLEVEL(4, "OK \n");

-        DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail", testNb++);
+        DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
        {
            size_t ret;
            MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -821,7 +847,7 @@ static int basicUnitTests(U32 seed, double compressibility)
        }
        DISPLAYLEVEL(4, "OK \n");

-        DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass", testNb++);
+        DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
        {
            size_t ret;
            MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -831,6 +857,50 @@ static int basicUnitTests(U32 seed, double compressibility)
        }
        DISPLAYLEVEL(4, "OK \n");

+        DISPLAYLEVEL(4, "test%3i : Dictionary with non-default repcodes : ", testNb++);
+        { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
+        dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
+                                         CNBuffer, samplesSizes, nbSamples);
+        if (ZDICT_isError(dictSize)) goto _output_error;
+        /* Set all the repcodes to non-default */
+        {
+            BYTE* dictPtr = (BYTE*)dictBuffer;
+            BYTE* dictLimit = dictPtr + dictSize - 12;
+            /* Find the repcodes */
+            while (dictPtr < dictLimit &&
+                   (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
+                    MEM_readLE32(dictPtr + 8) != 8)) {
+                ++dictPtr;
+            }
+            if (dictPtr >= dictLimit) goto _output_error;
+            MEM_writeLE32(dictPtr + 0, 10);
+            MEM_writeLE32(dictPtr + 4, 10);
+            MEM_writeLE32(dictPtr + 8, 10);
+            /* Set the last 8 bytes to 'x' */
+            memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
+        }
+        /* The optimal parser checks all the repcodes.
+         * Make sure at least one is a match >= targetLength so that it is
+         * immediately chosen. This will make sure that the compressor and
+         * decompressor agree on at least one of the repcodes.
+         */
+        {   size_t dSize;
+            BYTE data[1024];
+            ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
+            ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
+                                            ZSTD_dlm_byRef, ZSTD_dm_auto,
+                                            cParams, ZSTD_defaultCMem);
+            memset(data, 'x', sizeof(data));
+            cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
+                                             data, sizeof(data), cdict);
+            ZSTD_freeCDict(cdict);
+            if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
+            dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
+            if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
+            if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
        ZSTD_freeCCtx(cctx);
        free(dictBuffer);
        free(samplesSizes);
@@ -918,6 +988,43 @@ static int basicUnitTests(U32 seed, double compressibility)
        ZSTD_freeCCtx(cctx);
    }

+    /* custom formats tests */
+    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        size_t const inputSize = CNBuffSize / 2;   /* won't cause pb with small dict size */
+
+        /* basic block compression */
+        DISPLAYLEVEL(4, "test%3i : magic-less format test : ", testNb++);
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) );
+        {   ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
+            ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
+            size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+            if (result != 0) goto _output_error;
+            if (in.pos != in.size) goto _output_error;
+            cSize = out.pos;
+        }
+        DISPLAYLEVEL(4, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize);
+
+        DISPLAYLEVEL(4, "test%3i : decompress normally (should fail) : ", testNb++);
+        {   size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+            if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
+            DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(decodeResult));
+        }
+
+        DISPLAYLEVEL(4, "test%3i : decompress with magic-less instruction : ", testNb++);
+        ZSTD_DCtx_reset(dctx);
+        CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) );
+        {   ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
+            ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
+            size_t const result = ZSTD_decompress_generic(dctx, &out, &in);
+            if (result != 0) goto _output_error;
+            if (in.pos != in.size) goto _output_error;
+            if (out.pos != inputSize) goto _output_error;
+            DISPLAYLEVEL(4, "OK : regenerated %u bytes \n", (U32)out.pos);
+        }
+
+        ZSTD_freeCCtx(cctx);
+    }
+
    /* block API tests */
    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
        static const size_t dictSize = 65 KB;
@@ -961,8 +1068,8 @@ static int basicUnitTests(U32 seed, double compressibility)
        DISPLAYLEVEL(4, "OK \n");

        ZSTD_freeCCtx(cctx);
-        ZSTD_freeDCtx(dctx);
    }
+    ZSTD_freeDCtx(dctx);

    /* long rle test */
    {   size_t sampleSize = 0;
@@ -1164,8 +1271,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
    U32 result = 0;
    U32 testNb = 0;
    U32 coreSeed = seed, lseed = 0;
-    clock_t const startClock = clock();
-    clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
+    UTIL_time_t const startClock = UTIL_getTime();
+    U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
    int const cLevelLimiter = bigTests ? 3 : 2;

    /* allocation */
@@ -1190,7 +1297,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
    for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);

    /* main test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
        size_t sampleSize, maxTestSize, totalTestSize;
        size_t cSize, totalCSize, totalGenSize;
        U64 crcOrig;
@@ -17,13 +17,14 @@
 #include <stdio.h>     /* fprintf, fopen, ftello64 */
 #include <string.h>    /* strcmp */
 #include <math.h>      /* log */
-#include <time.h>      /* clock_t */
+#include <time.h>

 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_parameters, ZSTD_estimateCCtxSize */
 #include "zstd.h"
 #include "datagen.h"
 #include "xxhash.h"
+#include "util.h"


 /*-************************************
@@ -39,7 +40,7 @@
 #define GB *(1ULL<<30)

 #define NBLOOPS    2
-#define TIMELOOP  (2 * CLOCKS_PER_SEC)
+#define TIMELOOP  (2 * SEC_TO_MICRO)

 #define NB_LEVELS_TRACKED 30

@@ -49,8 +50,8 @@ static const size_t maxMemory = (sizeof(size_t)==4)  ?  (2 GB - 64 MB) : (size_t
 static const size_t sampleSize = 10000000;

 static const double g_grillDuration_s = 90000;   /* about 24 hours */
-static const clock_t g_maxParamTime = 15 * CLOCKS_PER_SEC;
-static const clock_t g_maxVariationTime = 60 * CLOCKS_PER_SEC;
+static const U64 g_maxParamTime = 15 * SEC_TO_MICRO;
+static const U64 g_maxVariationTime = 60 * SEC_TO_MICRO;
 static const int g_maxNbVariations = 64;


@@ -88,13 +89,9 @@ void BMK_SetNbIterations(int nbLoops)
 *  Private functions
 *********************************************************/

-/* works even if overflow ; max span ~ 30 mn */
-static clock_t BMK_clockSpan(clock_t cStart) { return clock() - cStart; }
-
 /* accuracy in seconds only, span can be multiple years */
 static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); }

-
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
    size_t const step = 64 MB;
@@ -221,7 +218,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
        size_t cSize = 0;
        double fastestC = 100000000., fastestD = 100000000.;
        double ratio = 0.;
-        clock_t const benchStart = clock();
+        UTIL_time_t const benchStart = UTIL_getTime();

        DISPLAY("\r%79s\r", "");
        memset(&params, 0, sizeof(params));
@@ -229,9 +226,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
        for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
            int nbLoops;
            U32 blockNb;
-            clock_t roundStart, roundClock;
+            UTIL_time_t roundStart;
+            U64 roundClock;

-            { clock_t const benchTime = BMK_clockSpan(benchStart);
+            { U64 const benchTime = UTIL_clockSpanMicro(benchStart);
              if (benchTime > g_maxParamTime) break; }

            /* Compression */
@@ -239,10 +237,9 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
            memset(compressedBuffer, 0xE5, maxCompressedSize);

            nbLoops = 0;
-            roundStart = clock();
-            while (clock() == roundStart);
-            roundStart = clock();
-            while (BMK_clockSpan(roundStart) < TIMELOOP) {
+            UTIL_waitForNextTick();
+            roundStart = UTIL_getTime();
+            while (UTIL_clockSpanMicro(roundStart) < TIMELOOP) {
                for (blockNb=0; blockNb<nbBlocks; blockNb++)
                    blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
                                                    blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
@@ -251,13 +248,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
                                                    params);
                nbLoops++;
            }
-            roundClock = BMK_clockSpan(roundStart);
+            roundClock = UTIL_clockSpanMicro(roundStart);

            cSize = 0;
            for (blockNb=0; blockNb<nbBlocks; blockNb++)
                cSize += blockTable[blockNb].cSize;
            ratio = (double)srcSize / (double)cSize;
-            if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
+            if ((double)roundClock < fastestC * SEC_TO_MICRO * nbLoops) fastestC = ((double)roundClock / SEC_TO_MICRO) / nbLoops;
            DISPLAY("\r");
            DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
            DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
@@ -269,17 +266,16 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
            memset(resultBuffer, 0xD6, srcSize);

            nbLoops = 0;
-            roundStart = clock();
-            while (clock() == roundStart);
-            roundStart = clock();
-            for ( ; BMK_clockSpan(roundStart) < TIMELOOP; nbLoops++) {
+            UTIL_waitForNextTick();
+            roundStart = UTIL_getTime();
+            for ( ; UTIL_clockSpanMicro(roundStart) < TIMELOOP; nbLoops++) {
                for (blockNb=0; blockNb<nbBlocks; blockNb++)
                    blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
                                                                  blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
            }
-            roundClock = BMK_clockSpan(roundStart);
+            roundClock = UTIL_clockSpanMicro(roundStart);

-            if ((double)roundClock < fastestD * CLOCKS_PER_SEC * nbLoops) fastestD = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
+            if ((double)roundClock < fastestD * SEC_TO_MICRO * nbLoops) fastestD = ((double)roundClock / SEC_TO_MICRO) / nbLoops;
            DISPLAY("\r");
            DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
            DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
@@ -310,14 +306,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
 }


-const char* g_stratName[] = { "ZSTD_fast    ",
-                              "ZSTD_dfast   ",
-                              "ZSTD_greedy  ",
-                              "ZSTD_lazy    ",
-                              "ZSTD_lazy2   ",
-                              "ZSTD_btlazy2 ",
-                              "ZSTD_btopt   ",
-                              "ZSTD_btultra "};
+const char* g_stratName[ZSTD_btultra+1] = {
+                "(none)       ", "ZSTD_fast    ", "ZSTD_dfast   ",
+                "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
+                "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};

 static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
 {
@@ -391,8 +383,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
            double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
            double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);

-            size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_advanced_usingCParams(params);
-            size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_advanced_usingCParams(winners[cLevel].params);
+            size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params);
+            size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params);
            double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
            double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);

@@ -525,9 +517,9 @@ static void playAround(FILE* f, winnerInfo_t* winners,
                       ZSTD_CCtx* ctx)
 {
    int nbVariations = 0;
-    clock_t const clockStart = clock();
+    UTIL_time_t const clockStart = UTIL_getTime();

-    while (BMK_clockSpan(clockStart) < g_maxVariationTime) {
+    while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
        ZSTD_compressionParameters p = params;

        if (nbVariations++ > g_maxNbVariations) break;
@@ -687,6 +679,11 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
            DISPLAY( "Pb opening %s\n", inFileName);
            return 11;
        }
+        if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAY("Pb evaluatin size of %s \n", inFileName);
+            fclose(inFile);
+            return 11;
+        }

        /* Memory allocation */
        benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
@@ -740,6 +737,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)

    /* Init */
    if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
+    if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
+        DISPLAY("Pb evaluatin size of %s \n", inFileName);
+        fclose(inFile);
+        return 11;
+    }

    /* Memory allocation & restrictions */
    if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
@@ -13,11 +13,16 @@ roundTripTest() {
        cLevel="$2"
        proba=""
    fi
+    if [ -n "$4" ]; then
+        dLevel="$4"
+    else
+        dLevel="$cLevel"
+    fi

    rm -f tmp1 tmp2
-    $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
+    $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
    ./datagen $1 $proba | $MD5SUM > tmp1
-    ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d  | $MD5SUM > tmp2
+    ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel  | $MD5SUM > tmp2
    $DIFF -q tmp1 tmp2
 }

@@ -29,12 +34,17 @@ fileRoundTripTest() {
        local_c="$2"
        local_p=""
    fi
+    if [ -n "$4" ]; then
+        local_d="$4"
+    else
+        local_d="$local_c"
+    fi

    rm -f tmp.zstd tmp.md5.1 tmp.md5.2
-    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
+    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
    ./datagen $1 $local_p > tmp
-    cat tmp | $MD5SUM > tmp.md5.1
-    $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
+    < tmp $MD5SUM > tmp.md5.1
+    $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
    $DIFF -q tmp.md5.1 tmp.md5.2
 }

@@ -45,12 +55,13 @@ then
 fi

 isWindows=false
-ECHO="echo -e"
 INTOVOID="/dev/null"
+DEVDEVICE="/dev/zero"
 case "$OS" in
  Windows*)
    isWindows=true
    INTOVOID="NUL"
+    DEVDEVICE="NUL"
    ;;
 esac

@@ -66,6 +77,11 @@ case "$UNAME" in
  SunOS) DIFF="gdiff" ;;
 esac

+ECHO="echo -e"
+case "$UNAME" in
+  Darwin) ECHO="echo" ;;
+esac
+
 $ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"

 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
@@ -77,7 +93,7 @@ else
    hasMT="true"
 fi

-$ECHO "\n**** simple tests **** "
+$ECHO "\n===>  simple tests "

 ./datagen > tmp
 $ECHO "test : basic compression "
@@ -110,6 +126,10 @@ $ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal
 fi
 $ECHO "test : null-length file roundtrip"
 $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
+$ECHO "test : ensure small file doesn't add 3-bytes null block"
+./datagen -g1 > tmp1
+$ZSTD tmp1 -c | wc -c | grep "14"
+$ZSTD < tmp1  | wc -c | grep "14"
 $ECHO "test : decompress file with wrong suffix (must fail)"
 $ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
 $ZSTD -df tmp && die "should have refused : wrong extension"
@@ -145,14 +165,36 @@ $ZSTD -f --rm tmp
 test ! -f tmp  # tmp should no longer be present
 $ZSTD -f -d --rm tmp.zst
 test ! -f tmp.zst   # tmp.zst should no longer be present
+$ECHO "test : should quietly not remove non-regular file"
+$ECHO hello > tmp
+$ZSTD tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
+grep -v "Refusing to remove non-regular file" tmplog
+rm -f tmplog
+$ZSTD tmp -f -o "$INTONULL" 2>&1 | grep -v "Refusing to remove non-regular file"
 $ECHO "test : --rm on stdin"
 $ECHO a | $ZSTD --rm > $INTOVOID   # --rm should remain silent
 rm tmp
 $ZSTD -f tmp && die "tmp not present : should have failed"
 test ! -f tmp.zst  # tmp.zst should not be created

+$ECHO "test : compress multiple files"
+$ECHO hello > tmp1
+$ECHO world > tmp2
+$ZSTD tmp1 tmp2 -o "$INTOVOID"
+$ZSTD tmp1 tmp2 -c | $ZSTD -t
+$ZSTD tmp1 tmp2 -o tmp.zst
+test ! -f tmp1.zst
+test ! -f tmp2.zst
+$ZSTD tmp1 tmp2
+$ZSTD -t tmp1.zst tmp2.zst
+$ZSTD -dc tmp1.zst tmp2.zst
+$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID"
+$ZSTD -d tmp1.zst tmp2.zst -o tmp
+rm tmp*

-$ECHO "\n**** Advanced compression parameters **** "
+
+
+$ECHO "\n===>  Advanced compression parameters "
 $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,      - -o tmp.zst && die "wrong parameters not detected!"
 $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21        - -o tmp.zst && die "wrong parameters not detected!"
 $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog  - -o tmp.zst && die "wrong parameters not detected!"
@@ -166,14 +208,14 @@ roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7
 roundTripTest -g512K 19


-$ECHO "\n**** Pass-Through mode **** "
+$ECHO "\n===>  Pass-Through mode "
 $ECHO "Hello world 1!" | $ZSTD -df
 $ECHO "Hello world 2!" | $ZSTD -dcf
 $ECHO "Hello world 3!" > tmp1
 $ZSTD -dcf tmp1


-$ECHO "\n**** frame concatenation **** "
+$ECHO "\n===>  frame concatenation "

 $ECHO "hello " > hello.tmp
 $ECHO "world!" > world.tmp
@@ -204,7 +246,7 @@ $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
 $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"


-$ECHO "\n**** symbolic link test **** "
+$ECHO "\n===>  symbolic link test "

 rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
 $ECHO "hello world" > hello.tmp
@@ -219,7 +261,7 @@ rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
 fi


-$ECHO "\n**** test sparse file support **** "
+$ECHO "\n===>  test sparse file support "

 ./datagen -g5M  -P100 > tmpSparse
 $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
@@ -246,7 +288,7 @@ $DIFF tmpSparse2M tmpSparseRegenerated
 rm tmpSparse*


-$ECHO "\n**** multiple files tests **** "
+$ECHO "\n===>  multiple files tests "

 ./datagen -s1        > tmp1 2> $INTOVOID
 ./datagen -s2 -g100K > tmp2 2> $INTOVOID
@@ -269,7 +311,7 @@ $ECHO "compress multiple files including a missing one (notHere) : "
 $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"


-$ECHO "\n**** dictionary tests **** "
+$ECHO "\n===>  dictionary tests "

 $ECHO "- test with raw dict (content only) "
 ./datagen > tmpDict
@@ -283,6 +325,11 @@ cp $TESTFILE tmp
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -fo result
 $DIFF $TESTFILE result
+if [ -n "$hasMT" ]
+then
+    $ECHO "- Test dictionary compression with multithreading "
+    ./datagen -g5M | $ZSTD -T2 -D tmpDict | $ZSTD -t -D tmpDict   # fails with v1.3.2
+fi
 $ECHO "- Create second (different) dictionary "
 $ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
@@ -325,7 +372,7 @@ $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is
 rm tmp*


-$ECHO "\n**** cover dictionary tests **** "
+$ECHO "\n===>  cover dictionary builder : advanced options "

 TESTFILE=../programs/zstdcli.c
 ./datagen > tmpDict
@@ -345,7 +392,7 @@ $ECHO "- Create dictionary with size limit"
 $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
 rm tmp*

-$ECHO "\n**** legacy dictionary tests **** "
+$ECHO "\n===>  legacy dictionary builder "

 TESTFILE=../programs/zstdcli.c
 ./datagen > tmpDict
@@ -366,7 +413,7 @@ $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
 rm tmp*


-$ECHO "\n**** integrity tests **** "
+$ECHO "\n===>  integrity tests "

 $ECHO "test one file (tmp1.zst) "
 ./datagen > tmp1
@@ -391,13 +438,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"



-$ECHO "\n**** golden files tests **** "
+$ECHO "\n===>  golden files tests "

 $ZSTD -t -r files
 $ZSTD -c -r files | $ZSTD -t


-$ECHO "\n**** benchmark mode tests **** "
+$ECHO "\n===>  benchmark mode tests "

 $ECHO "bench one file"
 ./datagen > tmp1
@@ -408,7 +455,7 @@ $ECHO "with recursive and quiet modes"
 $ZSTD -rqi1b1e2 tmp1


-$ECHO "\n**** gzip compatibility tests **** "
+$ECHO "\n===>  gzip compatibility tests "

 GZIPMODE=1
 $ZSTD --format=gzip -V || GZIPMODE=0
@@ -431,7 +478,7 @@ else
 fi


-$ECHO "\n**** gzip frame tests **** "
+$ECHO "\n===>  gzip frame tests "

 if [ $GZIPMODE -eq 1 ]; then
    ./datagen > tmp
@@ -445,7 +492,7 @@ else
 fi


-$ECHO "\n**** xz compatibility tests **** "
+$ECHO "\n===>  xz compatibility tests "

 LZMAMODE=1
 $ZSTD --format=xz -V || LZMAMODE=0
@@ -491,7 +538,7 @@ else
 fi


-$ECHO "\n**** xz frame tests **** "
+$ECHO "\n===>  xz frame tests "

 if [ $LZMAMODE -eq 1 ]; then
    ./datagen > tmp
@@ -506,7 +553,7 @@ else
    $ECHO "xz mode not supported"
 fi

-$ECHO "\n**** lz4 compatibility tests **** "
+$ECHO "\n===>  lz4 compatibility tests "

 LZ4MODE=1
 $ZSTD --format=lz4 -V || LZ4MODE=0
@@ -529,7 +576,7 @@ else
 fi


-$ECHO "\n**** lz4 frame tests **** "
+$ECHO "\n===>  lz4 frame tests "

 if [ $LZ4MODE -eq 1 ]; then
    ./datagen > tmp
@@ -542,7 +589,7 @@ else
    $ECHO "lz4 mode not supported"
 fi

-$ECHO "\n**** zstd round-trip tests **** "
+$ECHO "\n===>  zstd round-trip tests "

 roundTripTest
 roundTripTest -g15K       # TableID==3
@@ -555,7 +602,7 @@ roundTripTest -g516K 19   # btopt

 fileRoundTripTest -g500K

-$ECHO "\n**** zstd long distance matching round-trip tests **** "
+$ECHO "\n===>  zstd long distance matching round-trip tests "
 roundTripTest -g0 "2 --long"
 roundTripTest -g1000K "1 --long"
 roundTripTest -g517K "6 --long"
@@ -566,61 +613,56 @@ fileRoundTripTest -g5M "3 --long"

 if [ -n "$hasMT" ]
 then
-    $ECHO "\n**** zstdmt round-trip tests **** "
+    $ECHO "\n===>  zstdmt round-trip tests "
    roundTripTest -g4M "1 -T0"
    roundTripTest -g8M "3 -T2"
    roundTripTest -g8000K "2 --threads=2"
    fileRoundTripTest -g4M "19 -T2 -B1M"

-    $ECHO "\n**** zstdmt long distance matching round-trip tests **** "
+    $ECHO "\n===>  zstdmt long distance matching round-trip tests "
    roundTripTest -g8M "3 --long -T2"
 else
-    $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+    $ECHO "\n===>  no multithreading, skipping zstdmt tests "
 fi

 rm tmp*

-$ECHO "\n**** zstd --list/-l single frame tests ****"
+$ECHO "\n===>  zstd --list/-l single frame tests "
 ./datagen > tmp1
 ./datagen > tmp2
 ./datagen > tmp3
-./datagen > tmp4
 $ZSTD tmp*
 $ZSTD -l *.zst
-$ZSTD -lv *.zst
+$ZSTD -lv *.zst | grep "Decompressed Size:"  # check that decompressed size is present in header
 $ZSTD --list *.zst
 $ZSTD --list -v *.zst

-$ECHO "\n**** zstd --list/-l multiple frame tests ****"
+$ECHO "\n===>  zstd --list/-l multiple frame tests "
 cat tmp1.zst tmp2.zst > tmp12.zst
-cat tmp3.zst tmp4.zst > tmp34.zst
-cat tmp12.zst tmp34.zst > tmp1234.zst
-cat tmp12.zst tmp4.zst > tmp124.zst
+cat tmp12.zst tmp3.zst > tmp123.zst
 $ZSTD -l *.zst
 $ZSTD -lv *.zst
-$ZSTD --list *.zst
-$ZSTD --list -v *.zst

-$ECHO "\n**** zstd --list/-l error detection tests ****"
+$ECHO "\n===>  zstd --list/-l error detection tests "
 ! $ZSTD -l tmp1 tmp1.zst
 ! $ZSTD --list tmp*
 ! $ZSTD -lv tmp1*
-! $ZSTD --list -v tmp2 tmp23.zst
+! $ZSTD --list -v tmp2 tmp12.zst

-$ECHO "\n**** zstd --list/-l test with null files ****"
+$ECHO "\n===>  zstd --list/-l test with null files "
 ./datagen -g0 > tmp5
 $ZSTD tmp5
 $ZSTD -l tmp5.zst
 ! $ZSTD -l tmp5*
-$ZSTD -lv tmp5.zst
+$ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)"  # check that 0 size is present in header
 ! $ZSTD -lv tmp5*

-$ECHO "\n**** zstd --list/-l test with no content size field ****"
-./datagen -g1MB | $ZSTD > tmp6.zst
+$ECHO "\n===>  zstd --list/-l test with no content size field "
+./datagen -g513K | $ZSTD > tmp6.zst
 $ZSTD -l tmp6.zst
-$ZSTD -lv tmp6.zst
+! $ZSTD -lv tmp6.zst | grep "Decompressed Size:"  # must NOT be present in header

-$ECHO "\n**** zstd --list/-l test with no checksum ****"
+$ECHO "\n===>   zstd --list/-l test with no checksum "
 $ZSTD -f --no-check tmp1
 $ZSTD -l tmp1.zst
 $ZSTD -lv tmp1.zst
@@ -628,57 +670,70 @@ $ZSTD -lv tmp1.zst
 rm tmp*


+$ECHO "\n===>   zstd long distance matching tests "
+roundTripTest -g0 " --long"
+roundTripTest -g9M "2 --long"
+# Test parameter parsing
+roundTripTest -g1M -P50 "1 --long=29" " --memory=512MB"
+roundTripTest -g1M -P50 "1 --long=29 --zstd=wlog=28" " --memory=256MB"
+roundTripTest -g1M -P50 "1 --long=29" " --long=28 --memory=512MB"
+roundTripTest -g1M -P50 "1 --long=29" " --zstd=wlog=28 --memory=512MB"
+
+
 if [ "$1" != "--test-large-data" ]; then
    $ECHO "Skipping large data tests"
    exit 0
 fi

+$ECHO "\n===>   large files tests "
+
 roundTripTest -g270000000 1
-roundTripTest -g270000000 2
-roundTripTest -g270000000 3
+roundTripTest -g250000000 2
+roundTripTest -g230000000 3

 roundTripTest -g140000000 -P60 4
-roundTripTest -g140000000 -P60 5
-roundTripTest -g140000000 -P60 6
+roundTripTest -g130000000 -P62 5
+roundTripTest -g120000000 -P65 6

 roundTripTest -g70000000 -P70 7
-roundTripTest -g70000000 -P70 8
-roundTripTest -g70000000 -P70 9
+roundTripTest -g60000000 -P71 8
+roundTripTest -g50000000 -P73 9

 roundTripTest -g35000000 -P75 10
-roundTripTest -g35000000 -P75 11
-roundTripTest -g35000000 -P75 12
+roundTripTest -g30000000 -P76 11
+roundTripTest -g25000000 -P78 12

 roundTripTest -g18000013 -P80 13
 roundTripTest -g18000014 -P80 14
-roundTripTest -g18000015 -P80 15
-roundTripTest -g18000016 -P80 16
-roundTripTest -g18000017 -P80 17
+roundTripTest -g18000015 -P81 15
+roundTripTest -g18000016 -P84 16
+roundTripTest -g18000017 -P88 17
 roundTripTest -g18000018 -P94 18
-roundTripTest -g18000019 -P94 19
+roundTripTest -g18000019 -P96 19

-roundTripTest -g68000020 -P99 20
-roundTripTest -g6000000000 -P99 1
+roundTripTest -g5000000000 -P99 1

 fileRoundTripTest -g4193M -P99 1

-$ECHO "\n**** zstd long, long distance matching round-trip tests **** "
-roundTripTest -g0 "2 --long"
+
+$ECHO "\n===>   zstd long, long distance matching round-trip tests "
 roundTripTest -g270000000 "1 --long"
-roundTripTest -g140000000 -P60 "5 --long"
-roundTripTest -g70000000 -P70 "8 --long"
+roundTripTest -g130000000 -P60 "5 --long"
+roundTripTest -g35000000 -P70 "8 --long"
 roundTripTest -g18000001 -P80  "18 --long"
-fileRoundTripTest -g4100M -P99 "1 --long"
+# Test large window logs
+roundTripTest -g700M -P50 "1 --long=29"
+roundTripTest -g600M -P50 "1 --long --zstd=wlog=29,clog=28"


 if [ -n "$hasMT" ]
 then
-    $ECHO "\n**** zstdmt long round-trip tests **** "
-    roundTripTest -g99000000 -P99 "20 -T2"
-    roundTripTest -g6000000000 -P99 "1 -T2"
-    roundTripTest -g1500000000 -P97 "1 -T999"
-    fileRoundTripTest -g4195M -P98 " -T0"
-    roundTripTest -g1500000000 -P97 "1 --long -T999"
+    $ECHO "\n===>   zstdmt long round-trip tests "
+    roundTripTest -g80000000 -P99 "19 -T2" " "
+    roundTripTest -g5000000000 -P99 "1 -T2" " "
+    roundTripTest -g500000000 -P97 "1 -T999" " "
+    fileRoundTripTest -g4103M -P98 " -T0" " "
+    roundTripTest -g400000000 -P97 "1 --long=24 -T2" " "
 else
    $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
 fi
@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "seqgen.h"
+#include "mem.h"
+#include <string.h>
+
+#define MIN(a, b)  ((a) < (b) ? (a) : (b))
+
+static const size_t kMatchBytes = 128;
+
+#define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static BYTE SEQ_randByte(U32* src)
+{
+    static const U32 prime1 = 2654435761U;
+    static const U32 prime2 = 2246822519U;
+    U32 rand32 = *src;
+    rand32 *= prime1;
+    rand32 ^= prime2;
+    rand32  = SEQ_rotl32(rand32, 13);
+    *src = rand32;
+    return (BYTE)(rand32 >> 5);
+}
+
+SEQ_stream SEQ_initStream(unsigned seed)
+{
+    SEQ_stream stream;
+    stream.state = 0;
+    XXH64_reset(&stream.xxh, 0);
+    stream.seed = seed;
+    return stream;
+}
+
+/* Generates a single guard byte, then match length + 1 of a different byte,
+ * then another guard byte.
+ */
+static size_t SEQ_gen_matchLength(SEQ_stream* stream, unsigned value,
+                                  SEQ_outBuffer* out)
+{
+    typedef enum {
+        ml_first_byte = 0,
+        ml_match_bytes,
+        ml_last_byte,
+    } ml_state;
+    BYTE* const ostart = (BYTE*)out->dst;
+    BYTE* const oend = ostart + out->size;
+    BYTE* op = ostart + out->pos;
+
+    switch ((ml_state)stream->state) {
+    case ml_first_byte:
+        /* Generate a single byte and pick a different byte for the match */
+        if (op >= oend) {
+            stream->bytesLeft = 1;
+            break;
+        }
+        *op = SEQ_randByte(&stream->seed) & 0xFF;
+        do {
+            stream->saved = SEQ_randByte(&stream->seed) & 0xFF;
+        } while (*op == stream->saved);
+        ++op;
+        /* State transition */
+        stream->state = ml_match_bytes;
+        stream->bytesLeft = value + 1;
+    /* fall-through */
+    case ml_match_bytes: {
+        /* Copy matchLength + 1 bytes to the output buffer */
+        size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
+        if (setLength > 0) {
+            memset(op, stream->saved, setLength);
+            op += setLength;
+            stream->bytesLeft -= setLength;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = ml_last_byte;
+    }
+    /* fall-through */
+    case ml_last_byte:
+        /* Generate a single byte and pick a different byte for the match */
+        if (op >= oend) {
+            stream->bytesLeft = 1;
+            break;
+        }
+        do {
+            *op = SEQ_randByte(&stream->seed) & 0xFF;
+        } while (*op == stream->saved);
+        ++op;
+        /* State transition */
+    /* fall-through */
+    default:
+        stream->state = 0;
+        stream->bytesLeft = 0;
+        break;
+    }
+    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+    out->pos = op - ostart;
+    return stream->bytesLeft;
+}
+
+/* Saves the current seed then generates kMatchBytes random bytes >= 128.
+ * Generates literal length - kMatchBytes random bytes < 128.
+ * Generates another kMatchBytes using the saved seed to generate a match.
+ * This way the match is easy to find for the compressors.
+ */
+static size_t SEQ_gen_litLength(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
+{
+    typedef enum {
+        ll_start = 0,
+        ll_run_bytes,
+        ll_literals,
+        ll_run_match,
+    } ll_state;
+    BYTE* const ostart = (BYTE*)out->dst;
+    BYTE* const oend = ostart + out->size;
+    BYTE* op = ostart + out->pos;
+
+    switch ((ll_state)stream->state) {
+    case ll_start:
+        stream->state = ll_run_bytes;
+        stream->saved = stream->seed;
+        stream->bytesLeft = MIN(kMatchBytes, value);
+    /* fall-through */
+    case ll_run_bytes:
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->seed) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = ll_literals;
+        stream->bytesLeft = value - MIN(kMatchBytes, value);
+    /* fall-through */
+    case ll_literals:
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->seed) & 0x7F;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = ll_run_match;
+        stream->bytesLeft = MIN(kMatchBytes, value);
+    /* fall-through */
+    case ll_run_match: {
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->saved) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+    }
+    /* fall-through */
+    default:
+        stream->state = 0;
+        stream->bytesLeft = 0;
+        break;
+    }
+    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+    out->pos = op - ostart;
+    return stream->bytesLeft;
+}
+
+/* Saves the current seed then generates kMatchBytes random bytes >= 128.
+ * Generates offset - kMatchBytes of zeros to get a large offset without
+ * polluting the hash tables.
+ * Generates another kMatchBytes using the saved seed to generate a with the
+ * required offset.
+ */
+static size_t SEQ_gen_offset(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
+{
+    typedef enum {
+        of_start = 0,
+        of_run_bytes,
+        of_offset,
+        of_run_match,
+    } of_state;
+    BYTE* const ostart = (BYTE*)out->dst;
+    BYTE* const oend = ostart + out->size;
+    BYTE* op = ostart + out->pos;
+
+    switch ((of_state)stream->state) {
+    case of_start:
+        stream->state = of_run_bytes;
+        stream->saved = stream->seed;
+        stream->bytesLeft = MIN(value, kMatchBytes);
+    /* fall-through */
+    case of_run_bytes: {
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->seed) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = of_offset;
+        stream->bytesLeft = value - MIN(value, kMatchBytes);
+    }
+    /* fall-through */
+    case of_offset: {
+        /* Copy matchLength + 1 bytes to the output buffer */
+        size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
+        if (setLength > 0) {
+            memset(op, 0, setLength);
+            op += setLength;
+            stream->bytesLeft -= setLength;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = of_run_match;
+        stream->bytesLeft = MIN(value, kMatchBytes);
+    }
+    /* fall-through */
+    case of_run_match: {
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->saved) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+    }
+    /* fall-through */
+    default:
+        stream->state = 0;
+        stream->bytesLeft = 0;
+        break;
+    }
+    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+    out->pos = op - ostart;
+    return stream->bytesLeft;
+}
+
+/* Returns the number of bytes left to generate.
+ * Must pass the same type/value until it returns 0.
+ */
+size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, SEQ_outBuffer* out)
+{
+    switch (type) {
+        case SEQ_gen_ml: return SEQ_gen_matchLength(stream, value, out);
+        case SEQ_gen_ll: return SEQ_gen_litLength(stream, value, out);
+        case SEQ_gen_of: return SEQ_gen_offset(stream, value, out);
+        case SEQ_gen_max: /* fall-through */
+        default: return 0;
+    }
+}
+
+/* Returns the xxhash of the data produced so far */
+XXH64_hash_t SEQ_digest(SEQ_stream const* stream)
+{
+    return XXH64_digest(&stream->xxh);
+}
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef SEQGEN_H
+#define SEQGEN_H
+
+#define XXH_STATIC_LINKING_ONLY
+
+#include "xxhash.h"
+#include <stddef.h>   /* size_t */
+
+typedef enum {
+    SEQ_gen_ml = 0,
+    SEQ_gen_ll,
+    SEQ_gen_of,
+    SEQ_gen_max /* Must be the last value */
+} SEQ_gen_type;
+
+/* Internal state, do not use */
+typedef struct {
+    XXH64_state_t xxh; /* xxh state for all the data produced so far (seed=0) */
+    unsigned seed;
+    int state; /* enum to control state machine (clean=0) */
+    unsigned saved;
+    size_t bytesLeft;
+} SEQ_stream;
+
+SEQ_stream SEQ_initStream(unsigned seed);
+
+typedef struct {
+    void* dst;
+    size_t size;
+    size_t pos;
+} SEQ_outBuffer;
+
+/* Returns non-zero until the current type/value has been generated.
+ * Must pass the same type/value until it returns 0.
+ *
+ * Recommended to pick a value in the middle of the range you want, since there
+ * may be some noise that causes actual results to be slightly different.
+ * We try to be more accurate for smaller values.
+ *
+ * NOTE: Very small values don't work well (< 6).
+ */
+size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value,
+               SEQ_outBuffer* out);
+
+/* Returns the xxhash of the data produced so far */
+XXH64_hash_t SEQ_digest(SEQ_stream const* stream);
+
+#endif /* SEQGEN_H */
@@ -24,7 +24,6 @@
 **************************************/
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
-#include <time.h>         /* clock_t, clock() */
 #include <string.h>       /* strcmp */
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
@@ -34,6 +33,7 @@
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"       /* XXH64_* */
+#include "util.h"


 /*-************************************
@@ -58,26 +58,24 @@ static const U32 prime2 = 2246822519U;
 #define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static U32 g_displayLevel = 2;

-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
-            { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stderr); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_displayClock = 0;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

-static clock_t g_clockTime = 0;
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stderr); } }
+
+static U64 g_clockTime = 0;


 /*-*******************************************************
 *  Fuzzer functions
 *********************************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b)?(a):(b))
 #define MAX(a,b) ((a)>(b)?(a):(b))
-
-static clock_t FUZ_GetClockSpan(clock_t clockStart)
-{
-    return clock() - clockStart;  /* works even when overflow. Max span ~ 30 mn */
-}
-
 /*! FUZ_rand() :
    @return : a 27 bits random value, from a 32-bits `seed`.
    `seed` is also modified */
@@ -256,8 +254,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
    return FUZ_rLogLength(seed, logLength);
 }

-#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
-
 #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
                         DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }

@@ -278,7 +274,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
    U32 coreSeed = seed;
    ZBUFF_CCtx* zc;
    ZBUFF_DCtx* zd;
-    clock_t startClock = clock();
+    UTIL_time_t startClock = UTIL_getTime();

    /* allocations */
    zc = ZBUFF_createCCtx();
@@ -308,7 +304,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
        FUZ_rand(&coreSeed);

    /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
        U32 lseed;
        const BYTE* srcBuffer;
        const BYTE* dict;
@@ -357,7 +353,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
            {   ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
                params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
-                {   size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
+                {   size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, ZSTD_CONTENTSIZE_UNKNOWN);
                    CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
        }   }   }

@@ -548,7 +544,7 @@ int main(int argc, const char** argv)
                    }
                    if (*argument=='m') g_clockTime *=60, argument++;
                    if (*argument=='n') argument++;
-                    g_clockTime *= CLOCKS_PER_SEC;
+                    g_clockTime *= SEC_TO_MICRO;
                    break;

                case 's':
@@ -24,7 +24,6 @@
 **************************************/
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
-#include <time.h>         /* clock_t, clock() */
 #include <string.h>       /* strcmp */
 #include <assert.h>       /* assert */
 #include "mem.h"
@@ -36,6 +35,8 @@
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"       /* XXH64_* */
+#include "seqgen.h"
+#include "util.h"


 /*-************************************
@@ -61,26 +62,24 @@ static const U32 prime32 = 2654435761U;
                                  if (g_displayLevel>=4) fflush(stderr); }
 static U32 g_displayLevel = 2;

-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
-            { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
-              if (g_displayLevel>=4) fflush(stderr); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
-static clock_t g_displayClock = 0;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

-static clock_t g_clockTime = 0;
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stderr); } }
+
+static U64 g_clockTime = 0;


 /*-*******************************************************
 *  Fuzzer functions
 *********************************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b)?(a):(b))
 #define MAX(a,b) ((a)>(b)?(a):(b))
-
-static clock_t FUZ_GetClockSpan(clock_t clockStart)
-{
-    return clock() - clockStart;  /* works even when overflow. Max span ~ 30 mn */
-}
-
 /*! FUZ_rand() :
    @return : a 27 bits random value, from a 32-bits `seed`.
    `seed` is also modified */
@@ -96,6 +95,21 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
    return rand32 >> 5;
 }

+#define CHECK(cond, ...) {                                   \
+    if (cond) {                                              \
+        DISPLAY("Error => ");                                \
+        DISPLAY(__VA_ARGS__);                                \
+        DISPLAY(" (seed %u, test nb %u, line %u)  \n",       \
+                seed, testNb, __LINE__);                     \
+        goto _output_error;                                  \
+}   }
+
+#define CHECK_Z(f) {                                         \
+    size_t const err = f;                                    \
+    CHECK(ZSTD_isError(err), "%s : %s ",                     \
+          #f, ZSTD_getErrorName(err));                       \
+}
+

 /*======================================================
 *   Basic Unit tests
@@ -135,12 +149,69 @@ static void FUZ_freeDictionary(buffer_t dict)
    free(dict.start);
 }

+/* Round trips data and updates xxh with the decompressed data produced */
+static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+                            XXH64_state_t* xxh, void* data, size_t size,
+                            ZSTD_EndDirective endOp)
+{
+    static BYTE compressed[1024];
+    static BYTE uncompressed[1024];

-static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
+    ZSTD_inBuffer cin = {data, size, 0};
+    size_t cret;
+
+    do {
+        ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0};
+        ZSTD_inBuffer din = {compressed, 0, 0};
+        ZSTD_outBuffer dout = {uncompressed, 0, 0};
+
+        cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp);
+        if (ZSTD_isError(cret))
+            return cret;
+
+        din.size = cout.pos;
+        while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
+            size_t dret;
+
+            dout.pos = 0;
+            dout.size = sizeof(uncompressed);
+            dret = ZSTD_decompressStream(dctx, &dout, &din);
+            if (ZSTD_isError(dret))
+                return dret;
+            XXH64_update(xxh, dout.dst, dout.pos);
+            if (dret == 0)
+                break;
+        }
+    } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
+    return 0;
+}
+
+/* Generates some data and round trips it */
+static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+                                    XXH64_state_t* xxh, SEQ_stream* seq,
+                                    SEQ_gen_type type, unsigned value)
+{
+    static BYTE data[1024];
+    size_t gen;
+
+    do {
+        SEQ_outBuffer sout = {data, sizeof(data), 0};
+        size_t ret;
+        gen = SEQ_gen(seq, type, value, &sout);
+
+        ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
+        if (ZSTD_isError(ret))
+            return ret;
+    } while (gen != 0);
+
+    return 0;
+}
+
+static int basicUnitTests(U32 seed, double compressibility)
 {
    size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
    void* CNBuffer = malloc(CNBufferSize);
-    size_t const skippableFrameSize = 11;
+    size_t const skippableFrameSize = 200 KB;
    size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
    void* compressedBuffer = malloc(compressedBufferSize);
    size_t const decodedBufferSize = CNBufferSize;
@@ -148,8 +219,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    size_t cSize;
    int testResult = 0;
    U32 testNb = 1;
-    ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
-    ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
+    ZSTD_CStream* zc = ZSTD_createCStream();
+    ZSTD_DStream* zd = ZSTD_createDStream();
    ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);

    ZSTD_inBuffer  inBuff, inBuff2;
@@ -166,31 +237,45 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);

    /* Create dictionary */
-    MEM_STATIC_ASSERT(COMPRESSIBLE_NOISE_LENGTH >= 4 MB);
-    dictionary = FUZ_createDictionary(CNBuffer, 4 MB, 4 KB, 40 KB);
+    DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
+    dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 2, 8 KB, 40 KB);
    if (!dictionary.start) {
        DISPLAY("Error creating dictionary, aborting \n");
        goto _output_error;
    }
    dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);

-    /* generate skippable frame */
-    MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
-    MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
-    cSize = skippableFrameSize + 8;
-
    /* Basic compression test */
    DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
-    { size_t const r = ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1);
-      if (ZSTD_isError(r)) goto _output_error; }
-    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
+    outBuff.dst = (char*)(compressedBuffer);
    outBuff.size = compressedBufferSize;
    outBuff.pos = 0;
    inBuff.src = CNBuffer;
    inBuff.size = CNBufferSize;
    inBuff.pos = 0;
-    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
-      if (ZSTD_isError(r)) goto _output_error; }
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    { size_t const r = ZSTD_endStream(zc, &outBuff);
+      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+    DISPLAYLEVEL(3, "OK (%u bytes)\n", (U32)outBuff.pos);
+
+    /* generate skippable frame */
+    MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
+    MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
+    cSize = skippableFrameSize + 8;
+
+    /* Basic compression test using dict */
+    DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
+    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    assert(compressedBufferSize > cSize);
+    outBuff.size = compressedBufferSize - cSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
    { size_t const r = ZSTD_endStream(zc, &outBuff);
      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
@@ -200,11 +285,11 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    /* context size functions */
    DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
-        size_t const s = ZSTD_estimateCStreamSize_advanced_usingCParams(cParams)
-                        /* uses ZSTD_initCStream_usingDict() */
-                       + ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
-            if (ZSTD_isError(s)) goto _output_error;
-            DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
+        size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
+        size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
+        if (ZSTD_isError(cstreamSize)) goto _output_error;
+        if (ZSTD_isError(cdictSize)) goto _output_error;
+        DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)(cstreamSize + cdictSize));
    }

    DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
@@ -225,8 +310,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo

    /* skippable frame test */
    DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
-    if (ZSTD_isError( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) ))
-        goto _output_error;
+    CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
    inBuff.src = compressedBuffer;
    inBuff.size = cSize;
    inBuff.pos = 0;
@@ -244,8 +328,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    inBuff2 = inBuff;
    DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
    ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
-    { size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000);  /* large limit */
-      if (ZSTD_isError(r)) goto _output_error; }
+    CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000) );  /* large limit */
    { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
      if (remaining != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
    if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
@@ -335,8 +418,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    inBuff.src = CNBuffer;
    inBuff.size = CNBufferSize;
    inBuff.pos = 0;
-    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
-      if (ZSTD_isError(r)) goto _output_error; }
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
    { size_t const r = ZSTD_endStream(zc, &outBuff);
      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
@@ -353,8 +435,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    inBuff.src = CNBuffer;
    inBuff.size = CNBufferSize;
    inBuff.pos = 0;
-    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
-      if (ZSTD_isError(r)) goto _output_error; }
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
    { size_t const r = ZSTD_endStream(zc, &outBuff);
      if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error;    /* must fail : wrong srcSize */
@@ -363,7 +444,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    /* Complex context re-use scenario */
    DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
    ZSTD_freeCStream(zc);
-    zc = ZSTD_createCStream_advanced(customMem);
+    zc = ZSTD_createCStream();
    if (zc==NULL) goto _output_error;   /* memory allocation issue */
    /* use 1 */
    {   size_t const inSize = 513;
@@ -376,8 +457,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
        outBuff.size = ZSTD_compressBound(inSize);
        outBuff.pos = 0;
        DISPLAYLEVEL(5, "compress1 ");
-        { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
-            if (ZSTD_isError(r)) goto _output_error; }
+        CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
        DISPLAYLEVEL(5, "end1 ");
        { size_t const r = ZSTD_endStream(zc, &outBuff);
@@ -394,8 +474,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
        outBuff.size = ZSTD_compressBound(inSize);
        outBuff.pos = 0;
        DISPLAYLEVEL(5, "compress2 ");
-        { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
-            if (ZSTD_isError(r)) goto _output_error; }
+        CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
        DISPLAYLEVEL(5, "end2 ");
        { size_t const r = ZSTD_endStream(zc, &outBuff);
@@ -407,6 +486,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
    {   ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
        size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
+        DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
        if (ZSTD_isError(initError)) goto _output_error;
        cSize = 0;
        outBuff.dst = compressedBuffer;
@@ -415,11 +495,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
        inBuff.src = CNBuffer;
        inBuff.size = CNBufferSize;
        inBuff.pos = 0;
-        { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
-          if (ZSTD_isError(r)) goto _output_error; }
+        DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
+        CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
-        { size_t const r = ZSTD_endStream(zc, &outBuff);
-          if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+        {   size_t const r = ZSTD_endStream(zc, &outBuff);
+            DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (U32)r);
+            if (r != 0) goto _output_error;  /* error, or some data not flushed */
+        }
        cSize = outBuff.pos;
        ZSTD_freeCDict(cdict);
        DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -442,12 +524,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    {   ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
        size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
        if (ZSTD_isError(initError)) goto _output_error;
-        inBuff.src = compressedBuffer;
-        inBuff.size = cSize;
-        inBuff.pos = 0;
        outBuff.dst = decodedBuffer;
        outBuff.size = CNBufferSize;
        outBuff.pos = 0;
+        inBuff.src = compressedBuffer;
+        inBuff.size = cSize;
+        inBuff.pos = 0;
        { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
          if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
        if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
@@ -465,14 +547,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    /* Memory restriction */
    DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
    ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
-    { size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000);  /* too small limit */
-      if (ZSTD_isError(r)) goto _output_error; }
-    inBuff.src = compressedBuffer;
-    inBuff.size = cSize;
-    inBuff.pos = 0;
+    CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) );  /* too small limit */
    outBuff.dst = decodedBuffer;
    outBuff.size = CNBufferSize;
    outBuff.pos = 0;
+    inBuff.src = compressedBuffer;
+    inBuff.size = cSize;
+    inBuff.pos = 0;
    { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
      if (!ZSTD_isError(r)) goto _output_error;  /* must fail : frame requires > 100 bytes */
      DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
@@ -480,7 +561,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
        ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
-        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, customMem);
+        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
        size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
        if (ZSTD_isError(initError)) goto _output_error;
        cSize = 0;
@@ -490,8 +571,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
        inBuff.src = CNBuffer;
        inBuff.size = CNBufferSize;
        inBuff.pos = 0;
-        { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
-          if (ZSTD_isError(r)) goto _output_error; }
+        CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
        if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
        { size_t const r = ZSTD_endStream(zc, &outBuff);
          if (r != 0) goto _output_error; }  /* error, or some data not flushed */
@@ -513,16 +593,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    }

    DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
-    { size_t const refErr = ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled);
-      if (ZSTD_isError(refErr)) goto _output_error; }
+    CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
    outBuff.dst = compressedBuffer;
    outBuff.size = compressedBufferSize;
    outBuff.pos = 0;
    inBuff.src = CNBuffer;
    inBuff.size = CNBufferSize;
    inBuff.pos = 0;
-    { size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
-      if (ZSTD_isError(r)) goto _output_error; }
+    CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
    if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
    cSize = outBuff.pos;
    DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -549,31 +627,28 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    inBuff.src = CNBuffer;
    inBuff.size = CNBufferSize;
    inBuff.pos = 0;
-    { size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
-      if (ZSTD_isError(r)) goto _output_error; }
+    CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
    if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
    cSize = outBuff.pos;
    DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);

    DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
-    {   size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
-        if (ZSTD_isError(r)) goto _output_error;  /* must fail : dictionary not used */
-        DISPLAYLEVEL(3, "OK \n");
-    }
+    CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
+    DISPLAYLEVEL(3, "OK \n");

    /* Empty srcSize */
    DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
    {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
        params.fParams.contentSizeFlag = 1;
-        ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0);
+        CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
    } /* cstream advanced shall write content size = 0 */
-    inBuff.src = CNBuffer;
-    inBuff.size = 0;
-    inBuff.pos = 0;
    outBuff.dst = compressedBuffer;
    outBuff.size = compressedBufferSize;
    outBuff.pos = 0;
-    if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
+    inBuff.src = CNBuffer;
+    inBuff.size = 0;
+    inBuff.pos = 0;
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
    if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
    cSize = outBuff.pos;
    if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
@@ -582,7 +657,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
    {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
        params.fParams.contentSizeFlag = 1;
-        ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
+        CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
    } /* cstream advanced shall write content size = 0 */
    inBuff.src = CNBuffer;
    inBuff.size = 0;
@@ -590,19 +665,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    outBuff.dst = compressedBuffer;
    outBuff.size = compressedBufferSize;
    outBuff.pos = 0;
-    if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
    if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
    cSize = outBuff.pos;
    if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;

    ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
-    inBuff.src = CNBuffer;
-    inBuff.size = 0;
-    inBuff.pos = 0;
    outBuff.dst = compressedBuffer;
    outBuff.size = compressedBufferSize;
    outBuff.pos = 0;
-    if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
+    inBuff.src = CNBuffer;
+    inBuff.size = 0;
+    inBuff.pos = 0;
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
    if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
    cSize = outBuff.pos;
    if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
@@ -610,22 +685,123 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo

    /* Basic multithreading compression test */
    DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
-    { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
-      size_t const r = ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize);
-      if (ZSTD_isError(r)) goto _output_error; }
-    outBuff.dst = (char*)(compressedBuffer);
+    {   ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
+        CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) );
+    }
+    outBuff.dst = compressedBuffer;
    outBuff.size = compressedBufferSize;
    outBuff.pos = 0;
    inBuff.src = CNBuffer;
    inBuff.size = CNBufferSize;
    inBuff.pos = 0;
-    { size_t const r = ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end);
-      if (ZSTD_isError(r)) goto _output_error; }
+    CHECK_Z( ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end) );
    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
    { size_t const r = ZSTDMT_endStream(mtctx, &outBuff);
      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
    DISPLAYLEVEL(3, "OK \n");

+    /* Complex multithreading + dictionary test */
+    {   U32 const nbThreads = 2;
+        size_t const jobSize = 4 * 1 MB;
+        size_t const srcSize = jobSize * nbThreads;  /* we want each job to have predictable size */
+        size_t const segLength = 2 KB;
+        size_t const offset = 600 KB;   /* must be larger than window defined in cdict */
+        size_t const start = jobSize + (offset-1);
+        const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
+        BYTE* const dst = (BYTE*)CNBuffer + start - offset;
+        DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize);
+        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) );
+        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, 2) );
+        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) );
+        assert(start > offset);
+        assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
+        memcpy(dst, srcToCopy, segLength);   /* create a long repetition at long distance for job 2 */
+        outBuff.dst = compressedBuffer;
+        outBuff.size = compressedBufferSize;
+        outBuff.pos = 0;
+        inBuff.src = CNBuffer;
+        inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
+        inBuff.pos = 0;
+    }
+    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled);   /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
+        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+        DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
+        CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
+        CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
+        CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );  /* do not keep a reference to cdict, as its lifetime ends */
+        ZSTD_freeCDict(cdict);
+    }
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    cSize = outBuff.pos;
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
+    {   ZSTD_DStream* const dstream = ZSTD_createDCtx();
+        ZSTD_frameHeader zfh;
+        ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
+        DISPLAYLEVEL(5, "frame windowsize = %u : ", (U32)zfh.windowSize);
+        outBuff.dst = decodedBuffer;
+        outBuff.size = CNBufferSize;
+        outBuff.pos = 0;
+        inBuff.src = compressedBuffer;
+        inBuff.pos = 0;
+        CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
+        inBuff.size = 1;  /* avoid shortcut to single-pass mode */
+        CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
+        inBuff.size = cSize;
+        CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
+        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+        ZSTD_freeDStream(dstream);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
+    {   unsigned const kMaxWindowLog = 24;
+        unsigned value;
+        ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
+        ZSTD_CDict* cdict;
+        ZSTD_DDict* ddict;
+        SEQ_stream seq = SEQ_initStream(0x87654321);
+        SEQ_gen_type type;
+        XXH64_state_t xxh;
+
+        XXH64_reset(&xxh, 0);
+        cParams.windowLog = kMaxWindowLog;
+        cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+        ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
+
+        if (!cdict || !ddict) goto _output_error;
+
+        ZSTD_CCtx_reset(zc);
+        ZSTD_resetDStream(zd);
+        CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
+        CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
+        CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog));
+        /* Test all values < 300 */
+        for (value = 0; value < 300; ++value) {
+            for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
+                CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
+            }
+        }
+        /* Test values 2^8 to 2^17 */
+        for (value = (1 << 8); value < (1 << 17); value <<= 1) {
+            for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
+                CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
+                CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
+            }
+        }
+        /* Test offset values up to the max window log */
+        for (value = 8; value <= kMaxWindowLog; ++value) {
+            CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
+        }
+
+        CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
+        CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
+
+        ZSTD_freeCDict(cdict);
+        ZSTD_freeDDict(ddict);
+    }
+    DISPLAYLEVEL(3, "OK \n");

    /* Overlen overwriting window data bug */
    DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
@@ -641,9 +817,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
            "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
            "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
            "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
-        ZSTD_DStream* zds = ZSTD_createDStream();
+        ZSTD_DStream* const zds = ZSTD_createDStream();
+        if (zds==NULL) goto _output_error;

-        ZSTD_initDStream(zds);
+        CHECK_Z( ZSTD_initDStream(zds) );
        inBuff.src = testCase;
        inBuff.size = 47;
        inBuff.pos = 0;
@@ -652,9 +829,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
        outBuff.pos = 0;

        while (inBuff.pos < inBuff.size) {
-            size_t const r = ZSTD_decompressStream(zds, &outBuff, &inBuff);
-            /* Bug will cause checksum to fail */
-            if (ZSTD_isError(r)) goto _output_error;
+            CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
        }

        ZSTD_freeDStream(zds);
@@ -708,8 +883,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
    return FUZ_rLogLength(seed, logLength);
 }

-#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
-
 /* Return value in range minVal <= v <= maxVal */
 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
 {
@@ -717,43 +890,26 @@ static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
    return (U32)((FUZ_rand(seed) % mod) + minVal);
 }

-#define CHECK(cond, ...) {                                   \
-    if (cond) {                                              \
-        DISPLAY("Error => ");                                \
-        DISPLAY(__VA_ARGS__);                                \
-        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); \
-        goto _output_error;                                  \
-}   }
-
-#define CHECK_Z(f) {                                         \
-    size_t const err = f;                                    \
-    if (ZSTD_isError(err)) {                                 \
-        DISPLAY("Error => %s : %s ",                         \
-                #f, ZSTD_getErrorName(err));                 \
-        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); \
-        goto _output_error;                                  \
-}   }
-
 static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
 {
    U32 const maxSrcLog = bigTests ? 24 : 22;
    static const U32 maxSampleLog = 19;
    size_t const srcBufferSize = (size_t)1<<maxSrcLog;
    BYTE* cNoiseBuffer[5];
-    size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
+    size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
    BYTE*  const copyBuffer = (BYTE*)malloc (copyBufferSize);
-    size_t const cBufferSize   = ZSTD_compressBound(srcBufferSize);
+    size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
    BYTE*  const cBuffer = (BYTE*)malloc (cBufferSize);
    size_t const dstBufferSize = srcBufferSize;
    BYTE*  const dstBuffer = (BYTE*)malloc (dstBufferSize);
    U32 result = 0;
    U32 testNb = 0;
    U32 coreSeed = seed;
-    ZSTD_CStream* zc = ZSTD_createCStream();   /* will be reset sometimes */
-    ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
+    ZSTD_CStream* zc = ZSTD_createCStream();   /* will be re-created sometimes */
+    ZSTD_DStream* zd = ZSTD_createDStream();   /* will be re-created sometimes */
    ZSTD_DStream* const zd_noise = ZSTD_createDStream();
-    clock_t const startClock = clock();
-    const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
+    UTIL_time_t const startClock = UTIL_getTime();
+    const BYTE* dict = NULL;  /* can keep same dict on 2 consecutive tests */
    size_t dictSize = 0;
    U32 oldTestLog = 0;
    U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
@@ -782,7 +938,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
        FUZ_rand(&coreSeed);

    /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
        U32 lseed;
        const BYTE* srcBuffer;
        size_t totalTestSize, totalGenSize, cSize;
@@ -831,8 +987,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
        if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
            && oldTestLog /* at least one test happened */ && resetAllowed) {
            maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
-            if (maxTestSize >= srcBufferSize)
-                maxTestSize = srcBufferSize-1;
+            maxTestSize = MIN(maxTestSize, srcBufferSize-16);
            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
                CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) );
            }
@@ -851,10 +1006,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                dict = srcBuffer + dictStart;
            }
-            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
                params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+                params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
                CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
        }   }

@@ -865,7 +1021,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
            for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
                /* compress random chunks into randomly sized dst buffers */
                {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
-                    size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
+                    size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
                    size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                    size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
@@ -914,7 +1070,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
                inBuff.size = inBuff.pos + readCSrcSize;
-                outBuff.size = inBuff.pos + dstBuffSize;
+                outBuff.size = outBuff.pos + dstBuffSize;
                decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
                    DISPLAY("checksum error : \n");
@@ -998,15 +1154,16 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
    U32 result = 0;
    U32 testNb = 0;
    U32 coreSeed = seed;
-    ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(2);   /* will be reset sometimes */
+    U32 nbThreads = 2;
+    ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads);   /* will be reset sometimes */
    ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
    ZSTD_DStream* const zd_noise = ZSTD_createDStream();
-    clock_t const startClock = clock();
+    UTIL_time_t const startClock = UTIL_getTime();
    const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
    size_t dictSize = 0;
    U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
-    U32 const nbThreadsMax = bigTests ? 5 : 2;
+    int const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
+    U32 const nbThreadsMax = bigTests ? 4 : 2;

    /* allocations */
    cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -1032,7 +1189,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
        FUZ_rand(&coreSeed);

    /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
        U32 lseed;
        const BYTE* srcBuffer;
        size_t totalTestSize, totalGenSize, cSize;
@@ -1042,16 +1199,16 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
        size_t maxTestSize;

        /* init */
-        if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
-        else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
+        if (testNb < nbTests) {
+            DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests);
+        } else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
        FUZ_rand(&coreSeed);
        lseed = coreSeed ^ prime32;

        /* states full reset (deliberately not synchronized) */
        /* some issues can only happen when reusing states */
        if ((FUZ_rand(&lseed) & 0xFF) == 131) {
-            U32 const nbThreadsCandidate = (FUZ_rand(&lseed) % 6) + 1;
-            U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
+            nbThreads = (FUZ_rand(&lseed) % nbThreadsMax) + 1;
            DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads);
            ZSTDMT_freeCCtx(zc);
            zc = ZSTDMT_createCCtx(nbThreads);
@@ -1091,11 +1248,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
        } else {
            U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
-            U32 const cLevelCandidate = (FUZ_rand(&lseed) %
-                               (ZSTD_maxCLevel() -
-                               (MAX(testLog, dictLog) / 3))) +
-                               1;
-            U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
+            int const cLevelCandidate = ( FUZ_rand(&lseed)
+                            % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) )
+                            + 1;
+            int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2;  /* reduce cLevel when multiple threads to reduce memory consumption */
+            int const cLevelMin = MAX(cLevelThreadAdjusted, 1);  /* no negative cLevel yet */
+            int const cLevel = MIN(cLevelMin, cLevelMax);
            maxTestSize = FUZ_rLogLength(&lseed, testLog);
            oldTestLog = testLog;
            /* random dictionary selection */
@@ -1103,17 +1261,17 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                dict = srcBuffer + dictStart;
            }
-            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
-                DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
-                    params.cParams.windowLog, (U32)pledgedSrcSize);
+                DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n",
+                    params.cParams.windowLog, (U32)pledgedSrcSize, (U32)dictSize);
                params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
-                params.fParams.contentSizeFlag = pledgedSrcSize>0;
+                params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
                DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
-                CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
                CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
-                CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );
+                CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );   /* custome job size */
+                CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
        }   }

        /* multi-segments compression test */
@@ -1130,9 +1288,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                    ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
                    outBuff.size = outBuff.pos + dstBuffSize;

-                    DISPLAYLEVEL(5, "Sending %u bytes to compress \n", (U32)srcSize);
+                    DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (U32)srcSize);
                    CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) );
-                    DISPLAYLEVEL(5, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
+                    DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);

                    XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
                    memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
@@ -1178,11 +1336,11 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
                inBuff.size = inBuff.pos + readCSrcSize;
-                outBuff.size = inBuff.pos + dstBuffSize;
-                DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
+                outBuff.size = outBuff.pos + dstBuffSize;
+                DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
                decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
-                DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
+                DISPLAYLEVEL(6, "inBuff.pos = %u \n", (U32)readCSrcSize);
            }
            CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
            CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
@@ -1272,12 +1430,13 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
    ZSTD_CCtx* zc = ZSTD_createCCtx();   /* will be reset sometimes */
    ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
    ZSTD_DStream* const zd_noise = ZSTD_createDStream();
-    clock_t const startClock = clock();
+    UTIL_time_t const startClock = UTIL_getTime();
    const BYTE* dict = NULL;   /* can keep same dict on 2 consecutive tests */
    size_t dictSize = 0;
    U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
-    U32 const nbThreadsMax = bigTests ? 5 : 1;
+    U32 windowLogMalus = 0;   /* can survive between 2 loops */
+    U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
+    U32 const nbThreadsMax = bigTests ? 4 : 2;
    ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();

    /* allocations */
@@ -1304,7 +1463,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
        FUZ_rand(&coreSeed);

    /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
        U32 lseed;
        const BYTE* srcBuffer;
        size_t totalTestSize, totalGenSize, cSize;
@@ -1318,6 +1477,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
        else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
        FUZ_rand(&coreSeed);
        lseed = coreSeed ^ prime32;
+        DISPLAYLEVEL(5, " ***  Test %u  *** \n", testNb);

        /* states full reset (deliberately not synchronized) */
        /* some issues can only happen when reusing states */
@@ -1364,10 +1524,12 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
            U32 const cLevelCandidate = (FUZ_rand(&lseed) %
                               (ZSTD_maxCLevel() -
-                               (MAX(testLog, dictLog) / 3))) +
+                               (MAX(testLog, dictLog) / 2))) +
                               1;
            U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
+            DISPLAYLEVEL(5, "t%u: base cLevel : %u \n", testNb, cLevel);
            maxTestSize = FUZ_rLogLength(&lseed, testLog);
+            DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize);
            oldTestLog = testLog;
            /* random dictionary selection */
            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
@@ -1377,41 +1539,59 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
            }
            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
+                static const U32 windowLogMax = 24;

                /* mess with compression parameters */
                cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
                cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
                cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
                cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
                cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
-                cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
+                cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
                cParams = ZSTD_adjustCParams(cParams, 0, 0);

-                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
+                if (FUZ_rand(&lseed) & 1) {
+                    DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
+                    assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN);   /* guaranteed by ZSTD_adjustCParams() */
+                    windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
+                }
+                if (FUZ_rand(&lseed) & 1) {
+                    DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
+                }
+                if (FUZ_rand(&lseed) & 1) {
+                    DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
+                }
                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );

                /* mess with long distance matching parameters */
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
+                if (bigTests) {
+                    if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
+                }

-                /* unconditionally set, to be sync with decoder */
                /* mess with frame parameters */
                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
-                DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
+                if (FUZ_rand(&lseed) & 1) {
+                    DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
+                    CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
+                }

                /* multi-threading parameters */
                {   U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
-                    U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
+                    U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
+                    U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
+                    DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
                    if (nbThreads > 1) {
                        U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
@@ -1420,10 +1600,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                    }
                }

-                if (FUZ_rand(&lseed) & 1) CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );

                /* Apply parameters */
                if (useOpaqueAPI) {
+                    DISPLAYLEVEL(6," t%u: applying CCtxParams \n", testNb);
                    CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
                }

@@ -1463,8 +1644,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                outBuff.size = outBuff.pos + dstBuffSize;

                CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
-                DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n",
-                    (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
+                DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) \n",
+                    testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));

                XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
                memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
@@ -1478,11 +1659,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
                    outBuff.size = outBuff.pos + adjustedDstSize;
-                    DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);
+                    DISPLAYLEVEL(6, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);
                    remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
-                    CHECK(ZSTD_isError(remainingToFlush),
-                        "ZSTD_compress_generic w/ ZSTD_e_end error : %s",
-                        ZSTD_getErrorName(remainingToFlush) );
+                    CHECK( ZSTD_isError(remainingToFlush),
+                          "ZSTD_compress_generic w/ ZSTD_e_end error : %s",
+                           ZSTD_getErrorName(remainingToFlush) );
            }   }
            crcOrig = XXH64_digest(&xxhState);
            cSize = outBuff.pos;
@@ -1505,12 +1686,12 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
                inBuff.size = inBuff.pos + readCSrcSize;
-                outBuff.size = inBuff.pos + dstBuffSize;
-                DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n",
+                outBuff.size = outBuff.pos + dstBuffSize;
+                DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n",
                            (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize);
                decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
-                DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
+                DISPLAYLEVEL(6, "inBuff.pos = %u \n", (U32)readCSrcSize);
            }
            CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
            CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
@@ -1592,31 +1773,30 @@ typedef enum { simple_api, mt_api, advanced_api } e_api;

 int main(int argc, const char** argv)
 {
-    U32 seed=0;
-    int seedset=0;
-    int argNb;
+    U32 seed = 0;
+    int seedset = 0;
    int nbTests = nbTestsDefault;
    int testNb = 0;
    int proba = FUZ_COMPRESSIBILITY_DEFAULT;
-    int result=0;
+    int result = 0;
    int mainPause = 0;
    int bigTests = (sizeof(size_t) == 8);
    e_api selected_api = simple_api;
    const char* const programName = argv[0];
-    ZSTD_customMem const customNULL = ZSTD_defaultCMem;
    U32 useOpaqueAPI = 0;
+    int argNb;

    /* Check command line */
    for(argNb=1; argNb<argc; argNb++) {
        const char* argument = argv[argNb];
-        if(!argument) continue;   /* Protection if argument empty */
+        assert(argument != NULL);

        /* Parsing commands. Aggregated commands are allowed */
        if (argument[0]=='-') {

-            if (!strcmp(argument, "--mt")) { selected_api=mt_api; continue; }
-            if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; continue; }
-            if (!strcmp(argument, "--opaqueapi")) { selected_api=advanced_api; useOpaqueAPI = 1; continue; }
+            if (!strcmp(argument, "--mt")) { selected_api=mt_api; testNb += !testNb; continue; }
+            if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
+            if (!strcmp(argument, "--opaqueapi")) { selected_api=advanced_api; testNb += !testNb; useOpaqueAPI = 1; continue; }
            if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }

            argument++;
@@ -1659,15 +1839,17 @@ int main(int argc, const char** argv)
                        g_clockTime += *argument - '0';
                        argument++;
                    }
-                    if (*argument=='m') g_clockTime *=60, argument++;
-                    if (*argument=='n') argument++;
-                    g_clockTime *= CLOCKS_PER_SEC;
+                    if (*argument=='m') {    /* -T1m == -T60 */
+                        g_clockTime *=60, argument++;
+                        if (*argument=='n') argument++; /* -T1mn == -T60 */
+                    } else if (*argument=='s') argument++; /* -T10s == -T10 */
+                    g_clockTime *= SEC_TO_MICRO;
                    break;

                case 's':   /* manually select seed */
                    argument++;
-                    seed=0;
                    seedset=1;
+                    seed=0;
                    while ((*argument>='0') && (*argument<='9')) {
                        seed *= 10;
                        seed += *argument - '0';
@@ -1717,7 +1899,7 @@ int main(int argc, const char** argv)
    if (nbTests<=0) nbTests=1;

    if (testNb==0) {
-        result = basicUnitTests(0, ((double)proba) / 100, customNULL);  /* constant seed for predictability */
+        result = basicUnitTests(0, ((double)proba) / 100);  /* constant seed for predictability */
    }

    if (!result) {
@@ -12,7 +12,7 @@ cxx_library(
    deps=[
        '//lib:zstd',
        '//lib:zstd_common',
-    ]
+    ],
 )

 cxx_binary(
@@ -684,6 +684,11 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
            fileSizes[n] = 0;
            continue;
        }
+        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAYLEVEL(2, "Cannot determine size of %s ...    \n", fileNamesTable[n]);
+            fileSizes[n] = 0;
+            continue;
+        }
        f = fopen(fileNamesTable[n], "rb");
        if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
        DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
@@ -714,11 +719,13 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,

    /* Load dictionary */
    if (dictFileName != NULL) {
-        U64 dictFileSize = UTIL_getFileSize(dictFileName);
-        if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
+        U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+        if (dictFileSize > 64 MB)
+            EXM_THROW(10, "dictionary file %s too large", dictFileName);
        dictBufferSize = (size_t)dictFileSize;
        dictBuffer = malloc(dictBufferSize);
-        if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
+        if (dictBuffer==NULL)
+            EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
        BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
    }

@@ -270,7 +270,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm,
            zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
            if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
        }
-        { int res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, 0);
+        { int res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, ZSTD_CONTENTSIZE_UNKNOWN);
          if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); }
        zwc->comprState = ZWRAP_useReset;
    }
@@ -295,7 +295,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
    if (zwc->zbc == NULL) {
        zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
        if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
-        { int const initErr = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
+        { int const initErr = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : ZSTD_CONTENTSIZE_UNKNOWN);
          if (initErr != Z_OK) return ZWRAPC_finishWithError(zwc, strm, initErr); }
        if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
    } else {
@@ -308,7 +308,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
                    return ZWRAPC_finishWithError(zwc, strm, 0);
                }
            } else {
-                int const res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
+                int const res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : ZSTD_CONTENTSIZE_UNKNOWN);
                if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
                if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
            }