Merge pull request #4096 from tpetazzoni/build-fix

lib/libzstd.mk: fix typo in the definition of LIB_BINDIR
2024-07-16 14:09:43 -07:00 · 2024-07-13 13:53:53 +02:00 · 2024-07-04 00:23:55 -07:00 · 2024-07-01 23:52:31 -07:00 · 2024-06-30 16:17:03 -07:00 · 2024-06-29 16:04:05 -07:00
60 changed files with 665 additions and 203 deletions
@@ -3,7 +3,6 @@ task:
  freebsd_instance:
    matrix:
      image_family: freebsd-14-0
-      image_family: freebsd-13-2
  install_script: pkg install -y gmake coreutils
  script: |
    MOREFLAGS="-Werror" gmake -j all
@@ -31,12 +31,12 @@ jobs:
      run: make test

  # lasts ~26mn
-  make-test-osx:
+  make-test-macos:
    runs-on: macos-latest
    steps:
    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
-    - name: OS-X test
-      run: make test # make -c lib all doesn't work because of the fact that it's not a tty
+    - name: make test on macos
+      run: make test

  # lasts ~24mn
  make-test-32bit:
@@ -247,7 +247,7 @@ jobs:
        shell: msys2 {0}
    steps:
    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
-    - uses: msys2/setup-msys2@cc11e9188b693c2b100158c3322424c4cc1dadea # tag=v2.22.0
+    - uses: msys2/setup-msys2@d0e80f58dffbc64f6a3a1f43527d469b4fc7b6c8 # tag=v2.23.0
      with:
        msystem: MINGW64
        install: make
@@ -237,6 +237,50 @@ jobs:
          meson test -C builddir/ --print-errorlogs
          meson install -C builddir --destdir staging/

+  meson-mingw-cross-compilation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
+      - name: Install packages
+        run: |
+          sudo apt-get -qqq update
+          sudo apt-get -y install build-essential python3-pip ninja-build {gcc,g++}-mingw-w64-x86-64
+          pip install --pre meson
+      - name: Build with Meson
+        run: |
+          cat > cross.ini <<EOF
+          [binaries]
+          ar = 'x86_64-w64-mingw32-ar'
+          c = 'x86_64-w64-mingw32-gcc'
+          cpp = 'x86_64-w64-mingw32-g++'
+          ld = 'x86_64-w64-mingw32-ld'
+          objcopy = 'x86_64-w64-mingw32-objcopy'
+          objdump = 'x86_64-w64-mingw32-objdump'
+          strip = 'x86_64-w64-mingw32-strip'
+          windres = 'x86_64-w64-mingw32-windres'
+
+          [host_machine]
+          system = 'windows'
+          endian = 'little'
+          cpu_family = 'x86_64'
+          cpu = 'x86_64'
+          EOF
+
+          # pzstd doesn't build; skip -Dbin_contrib=true
+          meson setup \
+            --buildtype=debugoptimized \
+            --cross-file=cross.ini \
+            -Db_lundef=false \
+            -Dbin_programs=true \
+            -Dbin_tests=true \
+            -Ddefault_library=both \
+            build/meson builddir
+          ninja -C builddir/
+          if grep -- -pthread builddir/meson-private/libzstd.pc; then
+            echo "Error: found stray pthread dependency"
+            exit 1
+          fi
+
  meson-windows:
    runs-on: windows-latest
    steps:
@@ -455,7 +499,7 @@ jobs:
        shell: msys2 {0}
    steps:
    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
-    - uses: msys2/setup-msys2@cc11e9188b693c2b100158c3322424c4cc1dadea # tag=v2.22.0
+    - uses: msys2/setup-msys2@d0e80f58dffbc64f6a3a1f43527d469b4fc7b6c8 # tag=v2.23.0
      with:
        msystem: ${{ matrix.msystem }}
        install: make diffutils
@@ -532,6 +576,11 @@ jobs:
        make -j allzstd &&
        make -C tests fuzzer &&
        ./tests/fuzzer.exe -v -T1m
+    - name: cygwin install test
+      shell: C:\cygwin\bin\bash.exe --noprofile --norc -eo pipefail '{0}'
+      run: >-
+        make -j &&
+        make install

  pkg-config:
    runs-on: ubuntu-latest
@@ -59,6 +59,6 @@ jobs:

      # Upload the results to GitHub's code scanning dashboard.
      - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@3ab4101902695724f9365a384f86c1074d94e18c # tag=v3.24.7
+        uses: github/codeql-action/upload-sarif@c7f9125735019aa87cfc361530512d50ea439c71 # tag=v3.25.1
        with:
          sarif_file: results.sarif
@@ -24,7 +24,7 @@ jobs:
        shell: msys2 {0}
    steps:
    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v3
-    - uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
+    - uses: msys2/setup-msys2@d0e80f58dffbc64f6a3a1f43527d469b4fc7b6c8 # tag=v2.23.0
      with:
        msystem: ${{ matrix.msystem }}
        install: make zlib git p7zip mingw-w64-${{matrix.env}}-gcc
@@ -3,7 +3,7 @@ api: Promote `ZSTD_c_targetCBlockSize` to Stable API by @felixhandte
 api: new `ZSTD_d_maxBlockSize` experimental parameter, to reduce streaming decompression memory, by @terrelln
 perf: improve performance of param `ZSTD_c_targetCBlockSize`, by @Cyan4973
 perf: improved compression of arrays of integers at high compression, by @Cyan4973
-lib: reduce binary size with selective built-time exclusion, by @felixhandte
+lib: reduce binary size with selective build-time exclusion, by @felixhandte
 lib: improved huffman speed on small data and linux kernel, by @terrelln
 lib: accept dictionaries with partial literal tables, by @terrelln
 lib: fix CCtx size estimation with external sequence producer, by @embg
@@ -489,7 +489,7 @@ misc: added /contrib/docker script by @gyscos

 v1.3.3  (Dec 21, 2017)
 perf: faster zstd_opt strategy (levels 16-19)
-fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
+fix : bug #944 : multithreading with shared dictionary and large data, reported by @gsliepen
 cli : fix : content size written in header by default
 cli : fix : improved LZ4 format support, by @felixhandte
 cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
@@ -151,7 +151,7 @@ clean:
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT CYGWIN_NT Haiku AIX))
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT% CYGWIN_NT% Haiku AIX,$(shell sh -c 'MSYSTEM="MSYS" uname') ))

 HOST_OS = POSIX

@@ -293,10 +293,10 @@ regressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest

 uasanregressiontest:
-	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined -Werror" CXXFLAGS="-O3 -fsanitize=address,undefined -Werror"

 msanregressiontest:
-	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory -Werror" CXXFLAGS="-O3 -fsanitize=memory -Werror"

 update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
 update_regressionResults:
@@ -390,19 +390,19 @@ lz4install:
 endif


-ifneq (,$(filter MSYS%,$(shell uname)))
+ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
 HOST_OS = MSYS
 endif

 #------------------------------------------------------------------------
 # target specific tests
 #------------------------------------------------------------------------
-ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+ifneq (,$(filter MSYS POSIX,$(HOST_OS)))

 CMAKE ?= cmake
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON

-ifneq (,$(filter MSYS%,$(shell uname)))
+ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname')))
 CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif

@@ -29,10 +29,10 @@ a list of known ports and bindings is provided on [Zstandard homepage](https://f
 ## Benchmarks

 For reference, several fast compression algorithms were tested and compared
-on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`),
-with a Core i7-9700K CPU @ 4.9GHz,
+on a desktop featuring a Core i7-9700K CPU @ 4.9GHz
+and running Ubuntu 20.04 (`Linux ubu20 5.15.0-101-generic`),
 using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 9.3.0,
+compiled with [gcc] 9.4.0,
 on the [Silesia compression corpus].

 [lzbench]: https://github.com/inikep/lzbench
@@ -41,24 +41,23 @@ on the [Silesia compression corpus].

 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.5.1 -1**       | 2.887 |   530 MB/s |  1700 MB/s |
+| **zstd 1.5.6 -1**       | 2.887 |   510 MB/s |  1580 MB/s |
 | [zlib] 1.2.11 -1        | 2.743 |    95 MB/s |   400 MB/s |
-| brotli 1.0.9 -0         | 2.702 |   395 MB/s |   450 MB/s |
-| **zstd 1.5.1 --fast=1** | 2.437 |   600 MB/s |  2150 MB/s |
-| **zstd 1.5.1 --fast=3** | 2.239 |   670 MB/s |  2250 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   540 MB/s |   760 MB/s |
-| **zstd 1.5.1 --fast=4** | 2.148 |   710 MB/s |  2300 MB/s |
-| lzo1x 2.10 -1           | 2.106 |   660 MB/s |   845 MB/s |
-| [lz4] 1.9.3             | 2.101 |   740 MB/s |  4500 MB/s |
-| lzf 3.6 -1              | 2.077 |   410 MB/s |   830 MB/s |
-| snappy 1.1.9            | 2.073 |   550 MB/s |  1750 MB/s |
+| brotli 1.0.9 -0         | 2.702 |   395 MB/s |   430 MB/s |
+| **zstd 1.5.6 --fast=1** | 2.437 |   545 MB/s |  1890 MB/s |
+| **zstd 1.5.6 --fast=3** | 2.239 |   650 MB/s |  2000 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   525 MB/s |   750 MB/s |
+| lzo1x 2.10 -1           | 2.106 |   650 MB/s |   825 MB/s |
+| [lz4] 1.9.4             | 2.101 |   700 MB/s |  4000 MB/s |
+| lzf 3.6 -1              | 2.077 |   420 MB/s |   830 MB/s |
+| snappy 1.1.9            | 2.073 |   530 MB/s |  1660 MB/s |

 [zlib]: https://www.zlib.net/
 [lz4]: https://lz4.github.io/lz4/

 The negative compression levels, specified with `--fast=#`,
 offer faster compression and decompression speed
-at the cost of compression ratio (compared to level 1).
+at the cost of compression ratio.

 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments.
@@ -2,7 +2,7 @@

 IF "%1%" == "" GOTO display_help

-SETLOCAL
+SETLOCAL ENABLEDELAYEDEXPANSION

 SET msbuild_version=%1

@@ -19,7 +19,7 @@ GOTO build
 :display_help

 echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
-echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, VS2019, ...)
+echo   msbuild_version:          VS installed version (latest, VS2012, VS2013, VS2015, VS2017, VS2019, ...)
 echo   msbuild_platform:         Platform (x64 or Win32)
 echo   msbuild_configuration:    VS configuration (Release or Debug)
 echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, ...)
@@ -29,29 +29,23 @@ EXIT /B 1
 :build

 SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
-SET msbuild_vs2017community="%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe"
-SET msbuild_vs2017professional="%programfiles(x86)%\Microsoft Visual Studio\2017\Professional\MSBuild\15.0\Bin\MSBuild.exe"
-SET msbuild_vs2017enterprise="%programfiles(x86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2017Community SET msbuild=%msbuild_vs2017community%
-IF %msbuild_version% == VS2017Professional SET msbuild=%msbuild_vs2017professional%
-IF %msbuild_version% == VS2017Enterprise SET msbuild=%msbuild_vs2017enterprise%
-IF %msbuild_version% == VS2017 (
-	IF EXIST %msbuild_vs2017community% SET msbuild=%msbuild_vs2017community%
-	IF EXIST %msbuild_vs2017professional% SET msbuild=%msbuild_vs2017professional%
-	IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
-)
+IF %msbuild_version% == VS2017 SET vswhere_params=-version [15,16) -products *
+IF %msbuild_version% == VS2017Community SET vswhere_params=-version [15,16) -products Community
+IF %msbuild_version% == VS2017Enterprise SET vswhere_params=-version [15,16) -products Enterprise
+IF %msbuild_version% == VS2017Professional SET vswhere_params=-version [15,16) -products Professional
+IF %msbuild_version% == VS2019 SET vswhere_params=-version [16,17) -products *
+IF %msbuild_version% == VS2022 SET vswhere_params=-version [17,18) -products *
+REM Add the next Visual Studio version here.
+IF %msbuild_version% == latest SET vswhere_params=-latest -products *

-:: VS2019
-SET msbuild_vs2019community="%programfiles(x86)%\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019professional="%programfiles(x86)%\Microsoft Visual Studio\2019\Professional\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019enterprise="%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2019 (
-	IF EXIST %msbuild_vs2019community% SET msbuild=%msbuild_vs2019community%
-	IF EXIST %msbuild_vs2019professional% SET msbuild=%msbuild_vs2019professional%
-	IF EXIST %msbuild_vs2019enterprise% SET msbuild=%msbuild_vs2019enterprise%
+IF NOT DEFINED vswhere_params GOTO skip_vswhere
+SET vswhere="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
+FOR /F "USEBACKQ TOKENS=*" %%F IN (`%vswhere% !vswhere_params! -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe`) DO (
+	SET msbuild="%%F"
 )
+:skip_vswhere

 SET project="%~p0\..\VS2010\zstd.sln"

@@ -57,7 +57,7 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
        # Note that zstd source code is compatible with both C++98 and above
        # and C-gnu90 (c90 + long long + variadic macros ) and above
        # EnableCompilerFlag("-std=c++11" false true) # Set C++ compilation to c++11 standard
-        # EnableCompilerFlag("-std=c99" true false)   # Set C compiation to c99 standard
+        # EnableCompilerFlag("-std=c99" true false)   # Set C compilation to c99 standard
        if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
            # clang-cl normally maps -Wall to -Weverything.
            EnableCompilerFlag("/clang:-Wall" true true false)
@@ -118,7 +118,7 @@ endmacro ()

 # Define directories containing the library's public headers
 set(PUBLIC_INCLUDE_DIRS ${LIBRARY_DIR})
-
+set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} /I ${LIBRARY_DIR}")
 # Split project to static and shared libraries build
 set(library_targets)
 if (ZSTD_BUILD_SHARED)
@@ -88,8 +88,13 @@ feature_lz4 = get_option('lz4')
 # =============================================================================

 libm_dep = cc.find_library('m', required: false)
-thread_dep = dependency('threads', required: feature_multi_thread)
-use_multi_thread = thread_dep.found()
+if host_machine_os == os_windows
+  thread_dep = dependency('', required: false)
+  use_multi_thread = not feature_multi_thread.disabled()
+else
+  thread_dep = dependency('threads', required: feature_multi_thread)
+  use_multi_thread = thread_dep.found()
+endif
 # Arguments in dependency should be equivalent to those passed to pkg-config
 zlib_dep = dependency('zlib', required: feature_zlib)
 use_zlib = zlib_dep.found()
@@ -27,7 +27,7 @@ option('bin_contrib', type: 'boolean', value: false,
  description: 'Enable contrib build')

 option('multi_thread', type: 'feature', value: 'enabled',
-  description: 'Enable multi-threading when pthread is detected')
+  description: 'Enable multi-threading when pthread or Windows is detected')
 option('zlib', type: 'feature', value: 'auto',
  description: 'Enable zlib support')
 option('lzma', type: 'feature', value: 'auto',
@@ -153,6 +153,20 @@ typedef ZSTD_CCtx zstd_cctx;
 */
 size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters);

+/**
+ * zstd_cctx_workspace_bound_with_ext_seq_prod() - max memory needed to
+ * initialize a zstd_cctx when using the block-level external sequence
+ * producer API.
+ * @parameters: The compression parameters to be used.
+ *
+ * If multiple compression parameters might be used, the caller must call
+ * this function for each set of parameters and use the maximum size.
+ *
+ * Return:      A lower bound on the size of the workspace that is passed to
+ *              zstd_init_cctx().
+ */
+size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *parameters);
+
 /**
 * zstd_init_cctx() - initialize a zstd compression context
 * @workspace:      The workspace to emplace the context into. It must outlive
@@ -257,6 +271,16 @@ typedef ZSTD_CStream zstd_cstream;
 */
 size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams);

+/**
+ * zstd_cstream_workspace_bound_with_ext_seq_prod() - memory needed to initialize
+ * a zstd_cstream when using the block-level external sequence producer API.
+ * @cparams: The compression parameters to be used for compression.
+ *
+ * Return:   A lower bound on the size of the workspace that is passed to
+ *           zstd_init_cstream().
+ */
+size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *cparams);
+
 /**
 * zstd_init_cstream() - initialize a zstd streaming compression context
 * @parameters        The zstd parameters to use for compression.
@@ -416,6 +440,18 @@ size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
 */
 size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);

+/**
+ * zstd_register_sequence_producer() - exposes the zstd library function
+ * ZSTD_registerSequenceProducer(). This is used for the block-level external
+ * sequence producer API. See upstream zstd.h for detailed documentation.
+ */
+typedef ZSTD_sequenceProducer_F zstd_sequence_producer_f;
+void zstd_register_sequence_producer(
+  zstd_cctx *cctx,
+  void* sequence_producer_state,
+  zstd_sequence_producer_f sequence_producer
+);
+
 /**
 * struct zstd_frame_params - zstd frame parameters stored in the frame header
 * @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not
@@ -16,6 +16,7 @@

 #include "common/zstd_deps.h"
 #include "common/zstd_internal.h"
+#include "compress/zstd_compress_internal.h"

 #define ZSTD_FORWARD_IF_ERR(ret)            \
 	do {                                \
@@ -85,6 +86,52 @@ size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams)
 }
 EXPORT_SYMBOL(zstd_cctx_workspace_bound);

+// Used by zstd_cctx_workspace_bound_with_ext_seq_prod()
+static size_t dummy_external_sequence_producer(
+	void *sequenceProducerState,
+	ZSTD_Sequence *outSeqs, size_t outSeqsCapacity,
+	const void *src, size_t srcSize,
+	const void *dict, size_t dictSize,
+	int compressionLevel,
+	size_t windowSize)
+{
+	(void)sequenceProducerState;
+	(void)outSeqs; (void)outSeqsCapacity;
+	(void)src; (void)srcSize;
+	(void)dict; (void)dictSize;
+	(void)compressionLevel;
+	(void)windowSize;
+	return ZSTD_SEQUENCE_PRODUCER_ERROR;
+}
+
+static void init_cctx_params_from_compress_params(
+	ZSTD_CCtx_params *cctx_params,
+	const zstd_compression_parameters *compress_params)
+{
+	ZSTD_parameters zstd_params;
+	memset(&zstd_params, 0, sizeof(zstd_params));
+	zstd_params.cParams = *compress_params;
+	ZSTD_CCtxParams_init_advanced(cctx_params, zstd_params);
+}
+
+size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
+{
+	ZSTD_CCtx_params cctx_params;
+	init_cctx_params_from_compress_params(&cctx_params, compress_params);
+	ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
+	return ZSTD_estimateCCtxSize_usingCCtxParams(&cctx_params);
+}
+EXPORT_SYMBOL(zstd_cctx_workspace_bound_with_ext_seq_prod);
+
+size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
+{
+	ZSTD_CCtx_params cctx_params;
+	init_cctx_params_from_compress_params(&cctx_params, compress_params);
+	ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
+	return ZSTD_estimateCStreamSize_usingCCtxParams(&cctx_params);
+}
+EXPORT_SYMBOL(zstd_cstream_workspace_bound_with_ext_seq_prod);
+
 zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
 {
 	if (workspace == NULL)
@@ -160,5 +207,14 @@ size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output)
 }
 EXPORT_SYMBOL(zstd_end_stream);

+void zstd_register_sequence_producer(
+  zstd_cctx *cctx,
+  void* sequence_producer_state,
+  zstd_sequence_producer_f sequence_producer
+) {
+	ZSTD_registerSequenceProducer(cctx, sequence_producer_state, sequence_producer);
+}
+EXPORT_SYMBOL(zstd_register_sequence_producer);
+
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Zstd Compressor");
@@ -18,6 +18,26 @@ This document lists a few known cases where invalid data was formerly accepted
 by the decoder, and what has changed since.


+Truncated Huffman states
+------------------------
+
+**Last affected version**: v1.5.6
+
+**Produced by the reference compressor**: No
+
+**Example Frame**: `28b5 2ffd 0000 5500 0072 8001 0420 7e1f 02aa 00`
+
+When using FSE-compressed Huffman weights, the compressed weight bitstream
+could contain fewer bits than necessary to decode the initial states.
+
+The reference decompressor up to v1.5.6 will decode truncated or missing
+initial states as zero, which can result in a valid Huffman tree if only
+the second state is truncated.
+
+In newer versions, truncated initial states are reported as a corruption
+error by the decoder.
+
+
 Offset == 0
 -----------

@@ -10,7 +10,7 @@

 ZSTD ?= zstd   # note: requires zstd installation on local system

-UNAME?= $(shell uname)
+UNAME?= $(shell sh -c 'MSYSTEM="MSYS" uname') 
 ifeq ($(UNAME), SunOS)
 DIFF ?= gdiff
 else
@@ -1125,8 +1125,6 @@ If it is a 3, another 2-bits repeat flag follows, and so on.

 When last symbol reaches cumulated total of `1 << Accuracy_Log`,
 decoding is complete.
-If the last symbol makes cumulated total go above `1 << Accuracy_Log`,
-distribution is considered corrupted.
 If this process results in a non-zero probability for a value outside of the
 valid range of values that the FSE table is defined for, even if that value is
 not used, then the data is considered corrupted.
@@ -1364,6 +1362,10 @@ symbols for each of the final states are decoded and the process is complete.
 If this process would produce more weights than the maximum number of decoded
 weights (255), then the data is considered corrupted.

+If either of the 2 initial states are absent or truncated, then the data is
+considered corrupted.  Consequently, it is not possible to encode fewer than
+2 weights using this mode.
+
 #### Conversion from weights to Huffman prefix codes

 All present symbols shall now have a `Weight` value.
@@ -298,7 +298,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                              * Special: value 0 means "use default strategy". */

    ZSTD_c_targetCBlockSize=130, </b>/* v1.5.6+<b>
-                                  * Attempts to fit compressed block size into approximatively targetCBlockSize.
+                                  * Attempts to fit compressed block size into approximately targetCBlockSize.
                                  * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
                                  * Note that it's not a guarantee, just a convergence target (default:0).
                                  * No target when targetCBlockSize == 0.
@@ -63,6 +63,8 @@ CPPFLAGS_DYNLIB  += -DZSTD_MULTITHREAD # dynamic library build defaults to multi
 LDFLAGS_DYNLIB   += -pthread
 CPPFLAGS_STATICLIB +=                  # static library build defaults to single-threaded

+# pkg-config Libs.private points to LDFLAGS_DYNLIB
+PCLIB := $(LDFLAGS_DYNLIB)

 ifeq ($(findstring GCC,$(CCVER)),GCC)
 decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
@@ -186,12 +188,15 @@ lib : libzstd.a libzstd
 %-mt : CPPFLAGS_DYNLIB  := -DZSTD_MULTITHREAD
 %-mt : CPPFLAGS_STATICLIB := -DZSTD_MULTITHREAD
 %-mt : LDFLAGS_DYNLIB   := -pthread
+%-mt : PCLIB :=
+%-mt : PCMTLIB := $(LDFLAGS_DYNLIB)
 %-mt : %
 	@echo multi-threaded build completed

 %-nomt : CPPFLAGS_DYNLIB  :=
 %-nomt : LDFLAGS_DYNLIB   :=
 %-nomt : CPPFLAGS_STATICLIB :=
+%-nomt : PCLIB :=
 %-nomt : %
 	@echo single-threaded build completed

@@ -261,7 +266,7 @@ clean:
 #-----------------------------------------------------------------------------
 # make install is validated only for below listed environments
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT CYGWIN_NT))
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT% CYGWIN_NT%,$(UNAME)))

 lib: libzstd.pc

@@ -292,13 +297,21 @@ PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix})
 # to PREFIX, rather than as a resolved value.
 PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix})

-ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly))
+
+ifneq ($(MT),)
+  PCLIB :=
+  PCMTLIB := $(LDFLAGS_DYNLIB)
+else
+  PCLIB := $(LDFLAGS_DYNLIB)
+endif
+
+ifneq (,$(filter FreeBSD NetBSD DragonFly,$(UNAME)))
  PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
 else
  PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
 endif

-ifneq (,$(filter $(UNAME),SunOS))
+ifneq (,$(filter SunOS,$(UNAME)))
  INSTALL ?= ginstall
 else
  INSTALL ?= install
@@ -308,6 +321,10 @@ INSTALL_PROGRAM ?= $(INSTALL)
 INSTALL_DATA    ?= $(INSTALL) -m 644


+# pkg-config library define.
+# For static single-threaded library declare -pthread in Libs.private
+# For static multi-threaded library declare -pthread in Libs and Cflags
+.PHONY: libzstd.pc
 libzstd.pc: libzstd.pc.in
 	@echo creating pkgconfig
 	@sed \
@@ -316,7 +333,8 @@ libzstd.pc: libzstd.pc.in
 	        -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
 	        -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
 	        -e 's|@VERSION@|$(VERSION)|' \
-	        -e 's|@LIBS_PRIVATE@|$(LDFLAGS_DYNLIB)|' \
+	        -e 's|@LIBS_MT@|$(PCMTLIB)|' \
+	        -e 's|@LIBS_PRIVATE@|$(PCLIB)|' \
 	        $< >$@

 .PHONY: install
@@ -27,12 +27,16 @@ Enabling multithreading requires 2 conditions :

 For convenience, we provide a build target to generate multi and single threaded libraries:
 - Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`.
+  Note that the `.pc` generated on calling `make lib-mt` will already include the require Libs and Cflags.
 - Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`.
 - By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`.

 When linking a POSIX program with a multithreaded version of `libzstd`,
 note that it's necessary to invoke the `-pthread` flag during link stage.

+The `.pc` generated from `make install` or `make install-pc` always assume a single-threaded static library
+is compiled. To correctly generate a `.pc` for the multi-threaded static library, set `MT=1` as ENV variable.
+
 Multithreading capabilities are exposed
 via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).

@@ -43,6 +43,8 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4)
        return (unsigned)__builtin_ctz(val);
+#   elif defined(__ICCARM__)
+        return (unsigned)__builtin_ctz(val);
 #   else
        return ZSTD_countTrailingZeros32_fallback(val);
 #   endif
@@ -82,6 +84,8 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4)
        return (unsigned)__builtin_clz(val);
+#   elif defined(__ICCARM__)
+        return (unsigned)__builtin_clz(val);
 #   else
        return ZSTD_countLeadingZeros32_fallback(val);
 #   endif
@@ -105,6 +109,8 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
        return (unsigned)__builtin_ctzll(val);
+#   elif defined(__ICCARM__)
+        return (unsigned)__builtin_ctzll(val);
 #   else
        {
            U32 mostSignificantWord = (U32)(val >> 32);
@@ -136,6 +142,8 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4)
        return (unsigned)(__builtin_clzll(val));
+#   elif defined(__ICCARM__)
+        return (unsigned)(__builtin_clzll(val));
 #   else
        {
            U32 mostSignificantWord = (U32)(val >> 32);
@@ -27,7 +27,7 @@
 #  define INLINE_KEYWORD
 #endif

-#if defined(__GNUC__) || defined(__ICCARM__)
+#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
 #  define FORCE_INLINE_ATTR __attribute__((always_inline))
 #elif defined(_MSC_VER)
 #  define FORCE_INLINE_ATTR __forceinline
@@ -54,7 +54,7 @@
 #endif

 /* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
 #  define UNUSED_ATTR __attribute__((unused))
 #else
 #  define UNUSED_ATTR
@@ -95,6 +95,8 @@
 #ifndef MEM_STATIC  /* already defined in Linux Kernel mem.h */
 #if defined(__GNUC__)
 #  define MEM_STATIC static __inline UNUSED_ATTR
+#elif defined(__IAR_SYSTEMS_ICC__)
+#  define MEM_STATIC static inline UNUSED_ATTR
 #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #  define MEM_STATIC static inline
 #elif defined(_MSC_VER)
@@ -108,7 +110,7 @@
 #ifdef _MSC_VER
 #  define FORCE_NOINLINE static __declspec(noinline)
 #else
-#  if defined(__GNUC__) || defined(__ICCARM__)
+#  if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
 #    define FORCE_NOINLINE static __attribute__((__noinline__))
 #  else
 #    define FORCE_NOINLINE static
@@ -117,7 +119,7 @@


 /* target attribute */
-#if defined(__GNUC__) || defined(__ICCARM__)
+#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
 #  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
 #else
 #  define TARGET_ATTRIBUTE(target)
@@ -324,7 +326,7 @@
 #endif

 /**
- * Helper function to perform a wrapped pointer difference without trigging
+ * Helper function to perform a wrapped pointer difference without triggering
 * UBSAN.
 *
 * @returns lhs - rhs with wrapping
@@ -35,7 +35,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
    U32 f7b = 0;
    U32 f7c = 0;
 #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
-#if !defined(__clang__)
+#if !defined(__clang__) || __clang_major__ >= 16
    int reg[4];
    __cpuid((int*)reg, 0);
    {
@@ -190,6 +190,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
    FSE_initDState(&state1, &bitD, dt);
    FSE_initDState(&state2, &bitD, dt);

+    RETURN_ERROR_IF(BIT_reloadDStream(&bitD)==BIT_DStream_overflow, corruption_detected, "");
+
 #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)

    /* 4 symbols per loop */
@@ -30,6 +30,8 @@ extern "C" {
 #if defined(_MSC_VER)   /* Visual Studio */
 #   include <stdlib.h>  /* _byteswap_ulong */
 #   include <intrin.h>  /* _byteswap_* */
+#elif defined(__ICCARM__)
+#   include <intrinsics.h>
 #endif

 /*-**************************************************************
@@ -154,6 +156,8 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
    return 1;
 #elif defined(__DMC__) && defined(_M_IX86)
    return 1;
+#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__
+    return 1;
 #else
    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
    return one.c[0];
@@ -246,6 +250,8 @@ MEM_STATIC U32 MEM_swap32(U32 in)
 #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
  || (defined(__clang__) && __has_builtin(__builtin_bswap32))
    return __builtin_bswap32(in);
+#elif defined(__ICCARM__)
+    return __REV(in);
 #else
    return MEM_swap32_fallback(in);
 #endif
@@ -90,7 +90,7 @@
 #endif

 /**
- * Only enable assembly for GNUC compatible compilers,
+ * Only enable assembly for GNU C compatible compilers,
 * because other platforms may not support GAS assembly syntax.
 *
 * Only enable assembly for Linux / MacOS, other platforms may
@@ -24,6 +24,18 @@
 #ifndef ZSTD_DEPS_COMMON
 #define ZSTD_DEPS_COMMON

+/* Even though we use qsort_r only for the dictionary builder, the macro
+ * _GNU_SOURCE has to be declared *before* the inclusion of any standard
+ * header and the script 'combine.sh' combines the whole zstd source code
+ * in a single file.
+ */
+#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
+    defined(__CYGWIN__) || defined(__MSYS__)
+#if !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif
+#endif
+
 #include <limits.h>
 #include <stddef.h>
 #include <string.h>
@@ -2606,7 +2606,7 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa
    /* Protect special index values < ZSTD_WINDOW_START_INDEX. */
    U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX;
    assert((size & (ZSTD_ROWSIZE-1)) == 0);  /* multiple of ZSTD_ROWSIZE */
-    assert(size < (1U<<31));   /* can be casted to int */
+    assert(size < (1U<<31));   /* can be cast to int */

 #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
    /* To validate that the table reuse logic is sound, and that we don't
@@ -5622,7 +5622,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
 *  workspaceSize: Use ZSTD_estimateCDictSize()
 *                 to determine how large workspace must be.
 *  cParams : use ZSTD_getCParams() to transform a compression level
- *            into its relevants cParams.
+ *            into its relevant cParams.
 * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
 *  Note : there is no corresponding "free" function.
 *         Since workspace was allocated externally, it must be freed externally.
@@ -1315,6 +1315,13 @@ MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr,
    return matchLowest;
 }

+/* index_safety_check:
+ * intentional underflow : ensure repIndex isn't overlapping dict + prefix
+ * @return 1 if values are not overlapping,
+ * 0 otherwise */
+MEM_STATIC int ZSTD_index_overlap_check(const U32 prefixLowestIndex, const U32 repIndex) {
+    return ((U32)((prefixLowestIndex-1)  - repIndex) >= 3);
+}


 /* debug functions */
@@ -392,7 +392,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
        hashLong[h2] = hashSmall[h] = curr;   /* update hash tables */

        /* check repcode */
-        if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+        if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
            && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
            const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
            mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -513,7 +513,7 @@ _match_stored:
                const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
                        dictBase + repIndex2 - dictIndexDelta :
                        base + repIndex2;
-                if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
+                if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2))
                   && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
                    const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
                    size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
@@ -651,7 +651,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
        size_t mLength;
        hashSmall[hSmall] = hashLong[hLong] = curr;   /* update hash table */

-        if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
+        if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) 
            & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
          && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
            const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
@@ -719,7 +719,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
                U32 const current2 = (U32)(ip-base);
                U32 const repIndex2 = current2 - offset_2;
                const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
-                if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3)   /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
+                if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
                    & (offset_2 <= current2 - dictStartIndex))
                  && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
                    const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
@@ -546,8 +546,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
            size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls);
            hashTable[hash0] = curr;   /* update hash table */

-            if (((U32) ((prefixStartIndex - 1) - repIndex) >=
-                 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
+            if ((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) 
                && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) {
                const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
                mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4;
@@ -631,7 +630,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
                const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
                        dictBase - dictIndexDelta + repIndex2 :
                        base + repIndex2;
-                if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
+                if ( (ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
                   && (MEM_read32(repMatch2) == MEM_read32(ip0))) {
                    const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
                    size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
@@ -925,7 +924,7 @@ _match: /* Requires: ip0, match0, offcode, matchEnd */
        while (ip0 <= ilimit) {
            U32 const repIndex2 = (U32)(ip0-base) - offset_2;
            const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
-            if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0))  /* intentional underflow */
+            if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0))
                 && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) {
                const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
                size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
@@ -1123,9 +1123,9 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr

 /* The high-level approach of the SIMD row based match finder is as follows:
 * - Figure out where to insert the new entry:
- *      - Generate a hash for current input posistion and split it into a one byte of tag and `rowHashLog` bits of index.
- *           - The hash is salted by a value that changes on every contex reset, so when the same table is used
- *             we will avoid collisions that would otherwise slow us down by intorducing phantom matches.
+ *      - Generate a hash for current input position and split it into a one byte of tag and `rowHashLog` bits of index.
+ *           - The hash is salted by a value that changes on every context reset, so when the same table is used
+ *             we will avoid collisions that would otherwise slow us down by introducing phantom matches.
 *      - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines
 *        which row to insert into.
 *      - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can
@@ -1590,7 +1590,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                                && repIndex < prefixLowestIndex) ?
                                   dictBase + (repIndex - dictIndexDelta) :
                                   base + repIndex;
-            if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+            if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
                && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
                const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
                matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1642,7 +1642,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                const BYTE* repMatch = repIndex < prefixLowestIndex ?
                               dictBase + (repIndex - dictIndexDelta) :
                               base + repIndex;
-                if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+                if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
                    && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
                    const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
                    size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1678,7 +1678,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                    const BYTE* repMatch = repIndex < prefixLowestIndex ?
                                   dictBase + (repIndex - dictIndexDelta) :
                                   base + repIndex;
-                    if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+                    if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
                        && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
                        const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
                        size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1740,7 +1740,7 @@ _storeSequence:
                const BYTE* repMatch = repIndex < prefixLowestIndex ?
                        dictBase - dictIndexDelta + repIndex :
                        base + repIndex;
-                if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */)
+                if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
                   && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
                    const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
                    matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
@@ -1986,7 +1986,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
            const U32 repIndex = (U32)(curr+1 - offset_1);
            const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
            const BYTE* const repMatch = repBase + repIndex;
-            if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
+            if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
               & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */
            if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
                /* repcode detected we should take it */
@@ -2027,7 +2027,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
                const U32 repIndex = (U32)(curr - offset_1);
                const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
                const BYTE* const repMatch = repBase + repIndex;
-                if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments  */
+                if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
                   & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
                if (MEM_read32(ip) == MEM_read32(repMatch)) {
                    /* repcode detected */
@@ -2059,7 +2059,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
                    const U32 repIndex = (U32)(curr - offset_1);
                    const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
                    const BYTE* const repMatch = repBase + repIndex;
-                    if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments  */
+                    if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
                       & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
                    if (MEM_read32(ip) == MEM_read32(repMatch)) {
                        /* repcode detected */
@@ -2113,7 +2113,7 @@ _storeSequence:
            const U32 repIndex = repCurrent - offset_2;
            const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
            const BYTE* const repMatch = repBase + repIndex;
-            if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments  */
+            if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
               & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
            if (MEM_read32(ip) == MEM_read32(repMatch)) {
                /* repcode detected we should take it */
@@ -664,13 +664,13 @@ ZSTD_insertBtAndGetAllMatches (
                assert(curr >= windowLow);
                if ( dictMode == ZSTD_extDict
                  && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow)  /* equivalent to `curr > repIndex >= windowLow` */
-                     & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)
+                     & (ZSTD_index_overlap_check(dictLimit, repIndex)) )
                  && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
                    repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
                }
                if (dictMode == ZSTD_dictMatchState
                  && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta))  /* equivalent to `curr > repIndex >= dmsLowLimit` */
-                     & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */
+                     & (ZSTD_index_overlap_check(dictLimit, repIndex)) )
                  && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
                    repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch;
            }   }
@@ -811,7 +811,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)

 /** ZSTD_decompressBound() :
 *  compatible with legacy mode
- *  `src` must point to the start of a ZSTD frame or a skippeable frame
+ *  `src` must point to the start of a ZSTD frame or a skippable frame
 *  `srcSize` must be at least as large as the frame contained
 *  @return : the maximum decompressed size of the compressed source
 */
@@ -358,7 +358,7 @@ size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
 * - start from default distributions, present in /lib/common/zstd_internal.h
 * - generate tables normally, using ZSTD_buildFSETable()
 * - printout the content of tables
- * - pretify output, report below, test with fuzzer to ensure it's correct */
+ * - prettify output, report below, test with fuzzer to ensure it's correct */

 /* Default FSE distribution table for Literal Lengths */
 static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
@@ -21,8 +21,17 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
+/* qsort_r is an extension. */
+#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
+    defined(__CYGWIN__) || defined(__MSYS__)
+#if !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif
+#endif
+
 #include <stdio.h>  /* fprintf */
-#include <stdlib.h> /* malloc, free, qsort */
+#include <stdlib.h> /* malloc, free, qsort_r */
+
 #include <string.h> /* memset */
 #include <time.h>   /* clock */

@@ -232,8 +241,10 @@ typedef struct {
  unsigned d;
 } COVER_ctx_t;

-/* We need a global context for qsort... */
+#if !defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER)
+/* C90 only offers qsort() that needs a global context. */
 static COVER_ctx_t *g_coverCtx = NULL;
+#endif

 /*-*************************************
 *  Helper functions
@@ -276,11 +287,15 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {

 /**
 * Same as COVER_cmp() except ties are broken by pointer value
- * NOTE: g_coverCtx must be set to call this function.  A global is required because
- * qsort doesn't take an opaque pointer.
 */
-static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
-  int result = COVER_cmp(g_coverCtx, lp, rp);
+#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
+static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) {
+#elif defined(_GNU_SOURCE)
+static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) {
+#else /* C90 fallback.*/
+static int COVER_strict_cmp(const void *lp, const void *rp) {
+#endif
+  int result = COVER_cmp((COVER_ctx_t*)g_coverCtx, lp, rp);
  if (result == 0) {
    result = lp < rp ? -1 : 1;
  }
@@ -289,14 +304,50 @@ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
 /**
 * Faster version for d <= 8.
 */
-static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
-  int result = COVER_cmp8(g_coverCtx, lp, rp);
+#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
+static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) {
+#elif defined(_GNU_SOURCE)
+static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) {
+#else /* C90 fallback.*/
+static int COVER_strict_cmp8(const void *lp, const void *rp) {
+#endif
+  int result = COVER_cmp8((COVER_ctx_t*)g_coverCtx, lp, rp);
  if (result == 0) {
    result = lp < rp ? -1 : 1;
  }
  return result;
 }

+/**
+ * Abstract away divergence of qsort_r() parameters.
+ * Hopefully when C11 become the norm, we will be able
+ * to clean it up.
+ */
+static void stableSort(COVER_ctx_t *ctx) {
+#if defined(__APPLE__)
+    qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
+            ctx,
+            (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
+#elif defined(_GNU_SOURCE)
+    qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
+            (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
+            ctx);
+#elif defined(_WIN32) && defined(_MSC_VER)
+    qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
+            (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
+            ctx);
+#elif defined(__OpenBSD__)
+    g_coverCtx = ctx;
+    mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
+          (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
+#else /* C90 fallback.*/
+    g_coverCtx = ctx;
+    /* TODO(cavalcanti): implement a reentrant qsort() when is not available. */
+    qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
+          (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
+#endif
+}
+
 /**
 * Returns the first pointer in [first, last) whose element does not compare
 * less than value.  If no such element exists it returns last.
@@ -620,17 +671,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
    for (i = 0; i < ctx->suffixSize; ++i) {
      ctx->suffix[i] = i;
    }
-    /* qsort doesn't take an opaque pointer, so pass as a global.
-     * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
-     */
-    g_coverCtx = ctx;
-#if defined(__OpenBSD__)
-    mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
-          (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
-#else
-    qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
-          (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
-#endif
+    stableSort(ctx);
  }
  DISPLAYLEVEL(2, "Computing frequencies\n");
  /* For each dmer group (group of positions with the same first d bytes):
@@ -3919,6 +3919,10 @@ ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void)
    if (zbd==NULL) return NULL;
    memset(zbd, 0, sizeof(*zbd));
    zbd->zd = ZSTDv06_createDCtx();
+    if (zbd->zd==NULL) {
+        ZBUFFv06_freeDCtx(zbd); /* avoid leaking the context */
+        return NULL;
+    }
    zbd->stage = ZBUFFds_init;
    return zbd;
 }
@@ -22,7 +22,7 @@ LIBZSTD_MK_INCLUDED := 1

 # By default, library's directory is same as this included makefile
 LIB_SRCDIR ?= $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
-LIB_BINDIR ?= $(LIBSRC_DIR)
+LIB_BINDIR ?= $(LIB_SRCDIR)

 # ZSTD_LIB_MINIFY is a helper variable that
 # configures a bunch of other variables to space-optimized defaults.
@@ -206,15 +206,13 @@ endif
 endif
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)

-UNAME := $(shell uname)
+UNAME := $(shell sh -c 'MSYSTEM="MSYS" uname')

 ifndef BUILD_DIR
 ifeq ($(UNAME), Darwin)
  ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0)
    HASH ?= md5
  endif
-else ifeq ($(UNAME), FreeBSD)
-  HASH ?= gmd5sum
 else ifeq ($(UNAME), NetBSD)
  HASH ?= md5 -n
 else ifeq ($(UNAME), OpenBSD)
@@ -11,6 +11,6 @@ Name: zstd
 Description: fast lossless compression algorithm library
 URL: https://facebook.github.io/zstd/
 Version: @VERSION@
-Libs: -L${libdir} -lzstd
+Libs: -L${libdir} -lzstd @LIBS_MT@
 Libs.private: @LIBS_PRIVATE@
-Cflags: -I${includedir}
+Cflags: -I${includedir} @LIBS_MT@
@@ -57,7 +57,7 @@ extern "C" {
 #else
 #  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
 #    define ZSTD_DEPRECATED(message) [[deprecated(message)]]
-#  elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
+#  elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__)
 #    define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif defined(__GNUC__) && (__GNUC__ >= 3)
 #    define ZSTD_DEPRECATED(message) __attribute__((deprecated))
@@ -157,37 +157,40 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
                                  int compressionLevel);

 /*! ZSTD_decompress() :
- *  `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
- *  `dstCapacity` is an upper bound of originalSize to regenerate.
- *  If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
- *  @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
- *            or an errorCode if it fails (which can be tested using ZSTD_isError()). */
+ * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
+ *  Multiple compressed frames can be decompressed at once with this method.
+ *  The result will be the concatenation of all decompressed frames, back to back.
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
+ *  First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
+ *  If maximum upper bound isn't known, prefer using streaming mode to decompress data.
+ * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ *           or an errorCode if it fails (which can be tested using ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                              const void* src, size_t compressedSize);

 /*! ZSTD_getFrameContentSize() : requires v1.3.0+
- *  `src` should point to the start of a ZSTD encoded frame.
- *  `srcSize` must be at least as large as the frame header.
- *            hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
- *  @return : - decompressed size of `src` frame content, if known
- *            - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
- *            - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
- *   note 1 : a 0 return value means the frame is valid but "empty".
- *   note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
- *            When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
- *            In which case, it's necessary to use streaming mode to decompress data.
- *            Optionally, application can rely on some implicit limit,
- *            as ZSTD_decompress() only needs an upper bound of decompressed size.
- *            (For example, data could be necessarily cut into blocks <= 16 KB).
- *   note 3 : decompressed size is always present when compression is completed using single-pass functions,
- *            such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
- *   note 4 : decompressed size can be very large (64-bits value),
- *            potentially larger than what local system can handle as a single memory segment.
- *            In which case, it's necessary to use streaming mode to decompress data.
- *   note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
- *            Always ensure return value fits within application's authorized limits.
- *            Each application can set its own limits.
- *   note 6 : This function replaces ZSTD_getDecompressedSize() */
+ * `src` should point to the start of a ZSTD encoded frame.
+ * `srcSize` must be at least as large as the frame header.
+ *           hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
+ * @return : - decompressed size of `src` frame content, if known
+ *           - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ *           - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
+ *  note 1 : a 0 return value means the frame is valid but "empty".
+ *  note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode).
+ *           When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ *           In which case, it's necessary to use streaming mode to decompress data.
+ *           Optionally, application can rely on some implicit limit,
+ *           as ZSTD_decompress() only needs an upper bound of decompressed size.
+ *           (For example, data could be necessarily cut into blocks <= 16 KB).
+ *  note 3 : decompressed size is always present when compression is completed using single-pass functions,
+ *           such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
+ *  note 4 : decompressed size can be very large (64-bits value),
+ *           potentially larger than what local system can handle as a single memory segment.
+ *           In which case, it's necessary to use streaming mode to decompress data.
+ *  note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ *           Always ensure return value fits within application's authorized limits.
+ *           Each application can set its own limits.
+ *  note 6 : This function replaces ZSTD_getDecompressedSize() */
 #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
@@ -392,7 +395,7 @@ typedef enum {
                              * Special: value 0 means "use default strategy". */

    ZSTD_c_targetCBlockSize=130, /* v1.5.6+
-                                  * Attempts to fit compressed block size into approximatively targetCBlockSize.
+                                  * Attempts to fit compressed block size into approximately targetCBlockSize.
                                  * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
                                  * Note that it's not a guarantee, just a convergence target (default:0).
                                  * No target when targetCBlockSize == 0.
@@ -855,7 +858,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 *
 *  A ZSTD_DStream object is required to track streaming operations.
 *  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
-*  ZSTD_DStream objects can be reused multiple times.
+*  ZSTD_DStream objects can be re-employed multiple times.
 *
 *  Use ZSTD_initDStream() to start a new decompression operation.
 * @return : recommended first input size
@@ -865,16 +868,21 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 *  The function will update both `pos` fields.
 *  If `input.pos < input.size`, some input has not been consumed.
 *  It's up to the caller to present again remaining data.
+*
 *  The function tries to flush all data decoded immediately, respecting output buffer size.
 *  If `output.pos < output.size`, decoder has flushed everything it could.
-*  But if `output.pos == output.size`, there might be some data left within internal buffers.,
+*
+*  However, when `output.pos == output.size`, it's more difficult to know.
+*  If @return > 0, the frame is not complete, meaning
+*  either there is still some data left to flush within internal buffers,
+*  or there is more input to read to complete the frame (or both).
 *  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
 *  Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
 * @return : 0 when a frame is completely decoded and fully flushed,
 *        or an error code, which can be tested using ZSTD_isError(),
 *        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
 *                                the return value is a suggested next input size (just a hint for better latency)
-*                                that will never request more than the remaining frame size.
+*                                that will never request more than the remaining content of the compressed frame.
 * *******************************************************************************/

 typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
@@ -901,9 +909,10 @@ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 * Function will update both input and output `pos` fields exposing current state via these fields:
 * - `input.pos < input.size`, some input remaining and caller should provide remaining input
 *   on the next call.
- * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
- * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
- *   call ZSTD_decompressStream() again to flush remaining data to output.
+ * - `output.pos < output.size`, decoder flushed internal output buffer.
+ * - `output.pos == output.size`, unflushed data potentially present in the internal buffers,
+ *   check ZSTD_decompressStream() @return value,
+ *   if > 0, invoke it again to flush remaining data to output.
 * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
 *
 * @return : 0 when a frame is completely decoded and fully flushed,
@@ -1796,7 +1805,15 @@ static
 #ifdef __GNUC__
 __attribute__((__unused__))
 #endif
+
+#if defined(__clang__) && __clang_major__ >= 5
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
+#endif
 ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< this constant defers to stdlib's functions */
+#if defined(__clang__) && __clang_major__ >= 5
+#pragma clang diagnostic pop
+#endif

 ZSTDLIB_STATIC_API ZSTD_CCtx*    ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
@@ -345,7 +345,7 @@ include $(wildcard $(DEPFILES))
 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT CYGWIN_NT))
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT% CYGWIN_NT%,$(UNAME)))

 HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 EGREP_OPTIONS ?=
@@ -388,14 +388,14 @@ datarootdir ?= $(PREFIX)/share
 mandir      ?= $(datarootdir)/man
 man1dir     ?= $(mandir)/man1

-ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter OpenBSD FreeBSD NetBSD DragonFly SunOS,$(UNAME)))
  MANDIR  ?= $(PREFIX)/man
  MAN1DIR ?= $(MANDIR)/man1
 else
  MAN1DIR ?= $(man1dir)
 endif

-ifneq (,$(filter $(UNAME),SunOS))
+ifneq (,$(filter SunOS,$(UNAME)))
  INSTALL ?= ginstall
 else
  INSTALL ?= install
@@ -298,7 +298,7 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
          fs.oneSampleTooLarge |= (fileSize > 2*SAMPLESIZE_MAX);

          /* Limit to the first SAMPLESIZE_MAX (128kB) of the file */
-          DISPLAYLEVEL(3, "Sample file '%s' is too large, limiting to %d KB",
+          DISPLAYLEVEL(3, "Sample file '%s' is too large, limiting to %d KB\n",
              fileNamesTable[n], SAMPLESIZE_MAX / (1 KB));
        }
        fs.nbSamples += 1;
@@ -1494,7 +1494,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                      int compressionLevel, U64* readsize)
 {
    cRess_t const ress = *ressPtr;
-    IOJob_t *writeJob = AIO_WritePool_acquireJob(ressPtr->writeCtx);
+    IOJob_t* writeJob = AIO_WritePool_acquireJob(ressPtr->writeCtx);

    U64 compressedfilesize = 0;
    ZSTD_EndDirective directive = ZSTD_e_continue;
@@ -1526,8 +1526,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
      CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) );
    }

-    {
-        int windowLog;
+    {   int windowLog;
        UTIL_HumanReadableSize_t windowSize;
        CHECK(ZSTD_CCtx_getParameter(ress.cctx, ZSTD_c_windowLog, &windowLog));
        if (windowLog == 0) {
@@ -1542,7 +1541,6 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
        windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize)));
        DISPLAYLEVEL(4, "Decompression will require %.*f%s of memory\n", windowSize.precision, windowSize.value, windowSize.suffix);
    }
-    (void)srcFileName;

    /* Main compression loop */
    do {
@@ -2439,12 +2437,14 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
                        U64 alreadyDecoded)  /* for multi-frames streams */
 {
    U64 frameSize = 0;
-    IOJob_t *writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
+    const char* srcFName20 = srcFileName;
+    IOJob_t* writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
+    assert(writeJob);

    /* display last 20 characters only when not --verbose */
    {   size_t const srcFileLength = strlen(srcFileName);
        if ((srcFileLength>20) && (g_display_prefs.displayLevel<3))
-            srcFileName += srcFileLength-20;
+            srcFName20 += srcFileLength-20;
    }

    ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only);
@@ -2471,19 +2471,12 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
        AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
        frameSize += outBuff.pos;
        if (fCtx->nbFilesTotal > 1) {
-            size_t srcFileNameSize = strlen(srcFileName);
-            if (srcFileNameSize > 18) {
-                const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
-                DISPLAYUPDATE_PROGRESS(
-                        "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s...    ",
-                        fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
-            } else {
-                DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s...    ",
-                            fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
-            }
+            DISPLAYUPDATE_PROGRESS(
+                        "\rDecompress: %2u/%2u files. Current: %s : %.*f%s...    ",
+                        fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFName20, hrs.precision, hrs.value, hrs.suffix);
        } else {
            DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s...     ",
-                            srcFileName, hrs.precision, hrs.value, hrs.suffix);
+                            srcFName20, hrs.precision, hrs.value, hrs.suffix);
        }

        AIO_ReadPool_consumeBytes(ress->readCtx, inBuff.pos);
@@ -268,7 +268,7 @@ static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
 /* AIO_IOPool_acquireJob:
 * Returns an available io job to be used for a future io. */
 static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
-    IOJob_t *job;
+    IOJob_t* job;
    assert(ctx->file != NULL || ctx->prefs->testMode);
    AIO_IOPool_lockJobsMutex(ctx);
    assert(ctx->availableJobsCount > 0);
@@ -224,9 +224,9 @@ roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c
 CLEAN += longmatch
 longmatch : $(ZSTD_OBJECTS) longmatch.c

-CLEAN += bigdict
-bigdict: CFLAGS += $(MULTITHREAD)
-bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c
+CLEAN += largeDictionary
+largeDictionary: CFLAGS += $(MULTITHREAD)
+largeDictionary: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c largeDictionary.c

 CLEAN += invalidDictionaries
 invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
@@ -271,8 +271,8 @@ clean:
 #----------------------------------------------------------------------------------
 # valgrind tests validated only for some posix platforms
 #----------------------------------------------------------------------------------
-UNAME := $(shell uname)
-ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX CYGWIN_NT))
+UNAME := $(shell sh -c 'MSYSTEM="MSYS" uname') 
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX CYGWIN_NT%,$(UNAME)))
 HOST_OS = POSIX

 .PHONY: test-valgrind
@@ -300,10 +300,10 @@ endif
 #-----------------------------------------------------------------------------
 # make tests validated only for below targets
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+ifneq (,$(filter MSYS POSIX,$(HOST_OS)))

 DIFF:=diff
-ifneq (,$(filter $(UNAME),SunOS))
+ifneq (,$(filter SunOS,$(UNAME)))
  DIFF:=gdiff
 endif

@@ -391,8 +391,8 @@ test-zstream32: zstreamtest32
 test-longmatch: longmatch
 	$(QEMU_SYS) ./longmatch

-test-bigdict: bigdict
-	$(QEMU_SYS) ./bigdict
+test-largeDictionary: largeDictionary
+	$(QEMU_SYS) ./largeDictionary

 test-invalidDictionaries: invalidDictionaries
 	$(QEMU_SYS) ./invalidDictionaries
@@ -70,12 +70,14 @@ int main(int argc, const char** argv)
    char* buffer = (char*)malloc(bufferSize);
    void* out = malloc(outSize);
    void* roundtrip = malloc(dataSize);
+    int _exit_code = 0;
    (void)argc;
    (void)argv;

    if (!buffer || !out || !roundtrip || !cctx || !dctx) {
        fprintf(stderr, "Allocation failure\n");
-        return 1;
+        _exit_code = 1;
+        goto cleanup;
    }

    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31)))
@@ -119,10 +121,13 @@ int main(int argc, const char** argv)

    fprintf(stderr, "Success!\n");

+    goto cleanup;
+
+cleanup:
    free(roundtrip);
    free(out);
    free(buffer);
-    ZSTD_freeDCtx(dctx);
    ZSTD_freeCCtx(cctx);
-    return 0;
+    ZSTD_freeDCtx(dctx);
+    return _exit_code;
 }
@@ -18,7 +18,6 @@ esac

 case "$UNAME" in
  Darwin) MD5SUM="md5 -r" ;;
-  FreeBSD) MD5SUM="gmd5sum" ;;
  NetBSD) MD5SUM="md5 -n" ;;
  OpenBSD) MD5SUM="md5" ;;
  *) MD5SUM="md5sum" ;;
@@ -17,7 +17,7 @@ LDFLAGS ?=
 ARFLAGS ?=
 LIB_FUZZING_ENGINE ?= libregression.a
 PYTHON ?= python
-ifeq ($(shell uname), Darwin)
+ifeq ($(shell sh -c 'MSYSTEM="MSYS" uname') , Darwin)
 	DOWNLOAD?=curl -L -o
 else
 	DOWNLOAD?=wget -O
@@ -408,7 +408,6 @@ def build(args):
    mflags = shlex.split(args.mflags)
    # Flags to be added to both cflags and cxxflags
    common_flags = [
-        '-Werror',
        '-Wno-error=declaration-after-statement',
        '-Wno-error=c++-compat',
        '-Wno-error=deprecated' # C files are sometimes compiled with CXX
@@ -127,7 +127,7 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
    FUZZ_ASSERT(litPtr <= litEnd);
    if (mode == ZSTD_sf_noBlockDelimiters) {
        const uint32_t lastLLSize = (uint32_t)(litEnd - litPtr);
-        if (lastLLSize <= oend - op) {
+        if (lastLLSize <= (uint32_t)(oend - op)) {
            memcpy(op, litPtr, lastLLSize);
            generatedSrcBufferSize += lastLLSize;
    }   }
@@ -36,7 +36,7 @@ clean:
 #------------------------------------------------------------------------------
 # validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD,$(shell sh -c 'MSYSTEM="MSYS" uname') ))

 test-%: zstd
 	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include "datagen.h"
+#include "mem.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+static int
+compress(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+         void* dst, size_t dstCapacity,
+         void const* src, size_t srcSize,
+         void* roundtrip, ZSTD_EndDirective end)
+{
+    ZSTD_inBuffer in = {src, srcSize, 0};
+    ZSTD_outBuffer out = {dst, dstCapacity, 0};
+    int ended = 0;
+
+    while (!ended && (in.pos < in.size || out.pos > 0)) {
+        size_t rc;
+        out.pos = 0;
+        rc = ZSTD_compressStream2(cctx, &out, &in, end);
+        if (ZSTD_isError(rc))
+            return 1;
+        if (end == ZSTD_e_end && rc == 0)
+            ended = 1;
+        {
+            ZSTD_inBuffer rtIn = {dst, out.pos, 0};
+            ZSTD_outBuffer rtOut = {roundtrip, srcSize, 0};
+            rc = 1;
+            while (rtIn.pos < rtIn.size || rtOut.pos > 0) {
+                rtOut.pos = 0;
+                rc = ZSTD_decompressStream(dctx, &rtOut, &rtIn);
+                if (ZSTD_isError(rc)) {
+                    fprintf(stderr, "Decompression error: %s\n", ZSTD_getErrorName(rc));
+                    return 1;
+                }
+                if (rc == 0)
+                    break;
+            }
+            if (ended && rc != 0) {
+                fprintf(stderr, "Frame not finished!\n");
+                return 1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int main(int argc, const char** argv)
+{
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    const size_t dataSize = (size_t)1 << 30;
+    const size_t outSize = ZSTD_compressBound(dataSize);
+    const size_t bufferSize = (size_t)1 << 31;
+    char* buffer = (char*)malloc(bufferSize);
+    void* out = malloc(outSize);
+    void* roundtrip = malloc(dataSize);
+    (void)argc;
+    (void)argv;
+
+    if (!buffer || !out || !roundtrip || !cctx || !dctx) {
+        fprintf(stderr, "Allocation failure\n");
+        return 1;
+    }
+
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_overlapLog, 9)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, 7)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, 1)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 10)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, 10)))
+        return 1;
+
+    if (ZSTD_isError(ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 31)))
+        return 1;
+
+    RDG_genBuffer(buffer, bufferSize, 1.0, 0.0, 0xbeefcafe);
+
+    /* Compress 30 GB */
+    {
+        int i;
+        for (i = 0; i < 10; ++i) {
+            fprintf(stderr, "Compressing 1 GB\n");
+            if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_continue))
+                return 1;
+        }
+    }
+    fprintf(stderr, "Compressing 1 GB\n");
+    if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_end))
+        return 1;
+
+    fprintf(stderr, "Success!\n");
+
+    free(roundtrip);
+    free(out);
+    free(buffer);
+    ZSTD_freeDCtx(dctx);
+    ZSTD_freeCCtx(cctx);
+    return 0;
+}
@@ -1273,7 +1273,6 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
        f = fopen(fileNamesTable[n], "rb");
        if (f==NULL) {
            DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
-            fclose(f);
            ret = 10;
            goto _cleanUp;
        }
@@ -123,7 +123,6 @@ esac

 case "$UNAME" in
  Darwin) MD5SUM="md5 -r" ;;
-  FreeBSD) MD5SUM="gmd5sum" ;;
  NetBSD) MD5SUM="md5 -n" ;;
  OpenBSD) MD5SUM="md5" ;;
  *) MD5SUM="md5sum" ;;
@@ -24,5 +24,7 @@ char const* result_get_error_string(result_t result) {
            return "decompression error";
        case result_error_round_trip_error:
            return "round trip error";
+        default:
+            return "unknown error - " + result_get_error(result);
    }
 }
@@ -2356,7 +2356,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
    }
    DISPLAYLEVEL(3, "OK \n");

-    DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx: ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx (one-shot): ", testNb++);
    {
        size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
        BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
@@ -2375,7 +2375,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
            size_t const cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
            cctxBuf = malloc(cctxSize);
            staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
-            ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params);
+            CHECK_Z(ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params));
        }

        // Check that compression with external sequence producer succeeds when expected
@@ -2408,6 +2408,65 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
    }
    DISPLAYLEVEL(3, "OK \n");

+    DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx (streaming): ", testNb++);
+    {
+        size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
+        BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
+        size_t const checkBufSize = CNBufferSize;
+        BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
+        ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+        ZSTD_CCtx* staticCCtx;
+        void* cctxBuf;
+        EMF_testCase seqProdState;
+
+        CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_validateSequences, 1));
+        CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableSeqProducerFallback, 0));
+        ZSTD_CCtxParams_registerSequenceProducer(params, &seqProdState, zstreamSequenceProducer);
+
+        {
+            size_t const cctxSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+            cctxBuf = malloc(cctxSize);
+            staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
+            CHECK_Z(ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params));
+        }
+
+        // Check that compression with external sequence producer succeeds when expected
+        seqProdState = EMF_LOTS_OF_SEQS;
+        {
+            ZSTD_inBuffer inBuf = { CNBuffer, CNBufferSize, 0 };
+            ZSTD_outBuffer outBuf = { dstBuf, dstBufSize, 0 };
+            size_t dResult;
+            CHECK_Z(ZSTD_compressStream(staticCCtx, &outBuf, &inBuf));
+            CHECK_Z(ZSTD_endStream(staticCCtx, &outBuf));
+            CHECK(inBuf.pos != inBuf.size, "EMF: inBuf.pos != inBuf.size");
+            dResult = ZSTD_decompress(checkBuf, checkBufSize, outBuf.dst, outBuf.pos);
+            CHECK(ZSTD_isError(dResult), "EMF: Decompression error: %s", ZSTD_getErrorName(dResult));
+            CHECK(dResult != CNBufferSize, "EMF: Corruption!");
+            CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
+        }
+
+        CHECK_Z(ZSTD_CCtx_reset(staticCCtx, ZSTD_reset_session_only));
+
+        // Check that compression with external sequence producer fails when expected
+        seqProdState = EMF_BIG_ERROR;
+        {
+            ZSTD_inBuffer inBuf = { CNBuffer, CNBufferSize, 0 };
+            ZSTD_outBuffer outBuf = { dstBuf, dstBufSize, 0 };
+            size_t const cResult = ZSTD_compressStream(staticCCtx, &outBuf, &inBuf);
+            CHECK(!ZSTD_isError(cResult), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(cResult) != ZSTD_error_sequenceProducer_failed,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(cResult)
+            );
+        }
+
+        free(dstBuf);
+        free(checkBuf);
+        free(cctxBuf);
+        ZSTD_freeCCtxParams(params);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
    DISPLAYLEVEL(3, "test%3i : Decoder should reject invalid frame header on legacy frames: ", testNb++);
    {
        const unsigned char compressed[] = { 0x26,0xb5,0x2f,0xfd,0x50,0x91,0xfd,0xd8,0xb5 };
@@ -2419,7 +2478,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)

    DISPLAYLEVEL(3, "test%3i : Test single-shot fallback for magicless mode: ", testNb++);
    {
-        // Aquire resources
+        // Acquire resources
        size_t const srcSize = COMPRESSIBLE_NOISE_LENGTH;
        void* src = malloc(srcSize);
        size_t const dstSize = ZSTD_compressBound(srcSize);
@@ -234,7 +234,7 @@ int gzwrite _Z_OF((gzFile, const void *, unsigned));

 int gzwrite(gzFile gz, const void *buf, unsigned len) {
    z_stream *strm;
-    unsigned char out[BUFLEN];
+    unsigned char out[BUFLEN] = { 0 };

    if (gz == NULL || !gz->write)
        return 0;
@@ -287,7 +287,7 @@ int gzclose _Z_OF((gzFile));

 int gzclose(gzFile gz) {
    z_stream *strm;
-    unsigned char out[BUFLEN];
+    unsigned char out[BUFLEN] = { 0 };

    if (gz == NULL)
        return Z_STREAM_ERROR;
@@ -64,6 +64,7 @@ local int gz_init(gz_statep state) {
        strm->next_out = state.state->out;
        state.state->x.next = strm->next_out;
    }
+
    return 0;
 }

@@ -223,7 +224,7 @@ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) {
            z_size_t n = (unsigned)-1;
            if (n > len)
                n = len;
-            state.state->strm.avail_in = (z_uInt)n;
+            state.state->strm.avail_in = (uInt)n;
            state.state->x.pos += n;
            if (gz_comp(state, Z_NO_FLUSH) == -1)
                return 0;
Author	SHA1	Message	Date
Yann ColletandGitHub	0ff651dd87	Merge pull request #4096 from tpetazzoni/build-fix facebook/zstd/commit / short-tests-0 (push) Has been cancelled facebook/zstd/commit / short-tests-1 (push) Has been cancelled facebook/zstd/commit / regression-test (push) Has been cancelled facebook/zstd/nightly / regression-test (push) Has been cancelled Scorecards supply-chain security / Scorecards analysis (push) Has been cancelled lib/libzstd.mk: fix typo in the definition of LIB_BINDIR	2024-07-16 14:09:43 -07:00
Thomas Petazzoni	5d63f186cc	lib/libzstd.mk: fix typo in the definition of LIB_BINDIR Commit `f4dbfce79c` ("define LIB_SRCDIR and LIB_BINDIR") significantly reworked the build logic, but in its introduction of LIB_BINDIR a typo was made. It was introduced as such: +LIB_SRCDIR ?= $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) +LIB_BINDIR ?= $(LIBSRC_DIR) But the definition of LIB_BINDIR has a typo: it should use $(LIB_SRCDIR) not $(LIBSRC_DIR). Due to this, $(LIB_BINDIR) is empty, therefore in programs/Makefile, -L$(LIB_BINDIR) is expanded to just -L, and consequently when trying to link the "zstd" binary with the libzstd library, it cannot find it: host/lib/gcc/powerpc64-buildroot-linux-gnu/13.3.0/../../../../powerpc64-buildroot-linux-gnu/bin/ld: cannot find -lzstd: No such file or directory This commit fixes the build by fixing this typo. Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>	2024-07-13 13:53:53 +02:00
Yann ColletandGitHub	ab02fd342f	Merge pull request #4086 from Adenilson/fix01 [zstd][dict] Ensure that dictionary training functions are fully reen…	2024-07-04 00:23:55 -07:00
Adenilson Cavalcanti	345bcb5ff7	[zstd][dict] Ensure that dictionary training functions are fully reentrant The two main functions used for dictionary training using the COVER algorithm require initialization of a COVER_ctx_t where a call to qsort() is performed. The issue is that the standard C99 qsort() function doesn't offer a way to pass an extra parameter for the comparison function callback (e.g. a pointer to a context) and currently zstd relies on a global static variable to hold a pointer to a context needed to perform the sort operation. If a zstd library user invokes either ZDICT_trainFromBuffer_cover or ZDICT_optimizeTrainFromBuffer_cover from multiple threads, the global context may be overwritten before/during the call/execution to qsort() in the initialization of the COVER_ctx_t, thus yielding to crashes and other bad things (Tm) as reported on issue #4045. Enters qsort_r(): it was designed to address precisely this situation, to quote from the documention [1]: "the comparison function does not need to use global variables to pass through arbitrary arguments, and is therefore reentrant and safe to use in threads." It is available with small variations for multiple OSes (GNU, BSD[2], Windows[3]), and the ISO C11 [4] standard features on annex B-21 qsort_s() as part of the <stdlib.h>. Let's hope that compilers eventually catch up with it. For now, we have to handle the small variations in function parameters for each platform. The current fix solves the problem by allowing each executing thread pass its own COVER_ctx_t instance to qsort_r(), removing the use of a global pointer and allowing the code to be reentrant. Unfortunately for *BSD, we cannot leverage qsort_r() given that its API has changed on newer versions of FreeBSD (14.0) and the other BSD variants (e.g. NetBSD, OpenBSD) don't implement it. For such cases we provide a fallback that will work only requiring support for compilers implementing support for C90. [1] https://man7.org/linux/man-pages/man3/qsort_r.3.html [2] https://man.freebsd.org/cgi/man.cgi?query=qsort_r [3] https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/qsort-s?view=msvc-170 [4] https://www.open-std.org/jtc1/sc22/wg14/www/docs/n1548.pdf	2024-07-01 23:52:31 -07:00
Yann ColletandGitHub	3de0541aef	Merge pull request #4079 from elasota/truncated-huff-state-error Throw error if Huffman weight initial states are truncated	2024-06-30 16:17:03 -07:00
Yann ColletandGitHub	4fe0ba0328	Merge pull request #4068 from DimitriPapadopoulos/codespell Fix new typos found by codespell	2024-06-29 16:04:05 -07:00
elasota	0938308ff6	Throw error if Huffman weight initial states are truncated	2024-06-20 17:46:16 -04:00
Dimitri Papadopoulos	44e83e9180	Fix typos not found by codespell	2024-06-20 20:16:25 +02:00
Dimitri Papadopoulos	2d736d9c50	Fix new typos found by codespell	2024-06-20 20:12:16 +02:00
Yann ColletandGitHub	17b5315016	Merge pull request #4076 from facebook/fix_macos_build fix macos build	2024-06-18 22:16:00 -07:00
Yann Collet	80170f6aad	fix macos build weird: after replacing the UNAME line with an identical one, it does work properly now(??). Possibly a case of hidden special character?	2024-06-18 20:24:00 -07:00
Elliot GorokhovskyandGitHub	3242ac598e	Expose size estimation helpers for Linux (#4064 )	2024-06-14 14:38:48 -04:00
Yann ColletandGitHub	6a77f213c4	Merge pull request #4073 from facebook/cygwin_install added a cygwin install test to check #4067	2024-06-12 11:58:54 -07:00
Yann Collet	d7a84a683f	added cygwin install test	2024-06-12 10:57:15 -07:00
Yann ColletandGitHub	aa76eeafb7	Merge pull request #4067 from QBos07/filterfix Fix $filter operants and Msys/Cygwin	2024-06-12 10:55:18 -07:00
Yann ColletandGitHub	7ddf633a8a	Merge pull request #4065 from embg/freebsd_ci Drop support for FreeBSD 13.2 CI	2024-06-12 09:54:25 -07:00
Quentin Boswank	f19c98228f	Fix $filter and Msys/Cygwin - switched the patter and input of $filter into the right places - added pattern wildcard to MSYS_NT & CYGWIN_NT as they change with windows versions - correctly identify MSYS2, even in an env like MINGW64	2024-06-05 18:37:27 +02:00
Yann Collet	2acf90431a	minor:doc: specify decompression behavior in presence of multiple concatenated frames directly at ZSTD_decompress() level.	2024-06-03 18:30:23 -07:00
Elliot GorokhovskyandGitHub	be6a182006	Unit test for external sequence producer + static CCtx + streaming (#4063 )	2024-06-03 12:42:27 -04:00
Elliot Gorokhovsky	949689facf	Drop FreeBSD 13.2 CI	2024-06-03 08:36:07 -07:00
Yann ColletandGitHub	f70fb7c870	Merge pull request #4046 from josepho0918/iar Improve support for IAR compiler with attributes and intrinsics	2024-05-29 15:33:19 -07:00
Yann ColletandGitHub	c5862f7731	Merge pull request #4054 from jbajic/fix-missing-newline [fix] Add newline when file exceeds 128KB	2024-05-29 15:32:54 -07:00
Federico MarescaandGitHub	5e9a6c2fe4	Refactor dictionary matchfinder index safety check (#4039 )	2024-05-29 12:35:24 -04:00
Jure Bajic	4c6a519fdd	[fix] Add newline when file exceeds 128KB	2024-05-26 11:34:24 +02:00
Yann ColletandGitHub	0e2ceb2d50	Merge pull request #4050 from Adenilson/fix_legacy_nullptr01 [fix] Add check on failed allocation in legacy/zstd_v06	2024-05-21 10:30:45 -07:00
Adenilson Cavalcanti	1872688e0a	[fix] Add check on failed allocation in legacy/zstd_v06 As reported by Ben Hawkes in #4026, a failure to allocate a zstd context would lead to a dereference of a NULL pointer due to a missing check on the returned result of ZSTDv06_createDCtx(). This patch fix the issue by adding a check for valid returned pointer.	2024-05-17 15:40:28 -07:00
Joseph Chen	5fadd8e6b1	revert FSE_readNCount_body attribute	2024-05-15 10:47:50 +08:00
Joseph Chen	2955d92ac0	Improve support for IAR compiler with attributes and intrinsics	2024-05-14 17:01:19 +08:00
Yann ColletandGitHub	78955f5f9d	Merge pull request #4040 from facebook/dependabot/github_actions/msys2/setup-msys2-2.23.0 Bump msys2/setup-msys2 from 2.22.0 to 2.23.0	2024-05-13 09:15:07 -07:00
dependabot[bot]andGitHub	4356192cb2	Bump msys2/setup-msys2 from 2.22.0 to 2.23.0 Bumps [msys2/setup-msys2](https://github.com/msys2/setup-msys2) from 2.22.0 to 2.23.0. - [Release notes](https://github.com/msys2/setup-msys2/releases) - [Changelog](https://github.com/msys2/setup-msys2/blob/main/CHANGELOG.md) - [Commits](https://github.com/msys2/setup-msys2/compare/cc11e9188b693c2b100158c3322424c4cc1dadea...d0e80f58dffbc64f6a3a1f43527d469b4fc7b6c8) --- updated-dependencies: - dependency-name: msys2/setup-msys2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2024-05-13 05:18:11 +00:00
Richard BarnesandNick Terrell	97291fc502	Increase x-compatibility	2024-04-29 09:19:24 -04:00
Richard BarnesandNick Terrell	d7cb47036c	Make zstd.h compatible with -Wzero-as-null-pointer-constant	2024-04-29 09:19:24 -04:00
Felix HandteandGitHub	fe67503f84	Merge pull request #4029 from facebook/dependabot/github_actions/github/codeql-action-3.25.1 Bump github/codeql-action from 3.24.10 to 3.25.1	2024-04-26 08:39:39 -07:00
Yann ColletandGitHub	f65dde9860	Merge pull request #4031 from facebook/docDStream update documentation of ZSTD_decompressStream()	2024-04-25 14:11:32 -07:00
Yann Collet	a86f5f3f33	update documentation of ZSTD_decompressStream() slightly more precise, by recommending to check the return value. fix #4030	2024-04-23 09:31:35 -07:00
dependabot[bot]andGitHub	68a6d9b9f6	Bump github/codeql-action from 3.24.10 to 3.25.1 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.10 to 3.25.1. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/4355270be187e1b672a7a1c7c7bae5afdc1ab94a...c7f9125735019aa87cfc361530512d50ea439c71) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2024-04-22 05:20:46 +00:00
Yann Collet	ff7a151f2e	update test name	2024-04-21 10:44:50 -07:00
Yann ColletandGitHub	eb541403c4	Merge pull request #4025 from alexlnkp/dev Fixed all memory leaks and almost all undefined behaviour	2024-04-21 10:14:33 -07:00
Yann ColletandGitHub	1232d4c45d	Merge pull request #3931 from bgilbert/pthread meson: don't add `-pthread` to static linking flags on Windows	2024-04-20 20:25:19 -07:00
Yann ColletandGitHub	da797d0a43	Merge pull request #4019 from b7f7/fix_cmake_rc_build fix missing include folder for resource compiler	2024-04-16 16:06:17 -07:00
Yann ColletandGitHub	fb5a5e46a5	Merge pull request #4020 from Ansuel/install-mt-pc Provide variant pkg-config file for multi-threaded static lib	2024-04-16 16:04:23 -07:00
Alex Murkoff	849b2ad907	removed freeing of the ptr	2024-04-14 20:44:12 +07:00
Alex Murkoff	0b24fc0a11	init out char array with all members as 0 x2	2024-04-14 17:57:53 +07:00
Alex Murkoff	b4ecf724b1	made initialize out char array with all elements set to 0 fixed where i made it to init with just the first one being set to 0	2024-04-14 17:45:53 +07:00
Alex Murkoff	75d0f66c87	reverted the addition of free at the gz_init()	2024-04-13 12:51:20 +07:00
Alex Murkoff	01cea2e1e2	fixed ISO C incompatibility for good this time... probably	2024-04-13 02:37:13 +07:00
Alex Murkoff	4f41631aa4	fixed ISO C incompatibility	2024-04-13 02:35:32 +07:00
Alex Murkoff	1d5e9705db	fixed memory leaks and almost all undefined behaviour	2024-04-13 02:24:14 +07:00
Felix HandteandGitHub	592de19843	Merge pull request #4022 from facebook/dependabot/github_actions/github/codeql-action-3.24.10 Bump github/codeql-action from 3.24.9 to 3.24.10	2024-04-12 08:51:02 -07:00
Benjamin Gilbert	5be2a8721d	meson: don't add -pthread to static linking flags on Windows Meson always returns -pthread in dependency('threads') on non-MSVC compilers. On Windows we use Windows threading primitives, so we don't need this. Avoid adding -pthread to libzstd's link flags, either as a Meson subproject or via pkg-config Libs.private, so the application doesn't inadvertently depend on winpthreads. Add a Meson MinGW cross-compile CI test that checks for this. It turns out that pzstd fails to build in that environment, so have the test skip building contrib for now.	2024-04-11 01:27:24 +09:00
Yann ColletandGitHub	9f42fa0a04	Merge pull request #4021 from pstef/dev Fix zlibWrapper build	2024-04-09 11:02:49 -07:00
Christian Marangi	f1f1ae369a	Provide variant pkg-config file for multi-threaded static lib Multi-threaded static library require -pthread to correctly link and works. The pkg-config we provide tho only works with dynamic multi-threaded library and won't provide the correct libs and cflags values if lib-mt is used. To handle this, introduce an env variable MT to permit advanced user to install and generate a correct pkg-config file for lib-mt or detect if lib-mt target is called. With MT env set on calling make install-pc, libzstd.pc.in is a pkg-config file for a multi-threaded static library. On calling make lib-mt, a libzstd.pc is generated for a multi-threaded static library as it's what asked by the user by forcing it. libzstd.pc is changed to PHONY to force regeneration of it on calling lib targets or install-pc to handle case where the same directory is used for mixed compilation. This was notice while migrating from meson to make build system where meson generates a correct .pc file while make doesn't. Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>	2024-04-08 17:44:52 +02:00
dependabot[bot]andGitHub	7968c661af	Bump github/codeql-action from 3.24.9 to 3.24.10 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.9 to 3.24.10. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/1b1aada464948af03b950897e5eb522f92603cc2...4355270be187e1b672a7a1c7c7bae5afdc1ab94a) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-04-08 05:47:58 +00:00
Piotr Paweł Stefaniak	71def59890	Fix zlibWrapper build Just after a clone I'm getting this: ~/zstd/zlibWrapper$ cc -c zstd_zlibwrapper.o gz*.c -lz -lzstd -DSTDC gzwrite.c: In function ‘gz_write’: gzwrite.c:226:43: error: ‘z_uInt’ undeclared (first use in this function); did you mean ‘uInt’? 226 \| state.state->strm.avail_in = (z_uInt)n; \| ^~~~~~ \| uInt gzwrite.c:226:43: note: each undeclared identifier is reported only once for each function it appears in gzwrite.c:226:50: error: expected ‘;’ before ‘n’ 226 \| state.state->strm.avail_in = (z_uInt)n; \| ^ \| ; z_uInt is never used directly, zconf.h redefines uInt to z_uInt under the condition that Z_PREFIX is set. All examples use uInt, and the type of avail_in is also uInt. In this commit I modify the cast to refer to the same type as the type of lvalue. Arguably, the real fix here is to handle possible overflows, but that's beyond the scope of this commit.	2024-04-07 13:04:26 +02:00
Yann ColletandGitHub	87af5fb2df	Merge pull request #3996 from facebook/fix_nodejs_version_warning fix nodejs deprecation warning	2024-04-05 23:06:21 -07:00
Yann ColletandGitHub	6f7f59b232	Merge pull request #4012 from elasota/spec-remove-fse-overflow Remove specification text stating that probability overflow is invalid	2024-04-05 23:06:06 -07:00
Yann ColletandGitHub	3fd4f3bf04	Merge pull request #4011 from facebook/fix4005 decompression errors always display the full origin filename	2024-04-05 23:05:53 -07:00
BadWolf	fd5f8106a5	fix missing include folder for resource compiler	2024-04-06 00:05:58 +02:00
elasota	8cff66f2f5	Remove text specifying probability overflow as invalid, the variable-size value encoding scheme makes this impossible.	2024-04-01 20:08:42 -04:00
Yann Collet	a2f145f059	decompression errors always display the full origin filename instead of the truncated size-limited version.	2024-04-01 11:49:21 -07:00
Yuriy ChernyshovandNick Terrell	72c16b187d	Fix building on windows-x86 if clang already includes [D101338](https://reviews.llvm.org/D101338) landed in 2021, so clang16 should have it	2024-04-01 09:53:08 -07:00
Yann ColletandGitHub	e3566d62ac	Merge pull request #3994 from sunpoet/dev Use md5sum rather than gmd5sum for FreeBSD	2024-03-31 23:09:46 -07:00
Yann ColletandGitHub	4b5a266d7b	Merge pull request #3997 from facebook/readme_bench_156 updated benchmarks for v1.5.6	2024-03-31 13:17:01 -07:00
Yann Collet	e0ee0fccf8	updated benchmarks for v1.5.6	2024-03-27 18:19:31 -07:00
Yann ColletandGitHub	aaad8e2e8c	Merge pull request #3988 from facebook/dependabot/github_actions/github/codeql-action-3.24.9 Bump github/codeql-action from 3.24.7 to 3.24.9	2024-03-27 18:05:42 -07:00
Yann Collet	ebf24b7b77	fix nodejs deprecation warning by updating msys2 action	2024-03-27 16:10:15 -07:00
Yann ColletandGitHub	80a9a2dfcd	Merge pull request #3987 from ManuelBlanc/msbuild_vswhere Use vswhere to find MSBuild; add VS2022 support	2024-03-27 15:48:40 -07:00
Po-Chuan Hsieh	103a85e6f6	Use md5sum rather than gmd5sum for FreeBSD Reference: https://man.freebsd.org/cgi/man.cgi?query=md5sum	2024-03-27 20:53:52 +08:00
Nick TerrellandNick Terrell	81a5e5d438	[fuzz] Turn off -Werror by default This was causing OSS-Fuzz errors, due to compiler differences. * Fix the issue * Also turn off -Werror so we don't fail fuzzer builds for warnings * Turn on -Werror in our CI	2024-03-26 16:34:36 -07:00
dependabot[bot]andGitHub	101e601c79	Bump github/codeql-action from 3.24.7 to 3.24.9 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.7 to 3.24.9. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/3ab4101902695724f9365a384f86c1074d94e18c...1b1aada464948af03b950897e5eb522f92603cc2) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-25 05:31:42 +00:00
ManuelBlanc	9c442d6fc2	Improve MSBuild search; add latest option	2024-03-23 18:34:07 -04:00
ManuelBlanc	65ab6c267e	Use vswhere to find MSBuild; add VS2022 support See: https://github.com/microsoft/vswhere/wiki/Find-MSBuild	2024-03-23 09:13:49 -04:00