From f17652931c4f8cebeecebbc4f4a45c47eb81ffca Mon Sep 17 00:00:00 2001
From: yhoogstrate <y.hoogstrate@gmail.com>
Date: Tue, 8 Feb 2022 14:06:00 +0100
Subject: [PATCH 001/197] seekable_format no header when compressing empty
 string to stream

---
 .../seekable_format/tests/seekable_tests.c    | 30 +++++++++++++++++++
 contrib/seekable_format/zstdseek_compress.c   |  2 +-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/contrib/seekable_format/tests/seekable_tests.c b/contrib/seekable_format/tests/seekable_tests.c
index a482638b9..cf98e734f 100644
--- a/contrib/seekable_format/tests/seekable_tests.c
+++ b/contrib/seekable_format/tests/seekable_tests.c
@@ -186,6 +186,36 @@ int main(int argc, const char** argv)
     }
     printf("Success!\n");
 
+
+    printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++);
+    { // compressing empty string should return a zstd header
+        char *data_in = (char *) malloc(255 * sizeof(char));
+        assert(data_in != NULL);
+        data_in = "\0";
+
+        char *data_out = (char *) malloc(255 * 255 * sizeof(char));
+        assert(data_out != NULL);
+
+        ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream();
+        ZSTD_seekable_initCStream(s, 1, 1, 1024 * 1024);
+
+        ZSTD_inBuffer input = { data_in, 0, 0 };
+        ZSTD_outBuffer output = { data_out, 255*255, 0 };
+
+        ZSTD_seekable_compressStream(s, &output, &input);
+        ZSTD_seekable_endStream(s, &output);
+
+        if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) {
+            printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] );
+
+            ZSTD_seekable_freeCStream(s);
+            goto _test_error;
+        }
+
+        ZSTD_seekable_freeCStream(s);
+    }
+    printf("Success!\n");
+
     /* TODO: Add more tests */
     printf("Finished tests\n");
     return 0;
diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c
index 242bd2ac3..7ec9bb577 100644
--- a/contrib/seekable_format/zstdseek_compress.c
+++ b/contrib/seekable_format/zstdseek_compress.c
@@ -350,7 +350,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
 
 size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)
 {
-    if (!zcs->writingSeekTable && zcs->frameDSize) {
+    if (!zcs->writingSeekTable) {
         const size_t endFrame = ZSTD_seekable_endFrame(zcs, output);
         if (ZSTD_isError(endFrame)) return endFrame;
         /* return an accurate size hint */

From b87f3102fff25cce98b1d3ef17a4a9a9748c8ea1 Mon Sep 17 00:00:00 2001
From: Nathan Moinvaziri <nathan@nathanm.com>
Date: Thu, 13 Oct 2022 10:02:27 -0700
Subject: [PATCH 002/197] Only run playTests.sh cmake test if in unix shell
 environment.

---
 build/cmake/tests/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 6c1ea298a..444a99f52 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -96,13 +96,14 @@ add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
 AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "$ENV{PLAYTESTS_FLAGS}"
     "Semicolon-separated list of flags to pass to the playTests.sh test")
 add_test(NAME playTests COMMAND sh -c "\"${TESTS_DIR}/playTests.sh\" ${ZSTD_PLAYTESTS_FLAGS}")
-if (ZSTD_BUILD_PROGRAMS)
+find_program(UNAME uname) # Run script only in unix shell environments
+if (ZSTD_BUILD_PROGRAMS AND UNAME)
     set_property(TEST playTests APPEND PROPERTY ENVIRONMENT
         "ZSTD_BIN=$<TARGET_FILE:zstd>"
         "DATAGEN_BIN=$<TARGET_FILE:datagen>"
         )
 else()
-    message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled")
+    message(STATUS "Disabling playTests.sh test because requirements not met")
     set_tests_properties(playTests PROPERTIES DISABLED YES)
 endif()
 

From a08fabd51a4191b9b1a2d580454ae6b7b805ee35 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 17 Oct 2022 10:24:29 -0700
Subject: [PATCH 003/197] Rough draft speed optimization

---
 lib/common/huf.h            |  2 +-
 lib/compress/huf_compress.c | 30 +++++++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index d79acd1d5..0db6fb369 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -173,7 +173,7 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
-#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
+#define HUF_OPTIMAL_DEPTH_THRESHOLD 3
 typedef enum {
    HUF_depth_fast, /** Use heuristic to find the table depth**/
    HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 54b58f191..d8bbaa6b4 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1258,7 +1258,8 @@ unsigned HUF_minTableLog(size_t srcSize, unsigned symbolCardinality)
 
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
 {
-    unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+    unsigned optLogGuess = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+    unsigned optLog = optLogGuess;
 
     if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/
         BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
@@ -1267,10 +1268,11 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
         unsigned huffLog;
         size_t maxBits, hSize, newSize;
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
+        const unsigned minTableLog = HUF_minTableLog(srcSize, symbolCardinality);
 
         if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog;
 
-        for (huffLog = HUF_minTableLog(srcSize, symbolCardinality); huffLog <= maxTableLog; huffLog++) {
+        for (huffLog = optLogGuess; huffLog >= minTableLog; huffLog--) {
             maxBits = HUF_buildCTable_wksp(table, count,
                                             maxSymbolValue, huffLog,
                                             workSpace, wkspSize);
@@ -1282,7 +1284,29 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
 
             newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
 
-            if (newSize < optSize) {
+            if (newSize > optSize) {
+                break;
+            } else {
+                optSize = newSize;
+                optLog = huffLog;
+            }
+        }
+
+        for (huffLog = optLogGuess + 1; huffLog <= maxTableLog; huffLog++) {
+            maxBits = HUF_buildCTable_wksp(table, count,
+                                            maxSymbolValue, huffLog,
+                                            workSpace, wkspSize);
+            if (ERR_isError(maxBits)) continue;
+
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits,
+                                              workSpace, wkspSize);
+            if (ERR_isError(hSize)) continue;
+
+            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+
+            if (newSize > optSize) {
+                break;
+            } else {
                 optSize = newSize;
                 optLog = huffLog;
             }

From 401331909e85a5590d03786130187545fe1e12df Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 24 Oct 2022 12:35:16 -0700
Subject: [PATCH 004/197] Commit for benchmarking

---
 lib/compress/huf_compress.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index b6905d3a6..4c3af94b7 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1266,10 +1266,11 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
         unsigned huffLog;
         size_t maxBits, hSize, newSize;
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
-        const unsigned minTableLog = HUF_minTableLog(srcSize, symbolCardinality);
+        const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
 
         if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog;
 
+        /* Search left of guess until size increases */
         for (huffLog = optLogGuess; huffLog >= minTableLog; huffLog--) {
             maxBits = HUF_buildCTable_wksp(table, count,
                                             maxSymbolValue, huffLog,
@@ -1290,6 +1291,7 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
             }
         }
 
+        /* Search right of estimate until size increases */
         for (huffLog = optLogGuess + 1; huffLog <= maxTableLog; huffLog++) {
             maxBits = HUF_buildCTable_wksp(table, count,
                                             maxSymbolValue, huffLog,

From db74d043d6de8268d7c23c8781c26ecef60a86b7 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Thu, 27 Oct 2022 10:20:44 -0700
Subject: [PATCH 005/197] Speed optimizations with macro

---
 contrib/freestanding_lib/freestanding.py      |  27 ++-
 contrib/linux-kernel/Makefile                 |   7 +-
 contrib/linux-kernel/linux.mk                 |  20 +-
 contrib/linux-kernel/test/Makefile            |   1 +
 .../linux-kernel/test/include/linux/module.h  |   2 +
 contrib/linux-kernel/zstd_common_module.c     |  32 +++
 lib/common/compiler.h                         |   6 +
 lib/common/huf.h                              |   2 +-
 lib/common/pool.c                             |   8 +-
 lib/compress/huf_compress.c                   |  59 ++---
 lib/compress/zstd_lazy.c                      | 229 +++++++++---------
 lib/decompress/zstd_ddict.c                   |   2 +-
 programs/zstd.1                               |   2 +-
 programs/zstd.1.md                            |   2 +-
 tests/fuzz/zstd_helpers.c                     |   5 +
 15 files changed, 237 insertions(+), 167 deletions(-)
 create mode 100644 contrib/linux-kernel/zstd_common_module.c

diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index 4a02dea14..4e0a944f1 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -431,7 +431,7 @@ class Freestanding(object):
             external_xxhash: bool, xxh64_state: Optional[str],
             xxh64_prefix: Optional[str], rewritten_includes: [(str, str)],
             defs: [(str, Optional[str])], replaces: [(str, str)],
-            undefs: [str], excludes: [str], seds: [str],
+            undefs: [str], excludes: [str], seds: [str], spdx: bool,
     ):
         self._zstd_deps = zstd_deps
         self._mem = mem
@@ -446,6 +446,7 @@ class Freestanding(object):
         self._undefs = undefs
         self._excludes = excludes
         self._seds = seds
+        self._spdx = spdx
 
     def _dst_lib_file_paths(self):
         """
@@ -640,6 +641,27 @@ class Freestanding(object):
         for sed in self._seds:
             self._process_sed(sed)
 
+    def _process_spdx(self):
+        if not self._spdx:
+            return
+        self._log("Processing spdx")
+        SPDX_C = "// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause\n"
+        SPDX_H_S = "/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */\n"
+        for filepath in self._dst_lib_file_paths():
+            file = FileLines(filepath)
+            if file.lines[0] == SPDX_C or file.lines[0] == SPDX_H_S:
+                continue
+            for line in file.lines:
+                if "SPDX-License-Identifier" in line:
+                    raise RuntimeError(f"Unexpected SPDX license identifier: {file.filename} {repr(line)}")
+            if file.filename.endswith(".c"):
+                file.lines.insert(0, SPDX_C)
+            elif file.filename.endswith(".h") or file.filename.endswith(".S"):
+                file.lines.insert(0, SPDX_H_S)
+            else:
+                raise RuntimeError(f"Unexpected file extension: {file.filename}")
+            file.write()
+
 
 
     def go(self):
@@ -651,6 +673,7 @@ class Freestanding(object):
         self._rewrite_includes()
         self._replace_xxh64_prefix()
         self._process_seds()
+        self._process_spdx()
 
 
 def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]:
@@ -689,6 +712,7 @@ def main(name, args):
     parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64")
     parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')")
     parser.add_argument("--sed", default=[], dest="seds", action="append", help="Apply a sed replacement. Format: `s/REGEX/FORMAT/[g]`. REGEX is a Python regex. FORMAT is a Python format string formatted by the regex dict.")
+    parser.add_argument("--spdx", action="store_true", help="Add SPDX License Identifiers")
     parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)")
     parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed multiple times)")
     parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition")
@@ -743,6 +767,7 @@ def main(name, args):
         args.undefs,
         args.excludes,
         args.seds,
+        args.spdx,
     ).go()
 
 if __name__ == "__main__":
diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index 47a431740..baa1f24c6 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -26,6 +26,7 @@ libzstd:
 		--rewrite-include '"(\.\./)?zstd_errors.h"=<linux/zstd_errors.h>' \
 		--sed 's,/\*\*\*,/* *,g' \
 		--sed 's,/\*\*,/*,g' \
+		--spdx \
 		-DZSTD_NO_INTRINSICS \
 		-DZSTD_NO_UNUSED_FUNCTIONS \
 		-DZSTD_LEGACY_SUPPORT=0 \
@@ -55,10 +56,13 @@ libzstd:
 		-DZSTD_HAVE_WEAK_SYMBOLS=0 \
 		-DZSTD_TRACE=0 \
 		-DZSTD_NO_TRACE \
+		-DZSTD_DISABLE_ASM \
 		-DZSTD_LINUX_KERNEL
+	rm linux/lib/zstd/decompress/huf_decompress_amd64.S
 	mv linux/lib/zstd/zstd.h linux/include/linux/zstd_lib.h
 	mv linux/lib/zstd/zstd_errors.h linux/include/linux/
 	cp linux_zstd.h linux/include/linux/zstd.h
+	cp zstd_common_module.c linux/lib/zstd
 	cp zstd_compress_module.c linux/lib/zstd
 	cp zstd_decompress_module.c linux/lib/zstd
 	cp decompress_sources.h linux/lib/zstd
@@ -102,4 +106,5 @@ test: libzstd
 
 .PHONY: clean
 clean:
-	$(RM) -rf linux test/test test/static_test
+	$(RM) -rf linux
+	$(MAKE) -C test clean
diff --git a/contrib/linux-kernel/linux.mk b/contrib/linux-kernel/linux.mk
index f6f3a8983..20f08c644 100644
--- a/contrib/linux-kernel/linux.mk
+++ b/contrib/linux-kernel/linux.mk
@@ -10,16 +10,10 @@
 # ################################################################
 obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
 obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
-
-ccflags-y += -Wno-error=deprecated-declarations
+obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o
 
 zstd_compress-y := \
 		zstd_compress_module.o \
-		common/debug.o \
-		common/entropy_common.o \
-		common/error_private.o \
-		common/fse_decompress.o \
-		common/zstd_common.o \
 		compress/fse_compress.o \
 		compress/hist.o \
 		compress/huf_compress.o \
@@ -35,13 +29,15 @@ zstd_compress-y := \
 
 zstd_decompress-y := \
 		zstd_decompress_module.o \
+		decompress/huf_decompress.o \
+		decompress/zstd_ddict.o \
+		decompress/zstd_decompress.o \
+		decompress/zstd_decompress_block.o \
+
+zstd_common-y := \
+		zstd_common_module.o \
 		common/debug.o \
 		common/entropy_common.o \
 		common/error_private.o \
 		common/fse_decompress.o \
 		common/zstd_common.o \
-		decompress/huf_decompress.o \
-		decompress/huf_decompress_amd64.o \
-		decompress/zstd_ddict.o \
-		decompress/zstd_decompress.o \
-		decompress/zstd_decompress_block.o \
diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile
index be82b3fba..53b0c2a65 100644
--- a/contrib/linux-kernel/test/Makefile
+++ b/contrib/linux-kernel/test/Makefile
@@ -45,4 +45,5 @@ clean:
 	$(RM) -f $(LINUX_ZSTDLIB)/*.o
 	$(RM) -f $(LINUX_ZSTDLIB)/**/*.o
 	$(RM) -f *.o *.a
+	$(RM) -f static_test
 	$(RM) -f test
diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h
index be6d20dae..63a28d57b 100644
--- a/contrib/linux-kernel/test/include/linux/module.h
+++ b/contrib/linux-kernel/test/include/linux/module.h
@@ -12,6 +12,8 @@
 
 #define EXPORT_SYMBOL(symbol)                                                  \
   void* __##symbol = symbol
+#define EXPORT_SYMBOL_GPL(symbol)                                              \
+  void* __##symbol = symbol
 #define MODULE_LICENSE(license)
 #define MODULE_DESCRIPTION(description)
 
diff --git a/contrib/linux-kernel/zstd_common_module.c b/contrib/linux-kernel/zstd_common_module.c
new file mode 100644
index 000000000..22686e367
--- /dev/null
+++ b/contrib/linux-kernel/zstd_common_module.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <linux/module.h>
+
+#include "common/huf.h"
+#include "common/fse.h"
+#include "common/zstd_internal.h"
+
+// Export symbols shared by compress and decompress into a common module
+
+#undef ZSTD_isError   /* defined within zstd_internal.h */
+EXPORT_SYMBOL_GPL(FSE_readNCount);
+EXPORT_SYMBOL_GPL(HUF_readStats);
+EXPORT_SYMBOL_GPL(HUF_readStats_wksp);
+EXPORT_SYMBOL_GPL(ZSTD_isError);
+EXPORT_SYMBOL_GPL(ZSTD_getErrorName);
+EXPORT_SYMBOL_GPL(ZSTD_getErrorCode);
+EXPORT_SYMBOL_GPL(ZSTD_customMalloc);
+EXPORT_SYMBOL_GPL(ZSTD_customCalloc);
+EXPORT_SYMBOL_GPL(ZSTD_customFree);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Zstd Common");
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 6c7100e83..42f289e0b 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -165,6 +165,12 @@
 #define UNLIKELY(x) (x)
 #endif
 
+#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)))
+#  define ZSTD_UNREACHABLE { assert(0), __builtin_unreachable(); }
+#else
+#  define ZSTD_UNREACHABLE { assert(0); }
+#endif
+
 /* disable warnings */
 #ifdef _MSC_VER    /* Visual Studio */
 #  include <intrin.h>                    /* For Visual 2005 */
diff --git a/lib/common/huf.h b/lib/common/huf.h
index dee99da77..595b2f6db 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -173,7 +173,7 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
-#define HUF_OPTIMAL_DEPTH_THRESHOLD 3
+#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
 typedef enum {
    HUF_depth_fast, /** Use heuristic to find the table depth**/
    HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
diff --git a/lib/common/pool.c b/lib/common/pool.c
index 5c1d07d35..bf21c57ed 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -12,7 +12,7 @@
 /* ======   Dependencies   ======= */
 #include "zstd_deps.h" /* size_t */
 #include "debug.h"     /* assert */
-#include "zstd_internal.h"  /* ZSTD_customMalloc, ZSTD_customFree */
+#include "zstd_internal.h"  /* ZSTD_customCalloc, ZSTD_customFree */
 #include "pool.h"
 
 /* ======   Compiler specifics   ====== */
@@ -126,7 +126,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
      * empty and full queues.
      */
     ctx->queueSize = queueSize + 1;
-    ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem);
+    ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem);
     ctx->queueHead = 0;
     ctx->queueTail = 0;
     ctx->numThreadsBusy = 0;
@@ -140,7 +140,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
     }
     ctx->shutdown = 0;
     /* Allocate space for the thread handles */
-    ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
+    ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
     ctx->threadCapacity = 0;
     ctx->customMem = customMem;
     /* Check for errors */
@@ -220,7 +220,7 @@ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
         return 0;
     }
     /* numThreads > threadCapacity */
-    {   ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
+    {   ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
         if (!threadPool) return 1;
         /* replace existing thread pool */
         ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 4c3af94b7..a6a5a257c 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1253,63 +1253,54 @@ unsigned HUF_minTableLog(unsigned symbolCardinality)
     return minBitsSymbols;
 }
 
+#define ESTIMATE_TOTAL_SIZE(huffLog) {\
+            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, huffLog, workSpace, wkspSize);\
+            if (ERR_isError(maxBits)) continue;\
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);\
+            if (ERR_isError(hSize)) continue;\
+            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;\
+            }\
+
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
 {
-    unsigned optLogGuess = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
-    unsigned optLog = optLogGuess;
+    unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
 
     if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/
         BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
         size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
-        size_t optSize = ((size_t) ~0);
-        unsigned huffLog;
         size_t maxBits, hSize, newSize;
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
         const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
+        size_t optSize = ((size_t) ~0);
+        unsigned optLogGuess = optLog;
 
-        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog;
+        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/
 
-        /* Search left of guess until size increases */
-        for (huffLog = optLogGuess; huffLog >= minTableLog; huffLog--) {
-            maxBits = HUF_buildCTable_wksp(table, count,
-                                            maxSymbolValue, huffLog,
-                                            workSpace, wkspSize);
-            if (ERR_isError(maxBits)) continue;
-
-            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits,
-                                              workSpace, wkspSize);
-            if (ERR_isError(hSize)) continue;
-
-            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+        /* Search below estimate log until size increases */
+        for (; optLogGuess >= minTableLog; optLogGuess--) {
+            ESTIMATE_TOTAL_SIZE(optLogGuess);
 
             if (newSize > optSize) {
                 break;
-            } else {
-                optSize = newSize;
-                optLog = huffLog;
             }
+            optSize = newSize;
+            optLog = optLogGuess;
         }
 
-        /* Search right of estimate until size increases */
-        for (huffLog = optLogGuess + 1; huffLog <= maxTableLog; huffLog++) {
-            maxBits = HUF_buildCTable_wksp(table, count,
-                                            maxSymbolValue, huffLog,
-                                            workSpace, wkspSize);
-            if (ERR_isError(maxBits)) continue;
+        if (optSize < ((size_t) ~0)) {
+            return optLog;
+        }
 
-            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits,
-                                              workSpace, wkspSize);
-            if (ERR_isError(hSize)) continue;
-
-            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+        /* Search above estimate log until size increases */
+        for (; optLogGuess <= maxTableLog; optLogGuess++) {
+            ESTIMATE_TOTAL_SIZE(optLogGuess);
 
             if (newSize > optSize) {
                 break;
-            } else {
-                optSize = newSize;
-                optLog = huffLog;
             }
+            optSize = newSize;
+            optLog = optLogGuess;
         }
     }
     assert(optLog <= HUF_TABLELOG_MAX);
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 3e2ee1dda..2ee9cf630 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1317,14 +1317,10 @@ size_t ZSTD_RowFindBestMatch(
 }
 
 
-typedef size_t (*searchMax_f)(
-                    ZSTD_matchState_t* ms,
-                    const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
-
 /**
- * This struct contains the functions necessary for lazy to search.
- * Currently, that is only searchMax. However, it is still valuable to have the
- * VTable because this makes it easier to add more functions to the VTable later.
+ * Generate search functions templated on (dictMode, mls, rowLog).
+ * These functions are outlined for code size & compilation time.
+ * ZSTD_searchMax() dispatches to the correct implementation function.
  *
  * TODO: The start of the search function involves loading and calculating a
  * bunch of constants from the ZSTD_matchState_t. These computations could be
@@ -1342,25 +1338,25 @@ typedef size_t (*searchMax_f)(
  * the single segment loop. It should go in searchMax instead of its own
  * function to avoid having multiple virtual function calls per search.
  */
-typedef struct {
-    searchMax_f searchMax;
-} ZSTD_LazyVTable;
 
-#define GEN_ZSTD_BT_VTABLE(dictMode, mls)                                             \
-    static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls(                            \
-            ZSTD_matchState_t* ms,                                                    \
-            const BYTE* ip, const BYTE* const iLimit,                                 \
-            size_t* offBasePtr)                                                       \
-    {                                                                                 \
-        assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls);                          \
-        return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode);\
-    }                                                                                 \
-    static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = {                 \
-        ZSTD_BtFindBestMatch_##dictMode##_##mls                                       \
-    };
+#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls
+#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls
+#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog
 
-#define GEN_ZSTD_HC_VTABLE(dictMode, mls)                                             \
-    static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls(                            \
+#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE
+
+#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls)                                           \
+    ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)(                      \
+            ZSTD_matchState_t* ms,                                                     \
+            const BYTE* ip, const BYTE* const iLimit,                                  \
+            size_t* offBasePtr)                                                        \
+    {                                                                                  \
+        assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls);                           \
+        return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \
+    }                                                                                  \
+
+#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls)                                          \
+    ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)(                     \
             ZSTD_matchState_t* ms,                                                    \
             const BYTE* ip, const BYTE* const iLimit,                                 \
             size_t* offsetPtr)                                                        \
@@ -1368,12 +1364,9 @@ typedef struct {
         assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls);                          \
         return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \
     }                                                                                 \
-    static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = {                 \
-        ZSTD_HcFindBestMatch_##dictMode##_##mls                                       \
-    };
 
-#define GEN_ZSTD_ROW_VTABLE(dictMode, mls, rowLog)                                             \
-    static size_t ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog(                         \
+#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)                                          \
+    ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(                     \
             ZSTD_matchState_t* ms,                                                             \
             const BYTE* ip, const BYTE* const iLimit,                                          \
             size_t* offsetPtr)                                                                 \
@@ -1382,9 +1375,6 @@ typedef struct {
         assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog);                               \
         return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \
     }                                                                                          \
-    static const ZSTD_LazyVTable ZSTD_RowVTable_##dictMode##_##mls##_##rowLog = {              \
-        ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog                                    \
-    };
 
 #define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \
     X(dictMode, mls, 4)                        \
@@ -1407,83 +1397,102 @@ typedef struct {
     X(__VA_ARGS__, dictMatchState)      \
     X(__VA_ARGS__, dedicatedDictSearch)
 
-/* Generate Row VTables for each combination of (dictMode, mls, rowLog) */
-ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_VTABLE)
-/* Generate Binary Tree VTables for each combination of (dictMode, mls) */
-ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE)
-/* Generate Hash Chain VTables for each combination of (dictMode, mls) */
-ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE)
+/* Generate row search fns for each combination of (dictMode, mls, rowLog) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN)
+/* Generate binary Tree search fns for each combination of (dictMode, mls) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN)
+/* Generate hash chain search fns for each combination of (dictMode, mls) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN)
 
-#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \
-    {                                      \
-        &ZSTD_BtVTable_##dictMode##_4,     \
-        &ZSTD_BtVTable_##dictMode##_5,     \
-        &ZSTD_BtVTable_##dictMode##_6      \
+typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e;
+
+#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls)                         \
+    case mls:                                                             \
+        return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
+#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls)                         \
+    case mls:                                                             \
+        return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
+#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog)                         \
+    case rowLog:                                                                   \
+        return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr);
+
+#define ZSTD_SWITCH_MLS(X, dictMode)   \
+    switch (mls) {                     \
+        ZSTD_FOR_EACH_MLS(X, dictMode) \
     }
 
-#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \
-    {                                      \
-        &ZSTD_HcVTable_##dictMode##_4,     \
-        &ZSTD_HcVTable_##dictMode##_5,     \
-        &ZSTD_HcVTable_##dictMode##_6      \
-    }
+#define ZSTD_SWITCH_ROWLOG(dictMode, mls)                                    \
+    case mls:                                                                \
+        switch (rowLog) {                                                    \
+            ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \
+        }                                                                    \
+        ZSTD_UNREACHABLE;                                                    \
+        break;
 
-#define GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, mls) \
-    {                                             \
-        &ZSTD_RowVTable_##dictMode##_##mls##_4,   \
-        &ZSTD_RowVTable_##dictMode##_##mls##_5,   \
-        &ZSTD_RowVTable_##dictMode##_##mls##_6    \
-    }
+#define ZSTD_SWITCH_SEARCH_METHOD(dictMode)                       \
+    switch (searchMethod) {                                       \
+        case search_hashChain:                                    \
+            ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \
+            break;                                                \
+        case search_binaryTree:                                   \
+            ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \
+            break;                                                \
+        case search_rowHash:                                      \
+            ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode)         \
+            break;                                                \
+    }                                                             \
+    ZSTD_UNREACHABLE;
 
-#define GEN_ZSTD_ROW_VTABLE_ARRAY(dictMode)      \
-    {                                            \
-        GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 4), \
-        GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 5), \
-        GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 6)  \
-    }
-
-#define GEN_ZSTD_VTABLE_ARRAY(X) \
-    {                            \
-        X(noDict),               \
-        X(extDict),              \
-        X(dictMatchState),       \
-        X(dedicatedDictSearch)   \
+/**
+ * Searches for the longest match at @p ip.
+ * Dispatches to the correct implementation function based on the
+ * (searchMethod, dictMode, mls, rowLog). We use switch statements
+ * here instead of using an indirect function call through a function
+ * pointer because after Spectre and Meltdown mitigations, indirect
+ * function calls can be very costly, especially in the kernel.
+ *
+ * NOTE: dictMode and searchMethod should be templated, so those switch
+ * statements should be optimized out. Only the mls & rowLog switches
+ * should be left.
+ *
+ * @param ms The match state.
+ * @param ip The position to search at.
+ * @param iend The end of the input data.
+ * @param[out] offsetPtr Stores the match offset into this pointer.
+ * @param mls The minimum search length, in the range [4, 6].
+ * @param rowLog The row log (if applicable), in the range [4, 6].
+ * @param searchMethod The search method to use (templated).
+ * @param dictMode The dictMode (templated).
+ *
+ * @returns The length of the longest match found, or < mls if no match is found.
+ * If a match is found its offset is stored in @p offsetPtr.
+ */
+FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
+    ZSTD_matchState_t* ms,
+    const BYTE* ip,
+    const BYTE* iend,
+    size_t* offsetPtr,
+    U32 const mls,
+    U32 const rowLog,
+    searchMethod_e const searchMethod,
+    ZSTD_dictMode_e const dictMode)
+{
+    if (dictMode == ZSTD_noDict) {
+        ZSTD_SWITCH_SEARCH_METHOD(noDict)
+    } else if (dictMode == ZSTD_extDict) {
+        ZSTD_SWITCH_SEARCH_METHOD(extDict)
+    } else if (dictMode == ZSTD_dictMatchState) {
+        ZSTD_SWITCH_SEARCH_METHOD(dictMatchState)
+    } else if (dictMode == ZSTD_dedicatedDictSearch) {
+        ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch)
     }
+    ZSTD_UNREACHABLE;
+    return 0;
+}
 
 /* *******************************
 *  Common parser - lazy strategy
 *********************************/
-typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e;
-
-/**
- * This table is indexed first by the four ZSTD_dictMode_e values, and then
- * by the two searchMethod_e values. NULLs are placed for configurations
- * that should never occur (extDict modes go to the other implementation
- * below and there is no DDSS for binary tree search yet).
- */
-
-static ZSTD_LazyVTable const*
-ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode)
-{
-    /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */
-    ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY);
-    ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY);
-    /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */
-    ZSTD_LazyVTable const* const rowVTables[4][3][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_ROW_VTABLE_ARRAY);
-
-    U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch));
-    U32 const rowLog = MAX(4, MIN(6, ms->cParams.searchLog));
-    switch (searchMethod) {
-        case search_hashChain:
-            return hcVTables[dictMode][mls - 4];
-        case search_binaryTree:
-            return btVTables[dictMode][mls - 4];
-        case search_rowHash:
-            return rowVTables[dictMode][mls - 4][rowLog - 4];
-        default:
-            return NULL;
-    }
-}
 
 FORCE_INLINE_TEMPLATE size_t
 ZSTD_compressBlock_lazy_generic(
@@ -1501,8 +1510,9 @@ ZSTD_compressBlock_lazy_generic(
     const BYTE* const base = ms->window.base;
     const U32 prefixLowestIndex = ms->window.dictLimit;
     const BYTE* const prefixLowest = base + prefixLowestIndex;
+    const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
+    const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
 
-    searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax;
     U32 offset_1 = rep[0], offset_2 = rep[1];
     U32 offsetSaved1 = 0, offsetSaved2 = 0;
 
@@ -1519,8 +1529,6 @@ ZSTD_compressBlock_lazy_generic(
                                      0;
     const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
 
-    assert(searchMax != NULL);
-
     DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod);
     ip += (dictAndPrefixLength == 0);
     if (dictMode == ZSTD_noDict) {
@@ -1538,7 +1546,6 @@ ZSTD_compressBlock_lazy_generic(
     }
 
     if (searchMethod == search_rowHash) {
-        const U32 rowLog = MAX(4, MIN(6, ms->cParams.searchLog));
         ZSTD_row_fillHashCache(ms, base, rowLog,
                             MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
                             ms->nextToUpdate, ilimit);
@@ -1579,7 +1586,7 @@ ZSTD_compressBlock_lazy_generic(
 
         /* first search (depth 0) */
         {   size_t offbaseFound = 999999999;
-            size_t const ml2 = searchMax(ms, ip, iend, &offbaseFound);
+            size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode);
             if (ml2 > matchLength)
                 matchLength = ml2, start = ip, offBase = offbaseFound;
         }
@@ -1618,7 +1625,7 @@ ZSTD_compressBlock_lazy_generic(
                 }
             }
             {   size_t ofbCandidate=999999999;
-                size_t const ml2 = searchMax(ms, ip, iend, &ofbCandidate);
+                size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
                 int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
                 int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
                 if ((ml2 >= 4) && (gain2 > gain1)) {
@@ -1654,7 +1661,7 @@ ZSTD_compressBlock_lazy_generic(
                     }
                 }
                 {   size_t ofbCandidate=999999999;
-                    size_t const ml2 = searchMax(ms, ip, iend, &ofbCandidate);
+                    size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
                     int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
                     int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
                     if ((ml2 >= 4) && (gain2 > gain1)) {
@@ -1899,9 +1906,9 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
     const BYTE* const dictEnd  = dictBase + dictLimit;
     const BYTE* const dictStart  = dictBase + ms->window.lowLimit;
     const U32 windowLog = ms->cParams.windowLog;
-    const U32 rowLog = ms->cParams.searchLog < 5 ? 4 : 5;
+    const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
+    const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
 
-    searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax;
     U32 offset_1 = rep[0], offset_2 = rep[1];
 
     DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod);
@@ -1943,7 +1950,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 
         /* first search (depth 0) */
         {   size_t ofbCandidate = 999999999;
-            size_t const ml2 = searchMax(ms, ip, iend, &ofbCandidate);
+            size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
             if (ml2 > matchLength)
                 matchLength = ml2, start = ip, offBase = ofbCandidate;
         }
@@ -1978,7 +1985,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 
             /* search match, depth 1 */
             {   size_t ofbCandidate = 999999999;
-                size_t const ml2 = searchMax(ms, ip, iend, &ofbCandidate);
+                size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
                 int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
                 int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
                 if ((ml2 >= 4) && (gain2 > gain1)) {
@@ -2010,7 +2017,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 
                 /* search match, depth 2 */
                 {   size_t ofbCandidate = 999999999;
-                    size_t const ml2 = searchMax(ms, ip, iend, &ofbCandidate);
+                    size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
                     int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
                     int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
                     if ((ml2 >= 4) && (gain2 > gain1)) {
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index 889764a5e..6ffa35f6e 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -240,5 +240,5 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;
-    return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
+    return ddict->dictID;
 }
diff --git a/programs/zstd.1 b/programs/zstd.1
index a2bf7fd2f..5cf3c6bb1 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -144,7 +144,7 @@ Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be au
 \fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
 .
 .IP "\(bu" 4
-\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, Zstandard uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
+\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
 .
 .IP
 This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\.
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 3ab8404ee..37c2ba187 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -183,7 +183,7 @@ the last one takes effect.
     The decoder will have to rely on implicit knowledge about which dictionary to use,
     it won't be able to check if it's correct.
 * `-M#`, `--memory=#`:
-    Set a memory usage limit. By default, Zstandard uses 128 MB for decompression
+    Set a memory usage limit. By default, `zstd` uses 128 MB for decompression
     as the maximum amount of memory the decompressor is allowed to use, but you can
     override this manually if need be in either direction (i.e. you can increase or
     decrease it).
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 4f8727df9..b4a6509db 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -89,8 +89,13 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
             ZSTD_LDM_HASHRATELOG_MAX, producer);
     /* Set misc parameters */
+#ifndef ZSTD_MULTITHREAD
+    setRand(cctx, ZSTD_c_nbWorkers, 0, 0, producer);
+    setRand(cctx, ZSTD_c_rsyncable, 0, 0, producer);
+#else
     setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer);
     setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer);
+#endif
     setRand(cctx, ZSTD_c_useRowMatchFinder, 0, 2, producer);
     setRand(cctx, ZSTD_c_enableDedicatedDictSearch, 0, 1, producer);
     setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer);

From c2638212af253e8d9c9161743812b5e39056e0f5 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Thu, 27 Oct 2022 13:13:17 -0700
Subject: [PATCH 006/197] Change threshold for benchmarking

---
 lib/common/huf.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index 595b2f6db..dee99da77 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -173,7 +173,7 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
-#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
+#define HUF_OPTIMAL_DEPTH_THRESHOLD 3
 typedef enum {
    HUF_depth_fast, /** Use heuristic to find the table depth**/
    HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/

From 9730aa432697ecc15014c2006e26c8e3778e1bd2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 8 Dec 2022 18:39:58 +0000
Subject: [PATCH 007/197] Bump actions/upload-artifact from 1 to 3

Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 1 to 3.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/v1...v3)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/dev-long-tests.yml | 2 +-
 .github/workflows/scorecards.yml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index ac34c7165..dc1c5b355 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -279,7 +279,7 @@ jobs:
         dry-run: false
         sanitizer: ${{ matrix.sanitizer }}
     - name: Upload Crash
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v3
       if: failure() && steps.build.outcome == 'success'
       with:
         name: ${{ matrix.sanitizer }}-artifacts
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 1c19b48b2..e3d9a2e54 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -51,7 +51,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # tag=v3.0.0
+        uses: actions/upload-artifact@v3 # tag=v3.0.0
         with:
           name: SARIF file
           path: results.sarif

From aece0f258adfbd72401707d39094a477106af07a Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 13 Dec 2022 08:15:16 -0800
Subject: [PATCH 008/197] free memory in test case

---
 .../seekable_format/tests/seekable_tests.c    | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/contrib/seekable_format/tests/seekable_tests.c b/contrib/seekable_format/tests/seekable_tests.c
index cf98e734f..1bb2d0e81 100644
--- a/contrib/seekable_format/tests/seekable_tests.c
+++ b/contrib/seekable_format/tests/seekable_tests.c
@@ -189,18 +189,18 @@ int main(int argc, const char** argv)
 
     printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++);
     { // compressing empty string should return a zstd header
-        char *data_in = (char *) malloc(255 * sizeof(char));
-        assert(data_in != NULL);
-        data_in = "\0";
-
-        char *data_out = (char *) malloc(255 * 255 * sizeof(char));
-        assert(data_out != NULL);
+        size_t const capacity = 255;
+        char* inBuffer = malloc(capacity);
+        assert(inBuffer != NULL);
+        inBuffer[0] = '\0';
+        void* const outBuffer = malloc(capacity);
+        assert(outBuffer != NULL);
 
         ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream();
-        ZSTD_seekable_initCStream(s, 1, 1, 1024 * 1024);
+        ZSTD_seekable_initCStream(s, 1, 1, 255);
 
-        ZSTD_inBuffer input = { data_in, 0, 0 };
-        ZSTD_outBuffer output = { data_out, 255*255, 0 };
+        ZSTD_inBuffer input = { .src=inBuffer, .pos=0, .size=0 };
+        ZSTD_outBuffer output = { .dst=outBuffer, .pos=0, .size=capacity };
 
         ZSTD_seekable_compressStream(s, &output, &input);
         ZSTD_seekable_endStream(s, &output);
@@ -208,10 +208,14 @@ int main(int argc, const char** argv)
         if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) {
             printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] );
 
+            free(inBuffer);
+            free(outBuffer);
             ZSTD_seekable_freeCStream(s);
             goto _test_error;
         }
 
+        free(inBuffer);
+        free(outBuffer);
         ZSTD_seekable_freeCStream(s);
     }
     printf("Success!\n");

From 69ec75f0d5e028aa34b3b51f990f38a18a0b7783 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 13 Dec 2022 08:35:20 -0800
Subject: [PATCH 009/197] fix window resizing edge case

---
 lib/compress/zstd_compress.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 5bc7a9f3f..0069a7b1b 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1370,8 +1370,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     }
 
     /* resize windowLog if input is small enough, to use less memory */
-    if ( (srcSize < maxWindowResize)
-      && (dictSize < maxWindowResize) )  {
+    if ( (srcSize <= maxWindowResize)
+      && (dictSize <= maxWindowResize) )  {
         U32 const tSize = (U32)(srcSize + dictSize);
         static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
         U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :

From d081d98ae7239385fe5e62d41898ef4b3c5e06ea Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Tue, 13 Dec 2022 12:26:34 -0800
Subject: [PATCH 010/197] Fix m68k CI tests on Github Actions

seems there is a bug in the qemu version shipped with Ubuntu 22.04
---
 .github/workflows/dev-short-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index a31eec6bb..08df1c07c 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -275,7 +275,7 @@ jobs:
 
   qemu-consistency:
     name: QEMU ${{ matrix.name }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
       matrix:

From 32bb667138028f82170451a3260caffa51544e48 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 13 Dec 2022 12:45:41 -0800
Subject: [PATCH 011/197] added test to cli-tests

---
 tests/cli-tests/compression/window-resize.sh             | 9 +++++++++
 tests/cli-tests/compression/window-resize.sh.stderr.glob | 1 +
 tests/cli-tests/compression/window-resize.sh.stdout.glob | 3 +++
 3 files changed, 13 insertions(+)
 create mode 100755 tests/cli-tests/compression/window-resize.sh
 create mode 100644 tests/cli-tests/compression/window-resize.sh.stderr.glob
 create mode 100644 tests/cli-tests/compression/window-resize.sh.stdout.glob

diff --git a/tests/cli-tests/compression/window-resize.sh b/tests/cli-tests/compression/window-resize.sh
new file mode 100755
index 000000000..3b5e6fe24
--- /dev/null
+++ b/tests/cli-tests/compression/window-resize.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+datagen -g1G > file
+zstd --long=31 -1 --single-thread --no-content-size -f file
+zstd -l -v file.zst
+
+# We want to ignore stderr (its outputting "*** zstd command line interface
+# 64-bits v1.5.3, by Yann Collet ***")
+
+rm file file.zst
diff --git a/tests/cli-tests/compression/window-resize.sh.stderr.glob b/tests/cli-tests/compression/window-resize.sh.stderr.glob
new file mode 100644
index 000000000..eb1ae458f
--- /dev/null
+++ b/tests/cli-tests/compression/window-resize.sh.stderr.glob
@@ -0,0 +1 @@
+...
diff --git a/tests/cli-tests/compression/window-resize.sh.stdout.glob b/tests/cli-tests/compression/window-resize.sh.stdout.glob
new file mode 100644
index 000000000..313d216e1
--- /dev/null
+++ b/tests/cli-tests/compression/window-resize.sh.stdout.glob
@@ -0,0 +1,3 @@
+...
+Window Size: 1.000 GiB (1073741824 B)
+...

From 9b9ad5af3f78e517723277262cfd601d00ce4cd9 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 13 Dec 2022 13:52:54 -0800
Subject: [PATCH 012/197] use .ignore for stderr output in window-resize test
 case

---
 tests/cli-tests/compression/window-resize.sh.stderr.glob   | 1 -
 tests/cli-tests/compression/window-resize.sh.stderr.ignore | 0
 2 files changed, 1 deletion(-)
 delete mode 100644 tests/cli-tests/compression/window-resize.sh.stderr.glob
 create mode 100644 tests/cli-tests/compression/window-resize.sh.stderr.ignore

diff --git a/tests/cli-tests/compression/window-resize.sh.stderr.glob b/tests/cli-tests/compression/window-resize.sh.stderr.glob
deleted file mode 100644
index eb1ae458f..000000000
--- a/tests/cli-tests/compression/window-resize.sh.stderr.glob
+++ /dev/null
@@ -1 +0,0 @@
-...
diff --git a/tests/cli-tests/compression/window-resize.sh.stderr.ignore b/tests/cli-tests/compression/window-resize.sh.stderr.ignore
new file mode 100644
index 000000000..e69de29bb

From e767d5c7c144ab911842229a9a8fee527d9616e9 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 13 Dec 2022 15:03:23 -0800
Subject: [PATCH 013/197] [contrib][linux-kernel] Fix stack detection for newer
 gcc

Newer gcc versions were getting smart and omitting the `memset()`.
Get around this issue by outlining the `memset()` into a different
function. This test is still hacky, but it works...
---
 contrib/linux-kernel/test/test.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c
index 6cd1730bb..67d248e0c 100644
--- a/contrib/linux-kernel/test/test.c
+++ b/contrib/linux-kernel/test/test.c
@@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) {
   asm volatile("" : "+r"(x));
 }
 
+static void __attribute__((noinline)) fill_stack(void) {
+  memset(g_stack, 0x33, 8192);
+}
+
 static void __attribute__((noinline)) set_stack(void) {
 
   char stack[8192];
   g_stack = stack;
-  memset(g_stack, 0x33, 8192);
   use(g_stack);
 }
 
@@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) {
 
 static void test_stack_usage(test_data_t const *data) {
   set_stack();
+  fill_stack();
   test_f2fs();
   test_btrfs(data);
   test_decompress_unzstd(data);

From e1e82f74f1f992acb6d98577e167d2e7cfe45f70 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Tue, 13 Dec 2022 17:45:05 -0800
Subject: [PATCH 014/197] Reserve two fields in ZSTD_frameHeader

---
 lib/zstd.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/zstd.h b/lib/zstd.h
index 1867efc93..1dff31b4e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2573,6 +2573,8 @@ typedef struct {
     unsigned headerSize;
     unsigned dictID;
     unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
 } ZSTD_frameHeader;
 
 /*! ZSTD_getFrameHeader() :

From c43da3d6059a277ab5d76236bdb66ba2b36264a1 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Tue, 13 Dec 2022 18:01:32 -0800
Subject: [PATCH 015/197] Fix C90 compat

---
 lib/decompress/zstd_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 9fe326e73..df831b774 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1549,7 +1549,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
  *  ZSTD_getFrameHeader(), which will provide a more precise error code. */
 unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
 {
-    ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 };
+    ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
     size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
     if (ZSTD_isError(hError)) return 0;
     return zfp.dictID;

From 4dffc35f2edf2e5fa53c3ea9c5c975fe2f1d6ebc Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Wed, 14 Dec 2022 06:58:35 -0800
Subject: [PATCH 016/197] Convert references to https from http

---
 README.md                                     |  8 ++--
 build/cmake/CMakeLists.txt                    |  2 +-
 build/cmake/tests/CMakeLists.txt              |  2 +-
 build/meson/lib/meson.build                   |  2 +-
 .../linux-kernel/test/include/linux/xxhash.h  |  4 +-
 contrib/pzstd/README.md                       |  2 +-
 contrib/pzstd/utils/FileSystem.h              | 14 +++----
 contrib/seekable_format/zstdseek_decompress.c |  2 +-
 doc/README.md                                 |  2 +-
 doc/educational_decoder/zstd_decompress.c     |  2 +-
 doc/zstd_compression_format.md                |  8 ++--
 lib/common/mem.h                              |  2 +-
 lib/common/threading.c                        |  3 +-
 lib/common/threading.h                        |  3 +-
 lib/common/xxhash.c                           |  2 +-
 lib/common/xxhash.h                           |  8 ++--
 lib/compress/fse_compress.c                   |  2 +-
 lib/legacy/zstd_v01.c                         |  4 +-
 lib/legacy/zstd_v02.c                         | 22 +++++------
 lib/legacy/zstd_v03.c                         | 22 +++++------
 lib/legacy/zstd_v04.c                         | 18 ++++-----
 lib/legacy/zstd_v05.c                         | 28 +++++++-------
 lib/legacy/zstd_v06.c                         | 38 +++++++++----------
 lib/legacy/zstd_v07.c                         | 30 +++++++--------
 lib/libzstd.pc.in                             |  4 +-
 programs/Makefile                             |  2 +-
 programs/README.md                            |  2 +-
 programs/fileio.c                             |  4 +-
 programs/platform.h                           |  8 ++--
 tests/gzip/gzip-env.sh                        |  2 +-
 tests/gzip/helin-segv.sh                      |  2 +-
 tests/gzip/help-version.sh                    |  2 +-
 tests/gzip/hufts.sh                           |  2 +-
 tests/gzip/init.sh                            |  2 +-
 tests/gzip/keep.sh                            |  2 +-
 tests/gzip/list.sh                            |  2 +-
 tests/gzip/memcpy-abuse.sh                    |  2 +-
 tests/gzip/mixed.sh                           |  2 +-
 tests/gzip/null-suffix-clobber.sh             |  2 +-
 tests/gzip/stdin.sh                           |  2 +-
 tests/gzip/test-driver.sh                     |  2 +-
 tests/gzip/trailing-nul.sh                    |  2 +-
 tests/gzip/unpack-invalid.sh                  |  2 +-
 tests/gzip/z-suffix.sh                        |  2 +-
 tests/gzip/zdiff.sh                           |  2 +-
 tests/gzip/zgrep-context.sh                   |  2 +-
 tests/gzip/zgrep-f.sh                         |  2 +-
 tests/gzip/zgrep-signal.sh                    |  2 +-
 tests/gzip/znew-k.sh                          |  2 +-
 tests/test-zstd-versions.py                   |  2 +-
 zlibWrapper/README.md                         |  4 +-
 zlibWrapper/examples/minigzip.c               |  2 +-
 zlibWrapper/gzclose.c                         |  2 +-
 zlibWrapper/gzguts.h                          |  2 +-
 zlibWrapper/gzlib.c                           |  2 +-
 zlibWrapper/gzread.c                          |  2 +-
 zlibWrapper/gzwrite.c                         |  2 +-
 57 files changed, 153 insertions(+), 155 deletions(-)

diff --git a/README.md b/README.md
index da8622d72..1f5a3c798 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Zstandard's format is stable and documented in [RFC8878](https://datatracker.iet
 This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
 and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
 Should your project require another programming language,
-a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
+a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages).
 
 **Development branch status:**
 
@@ -39,7 +39,7 @@ compiled with [gcc] 9.3.0,
 on the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
-[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
+[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia
 [gcc]: https://gcc.gnu.org/
 
 | Compressor name         | Ratio | Compression| Decompress.|
@@ -56,8 +56,8 @@ on the [Silesia compression corpus].
 | lzf 3.6 -1              | 2.077 |   410 MB/s |   830 MB/s |
 | snappy 1.1.9            | 2.073 |   550 MB/s |  1750 MB/s |
 
-[zlib]: http://www.zlib.net/
-[lz4]: http://www.lz4.org/
+[zlib]: https://www.zlib.net/
+[lz4]: https://lz4.github.io/lz4/
 
 The negative compression levels, specified with `--fast=#`,
 offer faster compression and decompression speed
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 5e2cf3d56..f1eac9ada 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -49,7 +49,7 @@ else()
     )
 endif()
 message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
-set(zstd_HOMEPAGE_URL "http://www.zstd.net")
+set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd")
 set(zstd_DESCRIPTION  "Zstandard is a real-time compression algorithm, providing high compression ratios.")
 
 # Set a default build type if none was specified
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 6c1ea298a..02444c803 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -27,7 +27,7 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 # You can contact the author at :
-#  - zstd homepage : http://www.zstd.net/
+#  - zstd homepage : https://facebook.github.io/zstd/
 # ################################################################
 
 project(tests)
diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build
index 467007491..68db2ca6c 100644
--- a/build/meson/lib/meson.build
+++ b/build/meson/lib/meson.build
@@ -160,7 +160,7 @@ pkgconfig.generate(libzstd,
   filebase: 'libzstd',
   description: 'fast lossless compression algorithm library',
   version: zstd_libversion,
-  url: 'http://www.zstd.net/')
+  url: 'https://facebook.github.io/zstd/')
 
 install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
   join_paths(zstd_rootdir, 'lib/zdict.h'),
diff --git a/contrib/linux-kernel/test/include/linux/xxhash.h b/contrib/linux-kernel/test/include/linux/xxhash.h
index 7e92a706e..d41cbd933 100644
--- a/contrib/linux-kernel/test/include/linux/xxhash.h
+++ b/contrib/linux-kernel/test/include/linux/xxhash.h
@@ -2,7 +2,7 @@
  * xxHash - Extremely Fast Hash algorithm
  * Copyright (C) 2012-2016, Yann Collet.
  *
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -260,7 +260,7 @@ XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state
  * xxHash - Extremely Fast Hash algorithm
  * Copyright (C) 2012-2016, Yann Collet.
  *
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
diff --git a/contrib/pzstd/README.md b/contrib/pzstd/README.md
index 84d945815..bc8f831f8 100644
--- a/contrib/pzstd/README.md
+++ b/contrib/pzstd/README.md
@@ -31,7 +31,7 @@ If this number is not suitable, during compilation you can define `PZSTD_NUM_THR
 
 ## Benchmarks
 
-As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia).
+As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia).
 
 Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads
 ------------------------------------------|-----------------------------------
diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h
index 3cfbe86e5..8898b0357 100644
--- a/contrib/pzstd/utils/FileSystem.h
+++ b/contrib/pzstd/utils/FileSystem.h
@@ -17,7 +17,7 @@
 
 // A small subset of `std::filesystem`.
 // `std::filesystem` should be a drop in replacement.
-// See http://en.cppreference.com/w/cpp/filesystem for documentation.
+// See https://en.cppreference.com/w/cpp/filesystem for documentation.
 
 namespace pzstd {
 
@@ -28,7 +28,7 @@ typedef struct ::_stat64 file_status;
 typedef struct ::stat file_status;
 #endif
 
-/// http://en.cppreference.com/w/cpp/filesystem/status
+/// https://en.cppreference.com/w/cpp/filesystem/status
 inline file_status status(StringPiece path, std::error_code& ec) noexcept {
   file_status status;
 #if defined(_MSC_VER)
@@ -44,7 +44,7 @@ inline file_status status(StringPiece path, std::error_code& ec) noexcept {
   return status;
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
+/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file
 inline bool is_regular_file(file_status status) noexcept {
 #if defined(S_ISREG)
   return S_ISREG(status.st_mode);
@@ -55,12 +55,12 @@ inline bool is_regular_file(file_status status) noexcept {
 #endif
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
+/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file
 inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept {
   return is_regular_file(status(path, ec));
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+/// https://en.cppreference.com/w/cpp/filesystem/is_directory
 inline bool is_directory(file_status status) noexcept {
 #if defined(S_ISDIR)
   return S_ISDIR(status.st_mode);
@@ -71,12 +71,12 @@ inline bool is_directory(file_status status) noexcept {
 #endif
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+/// https://en.cppreference.com/w/cpp/filesystem/is_directory
 inline bool is_directory(StringPiece path, std::error_code& ec) noexcept {
   return is_directory(status(path, ec));
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/file_size
+/// https://en.cppreference.com/w/cpp/filesystem/file_size
 inline std::uintmax_t file_size(
     StringPiece path,
     std::error_code& ec) noexcept {
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index 0848d2519..24083aa52 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -38,7 +38,7 @@
       * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */
 #    define PLATFORM_POSIX_VERSION 200112L
 
-/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
+/* try to determine posix version through official unistd.h's _POSIX_VERSION (https://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
  * note : there is no simple way to know in advance if <unistd.h> is present or not on target system,
  * Posix specification mandates its presence and its content, but target system must respect this spec.
  * It's necessary to _not_ #include <unistd.h> whenever target OS is not unix-like
diff --git a/doc/README.md b/doc/README.md
index bb7a3e490..705adfe5b 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -5,7 +5,7 @@ This directory contains material defining the Zstandard format,
 as well as detailed instructions to use `zstd` library.
 
 __`zstd_manual.html`__ : Documentation of `zstd.h` API, in html format.
-Click on this link: [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html)
+Click on this link: [https://facebook.github.io/zstd/zstd_manual.html](https://facebook.github.io/zstd/zstd_manual.html)
 to display documentation of latest release in readable format within a browser.
 
 __`zstd_compression_format.md`__ : This document defines the Zstandard compression format.
diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index 936407086..54c059352 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -1890,7 +1890,7 @@ static size_t HUF_decompress_4stream(const HUF_dtable *const dtable,
 
 /// Initializes a Huffman table using canonical Huffman codes
 /// For more explanation on canonical Huffman codes see
-/// http://www.cs.uofs.edu/~mccloske/courses/cmps340/huff_canonical_dec2015.html
+/// https://www.cs.scranton.edu/~mccloske/courses/cmps340/huff_canonical_dec2015.html
 /// Codes within a level are allocated in symbol order (i.e. smaller symbols get
 /// earlier codes)
 static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits,
diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 419de38eb..20c4a10e3 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -26,7 +26,7 @@ The purpose of this document is to define a lossless compressed data format,
 that is independent of CPU type, operating system,
 file system and character set, suitable for
 file compression, pipe and streaming compression,
-using the [Zstandard algorithm](http://www.zstandard.org).
+using the [Zstandard algorithm](https://facebook.github.io/zstd/).
 The text of the specification assumes a basic background in programming
 at the level of bits and other primitive data representations.
 
@@ -35,7 +35,7 @@ even for an arbitrarily long sequentially presented input data stream,
 using only an a priori bounded amount of intermediate storage,
 and hence can be used in data communications.
 The format uses the Zstandard compression method,
-and optional [xxHash-64 checksum method](http://www.xxhash.org),
+and optional [xxHash-64 checksum method](https://cyan4973.github.io/xxHash/),
 for detection of data corruption.
 
 The data format defined by this specification
@@ -134,7 +134,7 @@ __`Content_Checksum`__
 
 An optional 32-bit checksum, only present if `Content_Checksum_flag` is set.
 The content checksum is the result
-of [xxh64() hash function](http://www.xxhash.org)
+of [xxh64() hash function](https://cyan4973.github.io/xxHash/)
 digesting the original (decoded) data as input, and a seed of zero.
 The low 4 bytes of the checksum are stored in __little-endian__ format.
 
@@ -967,7 +967,7 @@ into a flow of concatenated frames.
 
 Skippable frames defined in this specification are compatible with [LZ4] ones.
 
-[LZ4]:http://www.lz4.org
+[LZ4]:https://lz4.github.io/lz4/
 
 From a compliant decoder perspective, skippable frames need just be skipped,
 and their content ignored, resuming decoding after the skippable frame.
diff --git a/lib/common/mem.h b/lib/common/mem.h
index 4b10f7c1f..46596ed61 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -143,7 +143,7 @@ MEM_STATIC size_t MEM_swapST(size_t in);
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
  *            In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
diff --git a/lib/common/threading.c b/lib/common/threading.c
index 4f69f7662..c62c65d57 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -23,8 +23,7 @@ int g_ZSTD_threading_useless_symbol;
 #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
 
 /**
- * Windows minimalist Pthread Wrapper, based on :
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ * Windows minimalist Pthread Wrapper
  */
 
 
diff --git a/lib/common/threading.h b/lib/common/threading.h
index fd0060d5a..b1458054e 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -23,8 +23,7 @@ extern "C" {
 #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
 
 /**
- * Windows minimalist Pthread Wrapper, based on :
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ * Windows minimalist Pthread Wrapper
  */
 #ifdef WINVER
 #  undef WINVER
diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index d49497cf1..c2d61ca64 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -3,7 +3,7 @@
  *  Copyright (c) Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
- *  - xxHash homepage: http://www.xxhash.com
+ *  - xxHash homepage: https://cyan4973.github.io/xxHash/
  *  - xxHash source repository : https://github.com/Cyan4973/xxHash
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index f87102a1d..901bc0832 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -3,7 +3,7 @@
  *  Copyright (c) Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
- *  - xxHash homepage: http://www.xxhash.com
+ *  - xxHash homepage: https://cyan4973.github.io/xxHash/
  *  - xxHash source repository : https://github.com/Cyan4973/xxHash
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -1314,7 +1314,7 @@ XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
  *   care, as what works on one compiler/platform/optimization level may cause
  *   another to read garbage data or even crash.
  *
- * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
  *
  * Prefer these methods in priority order (0 > 3 > 1 > 2)
  */
@@ -1655,7 +1655,7 @@ static xxh_u32 XXH_read32(const void* ptr)
 
 /*
  * Portable and safe solution. Generally efficient.
- * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
  */
 static xxh_u32 XXH_read32(const void* memPtr)
 {
@@ -2296,7 +2296,7 @@ static xxh_u64 XXH_read64(const void* ptr)
 
 /*
  * Portable and safe solution. Generally efficient.
- * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
  */
 static xxh_u64 XXH_read64(const void* memPtr)
 {
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 21be8c54f..c2271f6ee 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -91,7 +91,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
     assert(tableLog < 16);   /* required for threshold strategy to work */
 
     /* For explanations on how to distribute symbol values over the table :
-     * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+     * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
 
      #ifdef __clang_analyzer__
      ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize);   /* useless initialization, just to keep scan-build happy */
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 23caaef56..d3d4055b8 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -200,7 +200,7 @@ typedef   signed long long  S64;
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets generating assembly depending on alignment.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef FSE_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
@@ -1190,7 +1190,7 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
     zstd - standard compression library
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 6eadd3948..c975143c6 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -28,7 +28,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -125,7 +125,7 @@ extern "C" {
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets generating assembly depending on alignment.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
@@ -269,7 +269,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
    header file (to include)
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -510,7 +510,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
    Error codes and messages
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -609,7 +609,7 @@ typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be mor
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -753,7 +753,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -822,7 +822,7 @@ static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, si
     Header File
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -882,7 +882,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx;   /* incomplete type */
     Header File for static linking only
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -946,7 +946,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
    FSE : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1450,7 +1450,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2609,7 +2609,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
     zstd - standard compression library
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 72fc9140d..b3ff1977c 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -29,7 +29,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -126,7 +126,7 @@ extern "C" {
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets generating assembly depending on alignment.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
@@ -271,7 +271,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
    header file (to include)
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -512,7 +512,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
    Error codes and messages
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -611,7 +611,7 @@ typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be mor
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -755,7 +755,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -823,7 +823,7 @@ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, si
     Header File
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -883,7 +883,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx;   /* incomplete type */
     Header File for static linking only
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -947,7 +947,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
    FSE : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1451,7 +1451,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2248,7 +2248,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
     zstd - standard compression library
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index a2d281eb9..b0a0cbee0 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -97,7 +97,7 @@ extern "C" {
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets generating assembly depending on alignment.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
@@ -542,7 +542,7 @@ If there is an error, the function will return an error code, which can be teste
    header file (to include)
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -781,7 +781,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -930,7 +930,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
    FSE : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1436,7 +1436,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
    header file
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1514,7 +1514,7 @@ static unsigned    HUF_isError(size_t code);        /* tells if a return value i
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1601,7 +1601,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2401,7 +2401,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
     zstd - decompression module fo v0.4 legacy format
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -3283,7 +3283,7 @@ static void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, s
     Buffered version of Zstd compression library
     Copyright (C) 2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 216c67d01..75bfe7b2c 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -19,7 +19,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -116,7 +116,7 @@ extern "C" {
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
  *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
@@ -262,7 +262,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     Header File for static linking only
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -286,7 +286,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 #ifndef ZSTD_STATIC_H
 #define ZSTD_STATIC_H
@@ -398,7 +398,7 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity
     Header File for include
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -553,7 +553,7 @@ typedef struct {
    header file
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -672,7 +672,7 @@ size_t FSEv05_decompress_usingDTable(void* dst, size_t dstCapacity, const void*
    header file (to include)
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -901,7 +901,7 @@ MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* DStream)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1051,7 +1051,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
    FSEv05 : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1537,7 +1537,7 @@ size_t FSEv05_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
    header file
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1610,7 +1610,7 @@ const char* HUFv05_getErrorName(size_t code);   /* provides error code string (u
    header file, for static linking only
    Copyright (C) 2013-2016, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1702,7 +1702,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2547,7 +2547,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
     zstd - standard compression library
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -3746,7 +3746,7 @@ size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, s
     Buffered version of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index b224355d4..346221b00 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -23,7 +23,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -118,7 +118,7 @@ extern "C" {
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
  *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
@@ -281,7 +281,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     Header File for static linking only
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -305,7 +305,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 #ifndef ZSTDv06_STATIC_H
 #define ZSTDv06_STATIC_H
@@ -412,7 +412,7 @@ ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, siz
     Header File for include
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -619,7 +619,7 @@ void ZSTDv06_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
    Public Prototypes declaration
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -767,7 +767,7 @@ If there is an error, the function will return an error code, which can be teste
    header file (to include)
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1002,7 +1002,7 @@ MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* DStream)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1210,7 +1210,7 @@ MEM_STATIC BYTE FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStre
    Common functions of New Generation Entropy library
    Copyright (C) 2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1355,7 +1355,7 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
    FSE : Finite State Entropy decoder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1679,7 +1679,7 @@ size_t FSEv06_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
    header file
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1749,7 +1749,7 @@ size_t HUFv06_compressBound(size_t size);       /**< maximum compressed size */
    header file, for static linking only
    Copyright (C) 2013-2016, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1931,7 +1931,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
    Huffman decoder, part of New Generation Entropy library
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2676,7 +2676,7 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
     Common functions of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2700,7 +2700,7 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
@@ -2730,7 +2730,7 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
     zstd - standard compression library
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2754,7 +2754,7 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 
 /* ***************************************************************
@@ -3889,7 +3889,7 @@ size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, s
     Buffered version of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -3913,7 +3913,7 @@ size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, s
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index c12a091cb..ea7aa21b4 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -184,7 +184,7 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -278,7 +278,7 @@ extern "C" {
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
  *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
@@ -439,7 +439,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
    header file (to include)
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -670,7 +670,7 @@ MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* DStream)
    Public Prototypes declaration
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -978,7 +978,7 @@ MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStre
    header file
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1151,7 +1151,7 @@ size_t HUFv07_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
    Common functions of New Generation Entropy library
    Copyright (C) 2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1375,7 +1375,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
    FSE : Finite State Entropy decoder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1699,7 +1699,7 @@ size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
    Huffman decoder, part of New Generation Entropy library
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2577,7 +2577,7 @@ size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize,
     Common functions of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2601,7 +2601,7 @@ size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize,
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
@@ -2647,7 +2647,7 @@ static void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
     Header File for include
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2854,7 +2854,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
     zstd - standard compression library
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2878,7 +2878,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 
 /* ***************************************************************
@@ -4253,7 +4253,7 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
     Buffered version of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -4277,7 +4277,7 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in
index 43ebaec35..33184297c 100644
--- a/lib/libzstd.pc.in
+++ b/lib/libzstd.pc.in
@@ -1,6 +1,6 @@
 #   ZSTD - standard compression algorithm
 #   Copyright (C) 2014-2016, Yann Collet, Facebook
-#   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+#   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
 prefix=@PREFIX@
 exec_prefix=@EXEC_PREFIX@
@@ -9,7 +9,7 @@ libdir=@LIBDIR@
 
 Name: zstd
 Description: fast lossless compression algorithm library
-URL: http://www.zstd.net/
+URL: https://facebook.github.io/zstd/
 Version: @VERSION@
 Libs: -L${libdir} -lzstd
 Libs.private: @LIBS_PRIVATE@
diff --git a/programs/Makefile b/programs/Makefile
index da9883a2e..1cdec4a64 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -276,7 +276,7 @@ generate_res: $(RES64_FILE) $(RES32_FILE)
 
 ifneq (,$(filter Windows%,$(OS)))
 RC ?= windres
-# http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
+# https://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
 $(RES64_FILE): windres/zstd.rc
 	$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64
 $(RES32_FILE): windres/zstd.rc
diff --git a/programs/README.md b/programs/README.md
index c192b8026..1b9f47cbb 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -276,7 +276,7 @@ compression speed (for lower levels) with minimal change in compression ratio.
 
 The below table illustrates this on the [Silesia compression corpus].
 
-[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
+[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia
 
 | Method | Compression ratio | Compression speed | Decompression speed  |
 |:-------|------------------:|------------------:|---------------------:|
diff --git a/programs/fileio.c b/programs/fileio.c
index e80d37011..b9d14211e 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1006,7 +1006,7 @@ FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but no
 
     {   int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
                         15 /* maxWindowLogSize */ + 16 /* gzip only */,
-                        8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
+                        8, Z_DEFAULT_STRATEGY); /* see https://www.zlib.net/manual.html */
         if (ret != Z_OK) {
             EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
     }   }
@@ -2134,7 +2134,7 @@ FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName)
     strm.opaque = Z_NULL;
     strm.next_in = 0;
     strm.avail_in = 0;
-    /* see http://www.zlib.net/manual.html */
+    /* see https://www.zlib.net/manual.html */
     if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK)
         return FIO_ERROR_FRAME_DECODING;
 
diff --git a/programs/platform.h b/programs/platform.h
index b858e3b48..f80d13559 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -33,7 +33,7 @@ extern "C" {
 
 /* **************************************
 *  Detect 64-bit OS
-*  http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
+*  https://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
 ****************************************/
 #if defined __ia64 || defined _M_IA64                                                                               /* Intel Itanium */ \
   || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__                                                /* POWER 64-bit */  \
@@ -80,7 +80,7 @@ extern "C" {
       * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */
 #    define PLATFORM_POSIX_VERSION 200112L
 
-/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
+/* try to determine posix version through official unistd.h's _POSIX_VERSION (https://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
  * note : there is no simple way to know in advance if <unistd.h> is present or not on target system,
  * Posix specification mandates its presence and its content, but target system must respect this spec.
  * It's necessary to _not_ #include <unistd.h> whenever target OS is not unix-like
@@ -192,13 +192,13 @@ static __inline int IS_CONSOLE(FILE* stdStream) {
 
 
 #ifndef ZSTD_SETPRIORITY_SUPPORT
-   /* mandates presence of <sys/resource.h> and support for setpriority() : http://man7.org/linux/man-pages/man2/setpriority.2.html */
+   /* mandates presence of <sys/resource.h> and support for setpriority() : https://man7.org/linux/man-pages/man2/setpriority.2.html */
 #  define ZSTD_SETPRIORITY_SUPPORT (PLATFORM_POSIX_VERSION >= 200112L)
 #endif
 
 
 #ifndef ZSTD_NANOSLEEP_SUPPORT
-   /* mandates support of nanosleep() within <time.h> : http://man7.org/linux/man-pages/man2/nanosleep.2.html */
+   /* mandates support of nanosleep() within <time.h> : https://man7.org/linux/man-pages/man2/nanosleep.2.html */
 #  if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) \
    || (PLATFORM_POSIX_VERSION >= 200112L)
 #     define ZSTD_NANOSLEEP_SUPPORT 1
diff --git a/tests/gzip/gzip-env.sh b/tests/gzip/gzip-env.sh
index 120e52d78..4570e81c3 100755
--- a/tests/gzip/gzip-env.sh
+++ b/tests/gzip/gzip-env.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/helin-segv.sh b/tests/gzip/helin-segv.sh
index f182c8066..b400c2445 100644
--- a/tests/gzip/helin-segv.sh
+++ b/tests/gzip/helin-segv.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/help-version.sh b/tests/gzip/help-version.sh
index ee0c19f7d..fcda1c3ac 100644
--- a/tests/gzip/help-version.sh
+++ b/tests/gzip/help-version.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 # Ensure that $SHELL is set to *some* value and exported.
 # This is required for dircolors, which would fail e.g., when
diff --git a/tests/gzip/hufts.sh b/tests/gzip/hufts.sh
index 9b9576ce3..49c369501 100644
--- a/tests/gzip/hufts.sh
+++ b/tests/gzip/hufts.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/init.sh b/tests/gzip/init.sh
index 97e4e4ba5..ebd8410be 100644
--- a/tests/gzip/init.sh
+++ b/tests/gzip/init.sh
@@ -13,7 +13,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 # Using this file in a test
 # =========================
diff --git a/tests/gzip/keep.sh b/tests/gzip/keep.sh
index ab9a21811..f87b1a49b 100644
--- a/tests/gzip/keep.sh
+++ b/tests/gzip/keep.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/list.sh b/tests/gzip/list.sh
index 75912e1e2..e218d7562 100644
--- a/tests/gzip/list.sh
+++ b/tests/gzip/list.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/memcpy-abuse.sh b/tests/gzip/memcpy-abuse.sh
index 7d5c056de..1478890cc 100644
--- a/tests/gzip/memcpy-abuse.sh
+++ b/tests/gzip/memcpy-abuse.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/mixed.sh b/tests/gzip/mixed.sh
index 383a54f5e..b47f4a56a 100644
--- a/tests/gzip/mixed.sh
+++ b/tests/gzip/mixed.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/null-suffix-clobber.sh b/tests/gzip/null-suffix-clobber.sh
index 0efd0e344..5acfb321c 100644
--- a/tests/gzip/null-suffix-clobber.sh
+++ b/tests/gzip/null-suffix-clobber.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/stdin.sh b/tests/gzip/stdin.sh
index eef4cd8b1..d697ab80a 100644
--- a/tests/gzip/stdin.sh
+++ b/tests/gzip/stdin.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/test-driver.sh b/tests/gzip/test-driver.sh
index 649c084e4..0529cc8fb 100644
--- a/tests/gzip/test-driver.sh
+++ b/tests/gzip/test-driver.sh
@@ -16,7 +16,7 @@ scriptversion=2016-01-11.22; # UTC
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
diff --git a/tests/gzip/trailing-nul.sh b/tests/gzip/trailing-nul.sh
index 7b15d5e55..b33b98f78 100644
--- a/tests/gzip/trailing-nul.sh
+++ b/tests/gzip/trailing-nul.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/unpack-invalid.sh b/tests/gzip/unpack-invalid.sh
index fe8384d73..ceda5ffa4 100644
--- a/tests/gzip/unpack-invalid.sh
+++ b/tests/gzip/unpack-invalid.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/z-suffix.sh b/tests/gzip/z-suffix.sh
index a870a5408..c0bf509c9 100644
--- a/tests/gzip/z-suffix.sh
+++ b/tests/gzip/z-suffix.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zdiff.sh b/tests/gzip/zdiff.sh
index d62a84606..6e99b66f4 100644
--- a/tests/gzip/zdiff.sh
+++ b/tests/gzip/zdiff.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zgrep-context.sh b/tests/gzip/zgrep-context.sh
index c8648b7e4..d213426a3 100644
--- a/tests/gzip/zgrep-context.sh
+++ b/tests/gzip/zgrep-context.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zgrep-f.sh b/tests/gzip/zgrep-f.sh
index d0cf27f7e..1e73ed204 100644
--- a/tests/gzip/zgrep-f.sh
+++ b/tests/gzip/zgrep-f.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zgrep-signal.sh b/tests/gzip/zgrep-signal.sh
index a8c53881a..dd8442c81 100644
--- a/tests/gzip/zgrep-signal.sh
+++ b/tests/gzip/zgrep-signal.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/znew-k.sh b/tests/gzip/znew-k.sh
index 6c239e28e..5cf99ed34 100644
--- a/tests/gzip/znew-k.sh
+++ b/tests/gzip/znew-k.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index baca251f5..94e3b8c5b 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -126,7 +126,7 @@ def compress_sample(tag, sample):
     print(tag + " : compression completed")
 
 
-# http://stackoverflow.com/a/19711609/2132223
+# https://stackoverflow.com/a/19711609/2132223
 def sha1_of_file(filepath):
     with open(filepath, 'rb') as f:
         return hashlib.sha1(f.read()).hexdigest()
diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md
index f1cc88c52..311e1db7a 100644
--- a/zlibWrapper/README.md
+++ b/zlibWrapper/README.md
@@ -1,7 +1,7 @@
 Zstandard wrapper for zlib
 ================================
 
-The main objective of creating a zstd wrapper for [zlib](http://zlib.net/) is to allow a quick and smooth transition to zstd for projects already using zlib.
+The main objective of creating a zstd wrapper for [zlib](https://zlib.net/) is to allow a quick and smooth transition to zstd for projects already using zlib.
 
 #### Required files
 
@@ -43,7 +43,7 @@ This behavior can be changed using `ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB)
 
 
 #### Example
-We have taken the file `test/example.c` from [the zlib library distribution](http://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c).
+We have taken the file `test/example.c` from [the zlib library distribution](https://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c).
 After compilation and execution it shows the following results: 
 ```
 zlib version 1.2.8 = 0x1280, compile flags = 0x65
diff --git a/zlibWrapper/examples/minigzip.c b/zlibWrapper/examples/minigzip.c
index fc9d263ca..717a94df9 100644
--- a/zlibWrapper/examples/minigzip.c
+++ b/zlibWrapper/examples/minigzip.c
@@ -3,7 +3,7 @@
 
 /* minigzip.c -- simulate gzip using the zlib compression library
  * Copyright (C) 1995-2006, 2010, 2011 Jean-loup Gailly.
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 /*
diff --git a/zlibWrapper/gzclose.c b/zlibWrapper/gzclose.c
index 25d3789b1..ba43b8c5e 100644
--- a/zlibWrapper/gzclose.c
+++ b/zlibWrapper/gzclose.c
@@ -3,7 +3,7 @@
 
 /* gzclose.c -- zlib gzclose() function
  * Copyright (C) 2004, 2010 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzguts.h b/zlibWrapper/gzguts.h
index 83db17c19..70a609d72 100644
--- a/zlibWrapper/gzguts.h
+++ b/zlibWrapper/gzguts.h
@@ -4,7 +4,7 @@
 
 /* gzguts.h -- zlib internal header definitions for gz* operations
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #ifdef _LARGEFILE64_SOURCE
diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c
index e2d2a31af..eea480a74 100644
--- a/zlibWrapper/gzlib.c
+++ b/zlibWrapper/gzlib.c
@@ -3,7 +3,7 @@
 
 /* gzlib.c -- zlib functions common to reading and writing gzip files
  * Copyright (C) 2004-2017 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c
index 18955de5f..584fad1ea 100644
--- a/zlibWrapper/gzread.c
+++ b/zlibWrapper/gzread.c
@@ -3,7 +3,7 @@
 
  /* gzread.c -- zlib functions for reading gzip files
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c
index 277388f7f..ccd4f71f1 100644
--- a/zlibWrapper/gzwrite.c
+++ b/zlibWrapper/gzwrite.c
@@ -3,7 +3,7 @@
 
  /* gzwrite.c -- zlib functions for writing gzip files
  * Copyright (C) 2004-2017 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include <assert.h>

From 031de3c69ccbf3282ed02fb49369b476730aeca8 Mon Sep 17 00:00:00 2001
From: Rosen Penev <rosenp@gmail.com>
Date: Wed, 28 Sep 2022 16:17:53 -0700
Subject: [PATCH 017/197] meson: make backtrace dependency on execinfo

musl libc for example has no such header.

Signed-off-by: Rosen Penev <rosenp@gmail.com>
---
 build/meson/meson_options.txt    | 2 +-
 build/meson/programs/meson.build | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt
index accf3fa10..f35cd5fc8 100644
--- a/build/meson/meson_options.txt
+++ b/build/meson/meson_options.txt
@@ -14,7 +14,7 @@ option('legacy_level', type: 'integer', min: 0, max: 7, value: 5,
   description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
 option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
   description: 'Enable run-time debug. See lib/common/debug.h')
-option('backtrace', type: 'boolean', value: false,
+option('backtrace', type: 'feature', value: 'disabled',
   description: 'Display a stack backtrace when execution generates a runtime exception')
 option('static_runtime', type: 'boolean', value: false,
   description: 'Link to static run-time libraries on MSVC')
diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build
index 8df3a5dcb..8cee115da 100644
--- a/build/meson/programs/meson.build
+++ b/build/meson/programs/meson.build
@@ -51,7 +51,8 @@ endif
 
 export_dynamic_on_windows = false
 # explicit backtrace enable/disable for Linux & Darwin
-if not use_backtrace
+execinfo = cc.has_header('execinfo.h', required: use_backtrace)
+if not execinfo.found()
   zstd_c_args += '-DBACKTRACE_ENABLE=0'
 elif use_debug and host_machine_os == os_windows  # MinGW target
   zstd_c_args += '-DBACKTRACE_ENABLE=1'

From e58a39f84e988e4229067372b4f30601dcfc484b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 14 Jan 2022 12:37:32 -0800
Subject: [PATCH 018/197] Allow tests to fake stdin/stdout/stderr is a console

We've been unable to effectively test cases where stdin/stdout/stderr
are consoles, because in our test cases they generally aren't. Allow the
command line flags `--fake-std{in,out,err}-is-console` to tell the CLI
to pretend that std{in,out,err} is a console.
---
 programs/fileio.c   |  2 +-
 programs/platform.h |  4 ++++
 programs/util.c     | 28 ++++++++++++++++++++++++++++
 programs/util.h     | 14 ++++++++++++++
 programs/zstdcli.c  | 13 ++++++++-----
 5 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index e80d37011..57192961e 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -3010,7 +3010,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
     }   }
 
     if (numFiles == 0) {
-        if (!IS_CONSOLE(stdin)) {
+        if (!UTIL_isConsole(stdin)) {
             DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n");
         }
         DISPLAYLEVEL(1, "No files given \n");
diff --git a/programs/platform.h b/programs/platform.h
index b858e3b48..3ee19580c 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -127,6 +127,10 @@ extern "C" {
 
 /*-*********************************************
 *  Detect if isatty() and fileno() are available
+*
+*  Note: Use UTIL_isConsole() for the zstd CLI
+*  instead, as it allows faking is console for
+*  testing.
 ************************************************/
 #if (defined(__linux__) && (PLATFORM_POSIX_VERSION > 1)) \
  || (PLATFORM_POSIX_VERSION >= 200112L) \
diff --git a/programs/util.c b/programs/util.c
index 63b3ae176..bdb651074 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -288,6 +288,34 @@ int UTIL_isLink(const char* infilename)
     return 0;
 }
 
+static int g_fakeStdinIsConsole = 0;
+static int g_fakeStderrIsConsole = 0;
+static int g_fakeStdoutIsConsole = 0;
+
+int UTIL_isConsole(FILE* file)
+{
+    if (file == stdin && g_fakeStdinIsConsole)
+        return 1;
+    if (file == stderr && g_fakeStderrIsConsole)
+        return 1;
+    if (file == stdout && g_fakeStdoutIsConsole)
+        return 1;
+    return IS_CONSOLE(file);
+}
+
+void UTIL_fakeStdinIsConsole(void)
+{
+    g_fakeStdinIsConsole = 1;
+}
+void UTIL_fakeStdoutIsConsole(void)
+{
+    g_fakeStdoutIsConsole = 1;
+}
+void UTIL_fakeStderrIsConsole(void)
+{
+    g_fakeStderrIsConsole = 1;
+}
+
 U64 UTIL_getFileSize(const char* infilename)
 {
     stat_t statbuf;
diff --git a/programs/util.h b/programs/util.h
index faf8c9f11..cf1738772 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -175,6 +175,20 @@ int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
 int UTIL_isLink(const char* infilename);
 int UTIL_isFIFO(const char* infilename);
 
+/**
+ * Returns with the given file descriptor is a console.
+ * Allows faking whether stdin/stdout/stderr is a console
+ * using UTIL_fake*IsConsole().
+ */
+int UTIL_isConsole(FILE* file);
+
+/**
+ * Pretends that stdin/stdout/stderr is a console for testing.
+ */
+void UTIL_fakeStdinIsConsole(void);
+void UTIL_fakeStdoutIsConsole(void);
+void UTIL_fakeStderrIsConsole(void);
+
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
 U64 UTIL_getFileSize(const char* infilename);
 U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 362f320a9..cc6bbb935 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -27,8 +27,8 @@
 /*-************************************
 *  Dependencies
 **************************************/
-#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
-#include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
+#include "platform.h" /* PLATFORM_POSIX_VERSION */
+#include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList, UTIL_isConsole */
 #include <stdlib.h>   /* getenv */
 #include <string.h>   /* strcmp, strlen */
 #include <stdio.h>    /* fprintf(), stdin, stdout, stderr */
@@ -987,6 +987,9 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
                 if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
                 if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
+                if (!strcmp(argument, "--fake-stdin-is-console")) { UTIL_fakeStdinIsConsole(); continue; }
+                if (!strcmp(argument, "--fake-stdout-is-console")) { UTIL_fakeStdoutIsConsole(); continue; }
+                if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }
 
                 /* long commands with arguments */
 #ifndef ZSTD_NODICT
@@ -1437,12 +1440,12 @@ int main(int argCount, const char* argv[])
     /* Check if input/output defined as console; trigger an error in this case */
     if (!forceStdin
      && (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
-     && IS_CONSOLE(stdin) ) {
+     && UTIL_isConsole(stdin) ) {
         DISPLAYLEVEL(1, "stdin is a console, aborting\n");
         CLEAN_RETURN(1);
     }
     if ( (!outFileName || !strcmp(outFileName, stdoutmark))
-      && IS_CONSOLE(stdout)
+      && UTIL_isConsole(stdout)
       && (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
       && !forceStdout
       && operation!=zom_decompress ) {
@@ -1479,7 +1482,7 @@ int main(int argCount, const char* argv[])
     /* No status message in pipe mode (stdin - stdout) */
     hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
 
-    if ((hasStdout || !IS_CONSOLE(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
+    if ((hasStdout || !UTIL_isConsole(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
 
     /* IO Stream/File */
     FIO_setHasStdoutOutput(fCtx, hasStdout);

From fbff7827faba30df7c49992cb39dc00ce36c6a06 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 7 Jan 2022 15:07:28 -0800
Subject: [PATCH 019/197] Refactor progress bar & summary line logic

* Centralize the logic about whether to print the progress bar or not in
  the `*_PROGRESS()` macros.
* Centralize the logc about whether to print the summary line or not in
  `FIO_shouldDisplayFileSummary()` and
  `FIO_shouldDisplayMultipleFileSummary()`.
* Make `--progress` work for non-zstd (de)compressors.
* Clean up several edge cases in compression and decompression progress
  printing along the way. E.g. wrong log level, or missing summary line.

One thing I don't like about stdout mode, which sets the display level
to 1, is that warnings aren't displayed. After this PR, we could change
stdout mode from lowering the display level, to defaulting to implied
`--no-progress`. But, I think that deserves a separate PR.
---
 programs/fileio.c                             | 123 ++++++++++--------
 programs/fileio_common.h                      |   8 ++
 tests/cli-tests/progress/no-progress.sh       |  46 +++++++
 .../progress/no-progress.sh.stderr.glob       |  90 +++++++++++++
 tests/cli-tests/progress/progress.sh          |  41 ++++++
 .../progress/progress.sh.stderr.glob          |  62 +++++++++
 6 files changed, 318 insertions(+), 52 deletions(-)
 create mode 100755 tests/cli-tests/progress/no-progress.sh
 create mode 100644 tests/cli-tests/progress/no-progress.sh.stderr.glob
 create mode 100755 tests/cli-tests/progress/progress.sh
 create mode 100644 tests/cli-tests/progress/progress.sh.stderr.glob

diff --git a/programs/fileio.c b/programs/fileio.c
index 57192961e..4c1331432 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -249,6 +249,18 @@ struct FIO_ctx_s {
     size_t totalBytesOutput;
 };
 
+static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx)
+{
+    return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3;
+}
+
+static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
+{
+    int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1);
+    assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0);
+    return shouldDisplay;
+}
+
 
 /*-*************************************
 *  Parameters: Initialization
@@ -1044,14 +1056,16 @@ FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but no
                 strm.avail_out = (uInt)writeJob->bufferSize;
             }   }
         if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
-            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
-                          (unsigned)(inFileSize>>20),
-                          (double)outFileSize/inFileSize*100)
+            DISPLAYUPDATE_PROGRESS(
+                    "\rRead : %u MB ==> %.2f%% ",
+                    (unsigned)(inFileSize>>20),
+                    (double)outFileSize/inFileSize*100)
         } else {
-            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
-                          (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                          (double)outFileSize/inFileSize*100);
-        }   }
+            DISPLAYUPDATE_PROGRESS(
+                    "\rRead : %u / %u MB ==> %.2f%% ",
+                    (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
+                    (double)outFileSize/inFileSize*100);
+    }   }
 
     while (1) {
         int const ret = deflate(&strm, Z_FINISH);
@@ -1141,11 +1155,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
                 strm.avail_out = writeJob->bufferSize;
         }   }
         if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
-            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
+            DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20),
                             (double)outFileSize/inFileSize*100)
         else
-            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
+            DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
                             (double)outFileSize/inFileSize*100);
         if (ret == LZMA_STREAM_END) break;
@@ -1225,11 +1239,11 @@ FIO_compressLz4Frame(cRess_t* ress,
                             srcFileName, LZ4F_getErrorName(outSize));
             outFileSize += outSize;
             if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
-                DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
+                DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20),
                                 (double)outFileSize/inFileSize*100)
             } else {
-                DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
+                DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
                                 (double)outFileSize/inFileSize*100);
             }
@@ -1364,32 +1378,34 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                 UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced);
 
                 /* display progress notifications */
+                DISPLAY_PROGRESS("\r%79s\r", "");    /* Clear out the current displayed line */
                 if (g_display_prefs.displayLevel >= 3) {
-                    DISPLAYUPDATE(3, "\r(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
-                                compressionLevel,
-                                buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
-                                consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
-                                produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
-                                cShare );
-                } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) {
+                    /* Verbose progress update */
+                    DISPLAY_PROGRESS(
+                            "(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
+                            compressionLevel,
+                            buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
+                            consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
+                            produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
+                            cShare );
+                } else {
                     /* Require level 2 or forcibly displayed progress counter for summarized updates */
-                    DISPLAYLEVEL(1, "\r%79s\r", "");    /* Clear out the current displayed line */
                     if (fCtx->nbFilesTotal > 1) {
                         size_t srcFileNameSize = strlen(srcFileName);
                         /* Ensure that the string we print is roughly the same size each time */
                         if (srcFileNameSize > 18) {
                             const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
-                            DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ",
+                            DISPLAY_PROGRESS("Compress: %u/%u files. Current: ...%s ",
                                         fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName);
                         } else {
-                            DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ",
+                            DISPLAY_PROGRESS("Compress: %u/%u files. Current: %*s ",
                                         fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName);
                         }
                     }
-                    DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
+                    DISPLAY_PROGRESS("Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
                     if (fileSize != UTIL_FILESIZE_UNKNOWN)
-                        DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
-                    DISPLAYLEVEL(1, " ==> %2.f%%", cShare);
+                        DISPLAY_PROGRESS("/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
+                    DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
                     DELAY_NEXT_UPDATE();
                 }
 
@@ -1555,20 +1571,18 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
     /* Status */
     fCtx->totalBytesInput += (size_t)readsize;
     fCtx->totalBytesOutput += (size_t)compressedfilesize;
-    DISPLAYLEVEL(2, "\r%79s\r", "");
-    if (g_display_prefs.displayLevel >= 2 &&
-        !fCtx->hasStdoutOutput &&
-        (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) {
+    DISPLAY_PROGRESS("\r%79s\r", "");
+    if (FIO_shouldDisplayFileSummary(fCtx)) {
         UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
         UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
         if (readsize == 0) {
-            DISPLAYLEVEL(2,"%-20s :  (%6.*f%s => %6.*f%s, %s) \n",
+            DISPLAY_SUMMARY("%-20s :  (%6.*f%s => %6.*f%s, %s) \n",
                 srcFileName,
                 hr_isize.precision, hr_isize.value, hr_isize.suffix,
                 hr_osize.precision, hr_osize.value, hr_osize.suffix,
                 dstFileName);
         } else {
-            DISPLAYLEVEL(2,"%-20s :%6.2f%%   (%6.*f%s => %6.*f%s, %s) \n",
+            DISPLAY_SUMMARY("%-20s :%6.2f%%   (%6.*f%s => %6.*f%s, %s) \n",
                 srcFileName,
                 (double)compressedfilesize / (double)readsize * 100,
                 hr_isize.precision, hr_isize.value, hr_isize.suffix,
@@ -1917,16 +1931,23 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
             FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
     }
 
-    if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
+    if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
         UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
         UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);
 
-        DISPLAYLEVEL(2, "\r%79s\r", "");
-        DISPLAYLEVEL(2, "%3d files compressed :%.2f%%   (%6.*f%4s => %6.*f%4s)\n",
-                        fCtx->nbFilesProcessed,
-                        (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
-                        hr_isize.precision, hr_isize.value, hr_isize.suffix,
-                        hr_osize.precision, hr_osize.value, hr_osize.suffix);
+        DISPLAY_PROGRESS("\r%79s\r", "");
+        if (fCtx->totalBytesInput == 0) {
+            DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
+                            fCtx->nbFilesProcessed,
+                            hr_isize.precision, hr_isize.value, hr_isize.suffix,
+                            hr_osize.precision, hr_osize.value, hr_osize.suffix);
+        } else {
+            DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
+                            fCtx->nbFilesProcessed,
+                            (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
+                            hr_isize.precision, hr_isize.value, hr_isize.suffix,
+                            hr_osize.precision, hr_osize.value, hr_osize.suffix);
+        }
     }
 
     FIO_freeCResources(&ress);
@@ -2067,7 +2088,6 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
         ZSTD_inBuffer  inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 };
         ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
         size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
-        const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2;
         UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
         if (ZSTD_isError(readSizeHint)) {
             DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
@@ -2085,14 +2105,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
             size_t srcFileNameSize = strlen(srcFileName);
             if (srcFileNameSize > 18) {
                 const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
-                DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s...    ",
-                              fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
+                DISPLAYUPDATE_PROGRESS(
+                        "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s...    ",
+                        fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
             } else {
-                DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s...    ",
+                DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s...    ",
                             fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
             }
         } else {
-            DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s...     ",
+            DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s...     ",
                             srcFileName, hrs.precision, hrs.value, hrs.suffix);
         }
 
@@ -2307,7 +2328,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
                 AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
                 filesize += decodedBytes;
                 hrs = UTIL_makeHumanReadableSize(filesize);
-                DISPLAYUPDATE(2, "\rDecompressed : %.*f%s  ", hrs.precision, hrs.value, hrs.suffix);
+                DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s  ", hrs.precision, hrs.value, hrs.suffix);
             }
 
             if (!nextToLoad) break;
@@ -2415,13 +2436,9 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
 
     /* Final Status */
     fCtx->totalBytesOutput += (size_t)filesize;
-    DISPLAYLEVEL(2, "\r%79s\r", "");
-    /* No status message in pipe mode (stdin - stdout) or multi-files mode */
-    if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) ||
-        g_display_prefs.displayLevel >= 3 ||
-        g_display_prefs.progressSetting == FIO_ps_always) {
-        DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);
-    }
+    DISPLAY_PROGRESS("\r%79s\r", "");
+    if (FIO_shouldDisplayFileSummary(fCtx))
+        DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize);
 
     return 0;
 }
@@ -2730,8 +2747,10 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
             FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
     }
 
-    if (fCtx->nbFilesProcessed >= 1  && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0)
-        DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
+    if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
+        DISPLAY_PROGRESS("\r%79s\r", "");
+        DISPLAY_SUMMARY("%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
+    }
 
     FIO_freeDResources(ress);
     return error;
diff --git a/programs/fileio_common.h b/programs/fileio_common.h
index 282c2f13b..b82d5b6b1 100644
--- a/programs/fileio_common.h
+++ b/programs/fileio_common.h
@@ -48,6 +48,14 @@ extern UTIL_time_t g_displayClock;
                 if (g_display_prefs.displayLevel>=4) fflush(stderr);       \
     }   }   }
 
+#define SHOULD_DISPLAY_SUMMARY() \
+    (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always)
+#define SHOULD_DISPLAY_PROGRESS()                       \
+    (g_display_prefs.progressSetting != FIO_ps_never && SHOULD_DISPLAY_SUMMARY())
+#define DISPLAY_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYLEVEL(1, __VA_ARGS__); }}
+#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
+#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
+
 #undef MIN  /* in case it would be already defined */
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
diff --git a/tests/cli-tests/progress/no-progress.sh b/tests/cli-tests/progress/no-progress.sh
new file mode 100755
index 000000000..708878f26
--- /dev/null
+++ b/tests/cli-tests/progress/no-progress.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+println >&2 "Tests cases where progress information should not be printed"
+
+for args in \
+	"" \
+	"--fake-stderr-is-console -q" \
+	"--fake-stderr-is-console -qq --progress" \
+	"--no-progress --fake-stderr-is-console" \
+	"--no-progress --fake-stderr-is-console -v"
+do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done
diff --git a/tests/cli-tests/progress/no-progress.sh.stderr.glob b/tests/cli-tests/progress/no-progress.sh.stderr.glob
new file mode 100644
index 000000000..f07ad3803
--- /dev/null
+++ b/tests/cli-tests/progress/no-progress.sh.stderr.glob
@@ -0,0 +1,90 @@
+Tests cases where progress information should not be printed
+args =
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --fake-stderr-is-console -q
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --fake-stderr-is-console -qq --progress
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --no-progress --fake-stderr-is-console
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --no-progress --fake-stderr-is-console -v
+compress file to file
+*zstd*
+hello*hello.zst*
+compress pipe to pipe
+*zstd*
+*stdin*stdout*
+compress pipe to file
+*zstd*
+*stdin*hello.zst*
+compress file to pipe
+*zstd*
+*hello*stdout*
+compress 2 files
+*zstd*
+*hello*hello.zst*
+*world*world.zst*
+2 files compressed*
+decompress file to file
+*zstd*
+hello.zst*
+decompress pipe to pipe
+*zstd*
+*stdin*
+decompress pipe to file
+*zstd*
+*stdin*
+decompress file to pipe
+*zstd*
+hello.zst*
+decompress 2 files
+*zstd*
+hello.zst*
+world.zst*
+2 files decompressed*
diff --git a/tests/cli-tests/progress/progress.sh b/tests/cli-tests/progress/progress.sh
new file mode 100755
index 000000000..eb464993a
--- /dev/null
+++ b/tests/cli-tests/progress/progress.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+println >&2 "Tests cases where progress information should be printed"
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+for args in \
+	"--progress" \
+	"--fake-stderr-is-console" \
+	"--progress --fake-stderr-is-console -q"; do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done
diff --git a/tests/cli-tests/progress/progress.sh.stderr.glob b/tests/cli-tests/progress/progress.sh.stderr.glob
new file mode 100644
index 000000000..ca620d3c2
--- /dev/null
+++ b/tests/cli-tests/progress/progress.sh.stderr.glob
@@ -0,0 +1,62 @@
+Tests cases where progress information should be printed
+args = --progress
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --fake-stderr-is-console
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --progress --fake-stderr-is-console -q
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*

From a78c91ae59b9487fc32224b67c4a854dc3720367 Mon Sep 17 00:00:00 2001
From: "Alex Xu (Hello71)" <alex_y_xu@yahoo.ca>
Date: Sun, 28 Nov 2021 21:54:24 -0500
Subject: [PATCH 020/197] Use proper unaligned access attributes

Instead of using packed attribute hack, just use aligned attribute. It
improves code generation on armv6 and armv7, and slightly improves code
generation on aarch64. GCC generates identical code to regular aligned
access on ARMv6 for all versions between 4.5 and trunk, except GCC 5
which is buggy and generates the same (bad) code as packed access:
https://gcc.godbolt.org/z/hq37rz7sb
---
 contrib/linux-kernel/Makefile |  1 -
 lib/common/mem.h              | 49 ++++++++++++-----------------------
 lib/legacy/zstd_v01.c         | 30 +++++++++------------
 lib/legacy/zstd_v02.c         | 32 ++++++++++-------------
 lib/legacy/zstd_v03.c         | 32 ++++++++++-------------
 lib/legacy/zstd_v04.c         | 32 ++++++++++-------------
 lib/legacy/zstd_v05.c         | 34 ++++++++++--------------
 lib/legacy/zstd_v06.c         | 30 +++++++++------------
 lib/legacy/zstd_v07.c         | 30 +++++++++------------
 tests/fuzz/fuzz.h             |  3 +--
 10 files changed, 106 insertions(+), 167 deletions(-)

diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index baa1f24c6..f80ee8653 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -34,7 +34,6 @@ libzstd:
 		-DFSE_STATIC_LINKING_ONLY \
 		-DHUF_STATIC_LINKING_ONLY \
 		-DXXH_STATIC_LINKING_ONLY \
-		-DMEM_FORCE_MEMORY_ACCESS=0 \
 		-D__GNUC__ \
 		-D__linux__=1 \
 		-DSTATIC_BMI2=0 \
diff --git a/lib/common/mem.h b/lib/common/mem.h
index 4b10f7c1f..493782f6f 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -133,21 +133,15 @@ MEM_STATIC size_t MEM_swapST(size_t in);
 /*-**************************************************************
 *  Memory I/O Implementation
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -190,30 +184,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
-    __pragma( pack(push, 1) )
-    typedef struct { U16 v; } unalign16;
-    typedef struct { U32 v; } unalign32;
-    typedef struct { U64 v; } unalign64;
-    typedef struct { size_t v; } unalignArch;
-    __pragma( pack(pop) )
-#else
-    typedef struct { U16 v; } __attribute__((packed)) unalign16;
-    typedef struct { U32 v; } __attribute__((packed)) unalign32;
-    typedef struct { U64 v; } __attribute__((packed)) unalign64;
-    typedef struct { size_t v; } __attribute__((packed)) unalignArch;
-#endif
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
+typedef __attribute__((aligned(1))) size_t unalignArch;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
-MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
+MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
 
 #else
 
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 23caaef56..3a8b6bddd 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -190,21 +190,15 @@ typedef   signed long long  S64;
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* FSE_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* FSE_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ *            It can generate buggy code on targets depending on alignment.
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef FSE_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define FSE_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -229,13 +223,13 @@ static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; }
 
 #elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
 
-static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+static U16 FSE_read16(const void* ptr) { return *(const unalign16*)ptr; }
+static U32 FSE_read32(const void* ptr) { return *(const unalign32*)ptr; }
+static U64 FSE_read64(const void* ptr) { return *(const unalign64*)ptr; }
 
 #else
 
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 6eadd3948..05766d203 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -115,21 +115,15 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ *            It can generate buggy code on targets depending on alignment.
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -155,15 +149,15 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
 
 #else
 
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 72fc9140d..15cf5f3d7 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -116,21 +116,15 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ *            It can generate buggy code on targets depending on alignment.
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -156,15 +150,15 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
 
 #else
 
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index a2d281eb9..d4594292d 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -87,21 +87,15 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ *            It can generate buggy code on targets depending on alignment.
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -127,15 +121,15 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
 
 #else
 
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 216c67d01..06aec20a0 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -106,21 +106,15 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -148,17 +142,17 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
 
 #else
 
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index b224355d4..66f256a53 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -108,21 +108,15 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -148,15 +142,15 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
 
 #else
 
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index c12a091cb..71c99ab63 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -268,21 +268,15 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -308,15 +302,15 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
 
 #else
 
diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h
index 810daa2ce..6f3fb2994 100644
--- a/tests/fuzz/fuzz.h
+++ b/tests/fuzz/fuzz.h
@@ -26,8 +26,7 @@
  * @param MEM_FORCE_MEMORY_ACCESS:
  *        This flag controls how the zstd library accesses unaligned memory.
  *        It can be undefined, or 0 through 2. If it is undefined, it selects
- *        the method to use based on the compiler. If testing with UBSAN set
- *        MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method.
+ *        the method to use based on the compiler.
  * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
  *        This is the canonical flag to enable deterministic builds for fuzzing.
  *        Changes to zstd for fuzzing are gated behind this define.

From 15f32ad74ccf1f10efc93fdc4a180e7eba3e387e Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 14 Dec 2022 15:17:05 -0800
Subject: [PATCH 021/197] [fileio] Separate parameter adaption from display
 update rate

Split the logic for parameter adaption from the logic to update the display rate.
This decouples the two updates, so changes to display updates don't affect
parameter adaption.

Also add a test case that checks that parameter adaption actually happens.

This fixes Issue #3353, where --adapt is broken when --no-progress is passed.
---
 programs/fileio.c                    | 189 ++++++++++++++-------------
 programs/fileio_common.h             |   4 +-
 tests/cli-tests/compression/adapt.sh |   8 ++
 3 files changed, 108 insertions(+), 93 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 4c1331432..15a0d5387 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1301,6 +1301,9 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
     unsigned inputPresented = 0;
     unsigned inputBlocked = 0;
     unsigned lastJobID = 0;
+    UTIL_time_t lastAdaptTime = UTIL_getTime();
+    U64 const adaptEveryMicro = REFRESH_RATE;
+
     UTIL_HumanReadableSize_t const file_hrs = UTIL_makeHumanReadableSize(fileSize);
 
     DISPLAYLEVEL(6, "compression using zstd format \n");
@@ -1369,14 +1372,106 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                 compressedfilesize += outBuff.pos;
             }
 
-            /* display notification; and adapt compression level */
-            if (READY_FOR_UPDATE()) {
+            /* adaptive mode : statistics measurement and speed correction */
+            if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) {
+                ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
+
+                lastAdaptTime = UTIL_getTime();
+
+                /* check output speed */
+                if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
+
+                    unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
+                    unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
+                    assert(zfp.produced >= previous_zfp_update.produced);
+                    assert(prefs->nbWorkers >= 1);
+
+                    /* test if compression is blocked
+                        * either because output is slow and all buffers are full
+                        * or because input is slow and no job can start while waiting for at least one buffer to be filled.
+                        * note : exclude starting part, since currentJobID > 1 */
+                    if ( (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
+                        && (zfp.nbActiveWorkers == 0)                       /* confirmed : no compression ongoing */
+                        ) {
+                        DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
+                        speedChange = slower;
+                    }
+
+                    previous_zfp_update = zfp;
+
+                    if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
+                        && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
+                        ) {
+                        DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
+                        speedChange = slower;
+                    }
+                    flushWaiting = 0;
+                }
+
+                /* course correct only if there is at least one new job completed */
+                if (zfp.currentJobID > lastJobID) {
+                    DISPLAYLEVEL(6, "compression level adaptation check \n")
+
+                    /* check input speed */
+                    if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) {   /* warm up period, to fill all workers */
+                        if (inputBlocked <= 0) {
+                            DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
+                            speedChange = slower;
+                        } else if (speedChange == noChange) {
+                            unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
+                            unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
+                            unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
+                            unsigned long long newlyFlushed  = zfp.flushed  - previous_zfp_correction.flushed;
+                            previous_zfp_correction = zfp;
+                            assert(inputPresented > 0);
+                            DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
+                                            inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
+                                            (unsigned)newlyIngested, (unsigned)newlyConsumed,
+                                            (unsigned)newlyFlushed, (unsigned)newlyProduced);
+                            if ( (inputBlocked > inputPresented / 8)     /* input is waiting often, because input buffers is full : compression or output too slow */
+                                && (newlyFlushed * 33 / 32 > newlyProduced)  /* flush everything that is produced */
+                                && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
+                            ) {
+                                DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
+                                                newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
+                                speedChange = faster;
+                            }
+                        }
+                        inputBlocked = 0;
+                        inputPresented = 0;
+                    }
+
+                    if (speedChange == slower) {
+                        DISPLAYLEVEL(6, "slower speed , higher compression \n")
+                        compressionLevel ++;
+                        if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
+                        if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
+                        compressionLevel += (compressionLevel == 0);   /* skip 0 */
+                        ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
+                    }
+                    if (speedChange == faster) {
+                        DISPLAYLEVEL(6, "faster speed , lighter compression \n")
+                        compressionLevel --;
+                        if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
+                        compressionLevel -= (compressionLevel == 0);   /* skip 0 */
+                        ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
+                    }
+                    speedChange = noChange;
+
+                    lastJobID = zfp.currentJobID;
+                }  /* if (zfp.currentJobID > lastJobID) */
+            } /* if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) */
+
+            /* display notification */
+            if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) {
                 ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
                 double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
                 UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed);
                 UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed);
                 UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced);
 
+                DELAY_NEXT_UPDATE();
+
                 /* display progress notifications */
                 DISPLAY_PROGRESS("\r%79s\r", "");    /* Clear out the current displayed line */
                 if (g_display_prefs.displayLevel >= 3) {
@@ -1406,96 +1501,8 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                     if (fileSize != UTIL_FILESIZE_UNKNOWN)
                         DISPLAY_PROGRESS("/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
                     DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
-                    DELAY_NEXT_UPDATE();
                 }
-
-                /* adaptive mode : statistics measurement and speed correction */
-                if (prefs->adaptiveMode) {
-
-                    /* check output speed */
-                    if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
-
-                        unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
-                        unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
-                        assert(zfp.produced >= previous_zfp_update.produced);
-                        assert(prefs->nbWorkers >= 1);
-
-                        /* test if compression is blocked
-                         * either because output is slow and all buffers are full
-                         * or because input is slow and no job can start while waiting for at least one buffer to be filled.
-                         * note : exclude starting part, since currentJobID > 1 */
-                        if ( (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
-                          && (zfp.nbActiveWorkers == 0)                       /* confirmed : no compression ongoing */
-                          ) {
-                            DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
-                            speedChange = slower;
-                        }
-
-                        previous_zfp_update = zfp;
-
-                        if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
-                          && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
-                          ) {
-                            DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
-                            speedChange = slower;
-                        }
-                        flushWaiting = 0;
-                    }
-
-                    /* course correct only if there is at least one new job completed */
-                    if (zfp.currentJobID > lastJobID) {
-                        DISPLAYLEVEL(6, "compression level adaptation check \n")
-
-                        /* check input speed */
-                        if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) {   /* warm up period, to fill all workers */
-                            if (inputBlocked <= 0) {
-                                DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
-                                speedChange = slower;
-                            } else if (speedChange == noChange) {
-                                unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
-                                unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
-                                unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
-                                unsigned long long newlyFlushed  = zfp.flushed  - previous_zfp_correction.flushed;
-                                previous_zfp_correction = zfp;
-                                assert(inputPresented > 0);
-                                DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
-                                                inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
-                                                (unsigned)newlyIngested, (unsigned)newlyConsumed,
-                                                (unsigned)newlyFlushed, (unsigned)newlyProduced);
-                                if ( (inputBlocked > inputPresented / 8)     /* input is waiting often, because input buffers is full : compression or output too slow */
-                                  && (newlyFlushed * 33 / 32 > newlyProduced)  /* flush everything that is produced */
-                                  && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
-                                ) {
-                                    DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
-                                                    newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
-                                    speedChange = faster;
-                                }
-                            }
-                            inputBlocked = 0;
-                            inputPresented = 0;
-                        }
-
-                        if (speedChange == slower) {
-                            DISPLAYLEVEL(6, "slower speed , higher compression \n")
-                            compressionLevel ++;
-                            if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
-                            if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
-                            compressionLevel += (compressionLevel == 0);   /* skip 0 */
-                            ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
-                        }
-                        if (speedChange == faster) {
-                            DISPLAYLEVEL(6, "faster speed , lighter compression \n")
-                            compressionLevel --;
-                            if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
-                            compressionLevel -= (compressionLevel == 0);   /* skip 0 */
-                            ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
-                        }
-                        speedChange = noChange;
-
-                        lastJobID = zfp.currentJobID;
-                    }  /* if (zfp.currentJobID > lastJobID) */
-                }  /* if (g_adaptiveMode) */
-            }  /* if (READY_FOR_UPDATE()) */
+            }  /* if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) */
         }  /* while ((inBuff.pos != inBuff.size) */
     } while (directive != ZSTD_e_end);
 
diff --git a/programs/fileio_common.h b/programs/fileio_common.h
index b82d5b6b1..aec2e8d56 100644
--- a/programs/fileio_common.h
+++ b/programs/fileio_common.h
@@ -38,11 +38,11 @@ extern FIO_display_prefs_t g_display_prefs;
 extern UTIL_time_t g_displayClock;
 
 #define REFRESH_RATE  ((U64)(SEC_TO_MICRO / 6))
-#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > REFRESH_RATE)
+#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > REFRESH_RATE || g_display_prefs.displayLevel >= 4)
 #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
 #define DISPLAYUPDATE(l, ...) {                              \
         if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \
-            if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \
+            if (READY_FOR_UPDATE()) { \
                 DELAY_NEXT_UPDATE();                         \
                 DISPLAY(__VA_ARGS__);                        \
                 if (g_display_prefs.displayLevel>=4) fflush(stderr);       \
diff --git a/tests/cli-tests/compression/adapt.sh b/tests/cli-tests/compression/adapt.sh
index 564e955b5..30b9afaa0 100755
--- a/tests/cli-tests/compression/adapt.sh
+++ b/tests/cli-tests/compression/adapt.sh
@@ -4,3 +4,11 @@ set -e
 
 # Test --adapt
 zstd -f file --adapt -c | zstd -t
+
+datagen -g100M > file100M
+
+# Pick parameters to force fast adaptation, even on slow systems
+zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
+
+# Adaption still happens with --no-progress
+zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"

From f31b83ff34236b4c8ec7dc5332c52a7e67952215 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 14 Dec 2022 17:00:54 -0800
Subject: [PATCH 022/197] [decompress] Fix nullptr addition & improve fuzzer

Fix an instance of `NULL + 0` in `ZSTD_decompressStream()`. Also, improve our
`stream_decompress` fuzzer to pass `NULL` in/out buffers to
`ZSTD_decompressStream()`, and fix 2 issues that were immediately surfaced.

Fixes #3351
---
 lib/decompress/zstd_decompress.c | 9 +++++++--
 lib/legacy/zstd_v06.c            | 3 ++-
 lib/legacy/zstd_v07.c            | 3 ++-
 tests/fuzz/stream_decompress.c   | 8 ++++----
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index df831b774..e95b8822f 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2058,6 +2058,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
                     if (ZSTD_isError(decompressedSize)) return decompressedSize;
                     DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
+                    assert(istart != NULL);
                     ip = istart + cSize;
                     op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
                     zds->expected = 0;
@@ -2143,6 +2144,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 }
                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
                     FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
+                    assert(ip != NULL);
                     ip += neededInSize;
                     /* Function modifies the stage so we must break */
                     break;
@@ -2166,8 +2168,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                                     "should never happen");
                     loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
                 }
-                ip += loadedSize;
-                zds->inPos += loadedSize;
+                if (loadedSize != 0) {
+                    /* ip may be NULL */
+                    ip += loadedSize;
+                    zds->inPos += loadedSize;
+                }
                 if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
 
                 /* decode loaded input */
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 66f256a53..1456250b6 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -4029,7 +4029,8 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
                     size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
                     if (ZSTDv06_isError(hSize)) return hSize;
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
-                        memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
+                        if (ip != NULL)
+                            memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
                         zbd->lhSize += iend-ip;
                         *dstCapacityPtr = 0;
                         return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize;   /* remaining header bytes + next block header */
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 71c99ab63..9cfba8de9 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -4411,7 +4411,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
                 if (hSize != 0) {
                     size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
-                        memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
+                        if (ip != NULL)
+                            memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
                         zbd->lhSize += iend-ip;
                         *dstCapacityPtr = 0;
                         return (hSize - zbd->lhSize) + ZSTDv07_blockHeaderSize;   /* remaining header bytes + next block header */
diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c
index e0cdd34d9..86a39b8c9 100644
--- a/tests/fuzz/stream_decompress.c
+++ b/tests/fuzz/stream_decompress.c
@@ -99,14 +99,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     while (size > 0) {
         ZSTD_inBuffer in = makeInBuffer(&src, &size, producer);
-        while (in.pos != in.size) {
+        do {
+            size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
+            if (ZSTD_isError(rc)) goto error;
             if (out.pos == out.size) {
                 if (stableOutBuffer) goto error;
                 out = makeOutBuffer(producer, buf, bufSize);
             }
-            size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
-            if (ZSTD_isError(rc)) goto error;
-        }
+        } while (in.pos != in.size);
     }
 
 error:

From 728e73ebb49e316233cc79f8afe79209eb2a5e90 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 14 Dec 2022 16:07:22 -0800
Subject: [PATCH 023/197] [legacy] Remove FORCE_MEMORY_ACCESS and only use
 memcpy

Delete unaligned memory access code from the legacy codebase by removing all the
non-memcpy functions. We don't care about speed at all for this codebase, only
simplicity.
---
 lib/legacy/zstd_v01.c | 33 -------------------------------
 lib/legacy/zstd_v02.c | 42 ---------------------------------------
 lib/legacy/zstd_v03.c | 43 ----------------------------------------
 lib/legacy/zstd_v04.c | 42 ---------------------------------------
 lib/legacy/zstd_v05.c | 46 -------------------------------------------
 lib/legacy/zstd_v06.c | 42 ---------------------------------------
 lib/legacy/zstd_v07.c | 41 --------------------------------------
 7 files changed, 289 deletions(-)

diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 3a8b6bddd..3a0733f7e 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -190,19 +190,6 @@ typedef   signed long long  S64;
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* FSE_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
- * Method 0 : always use `memcpy()`. Safe and portable.
- * Method 1 : Use compiler extension to set unaligned access.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- * Default  : method 1 if supported, else method 0
- */
-#ifndef FSE_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  ifdef __GNUC__
-#    define FSE_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
-
 
 static unsigned FSE_32bits(void)
 {
@@ -215,24 +202,6 @@ static unsigned FSE_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2)
-
-static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; }
-static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; }
-static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-#elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1)
-
-typedef __attribute__((aligned(1))) U16 unalign16;
-typedef __attribute__((aligned(1))) U32 unalign32;
-typedef __attribute__((aligned(1))) U64 unalign64;
-
-static U16 FSE_read16(const void* ptr) { return *(const unalign16*)ptr; }
-static U32 FSE_read32(const void* ptr) { return *(const unalign32*)ptr; }
-static U64 FSE_read64(const void* ptr) { return *(const unalign64*)ptr; }
-
-#else
-
 static U16 FSE_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -248,8 +217,6 @@ static U64 FSE_read64(const void* memPtr)
     U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
-#endif /* FSE_FORCE_MEMORY_ACCESS */
-
 static U16 FSE_readLE16(const void* memPtr)
 {
     if (FSE_isLittleEndian())
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 05766d203..817aaa68f 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -115,18 +115,6 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
- * Method 0 : always use `memcpy()`. Safe and portable.
- * Method 1 : Use compiler extension to set unaligned access.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- * Default  : method 1 if supported, else method 0
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  ifdef __GNUC__
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -137,33 +125,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-typedef __attribute__((aligned(1))) U16 unalign16;
-typedef __attribute__((aligned(1))) U32 unalign32;
-typedef __attribute__((aligned(1))) U64 unalign64;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -184,9 +145,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 15cf5f3d7..858eb848d 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -116,18 +116,6 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
- * Method 0 : always use `memcpy()`. Safe and portable.
- * Method 1 : Use compiler extension to set unaligned access.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- * Default  : method 1 if supported, else method 0
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  ifdef __GNUC__
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -138,33 +126,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-typedef __attribute__((aligned(1))) U16 unalign16;
-typedef __attribute__((aligned(1))) U32 unalign32;
-typedef __attribute__((aligned(1))) U64 unalign64;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -185,10 +146,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index d4594292d..084a67b1d 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -87,18 +87,6 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
- * Method 0 : always use `memcpy()`. Safe and portable.
- * Method 1 : Use compiler extension to set unaligned access.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- * Default  : method 1 if supported, else method 0
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  ifdef __GNUC__
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -109,33 +97,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-typedef __attribute__((aligned(1))) U16 unalign16;
-typedef __attribute__((aligned(1))) U32 unalign32;
-typedef __attribute__((aligned(1))) U64 unalign64;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -156,9 +117,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 06aec20a0..e48895a5c 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -106,18 +106,6 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
- * Method 0 : always use `memcpy()`. Safe and portable.
- * Method 1 : Use compiler extension to set unaligned access.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- * Default  : method 1 if supported, else method 0
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  ifdef __GNUC__
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -128,37 +116,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard, by lying on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-typedef __attribute__((aligned(1))) U16 unalign16;
-typedef __attribute__((aligned(1))) U32 unalign32;
-typedef __attribute__((aligned(1))) U64 unalign64;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -189,9 +146,6 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 1456250b6..ca68cb432 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -108,18 +108,6 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
- * Method 0 : always use `memcpy()`. Safe and portable.
- * Method 1 : Use compiler extension to set unaligned access.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- * Default  : method 1 if supported, else method 0
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  ifdef __GNUC__
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
@@ -130,33 +118,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard, by lying on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-typedef __attribute__((aligned(1))) U16 unalign16;
-typedef __attribute__((aligned(1))) U32 unalign32;
-typedef __attribute__((aligned(1))) U64 unalign64;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -177,9 +138,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
 MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 9cfba8de9..274af788b 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -268,18 +268,6 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
- * Method 0 : always use `memcpy()`. Safe and portable.
- * Method 1 : Use compiler extension to set unaligned access.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- * Default  : method 1 if supported, else method 0
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  ifdef __GNUC__
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
@@ -290,33 +278,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard, by lying on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-typedef __attribute__((aligned(1))) U16 unalign16;
-typedef __attribute__((aligned(1))) U32 unalign32;
-typedef __attribute__((aligned(1))) U64 unalign64;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -337,8 +298,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
 MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */

From 6be3181307f6bb3824eb322db67675f393671266 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 15 Dec 2022 10:46:47 -0800
Subject: [PATCH 024/197] added mention of compilation flags

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 1f5a3c798..d34bda576 100644
--- a/README.md
+++ b/README.md
@@ -142,6 +142,10 @@ Other available options include:
 The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html),
 allowing staged install, standard flags, directory variables and command variables.
 
+For advanced use cases, specialized compilation flags which control binary generation
+are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library
+and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI.
+
 ### cmake
 
 A `cmake` project generator is provided within `build/cmake`.

From a91e7ec175d4f73b54a2d7ebf22d86d262ffb01d Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 15 Dec 2022 13:43:27 -0800
Subject: [PATCH 025/197] Fix corruption that rarely occurs in 32-bit mode with
 wlog=25

Fix an off-by-one error in the compressor that emits corrupt blocks if:

* Zstd is compiled in 32-bit mode
* The windowLog == 25 exactly
* An offset of 2^25-3, 2^25-2, 2^25-1, or 2^25 is emitted
* The bitstream had 7 bits leftover before writing the offset

This bug has been present since before v1.0, but wasn't able to easily
be triggered, since until somewhat recently zstd wasn't able to find
matches that were within 128KB of the window size.

Add a test case, and fix 2 bugs in `ZSTD_compressSequences()`:
* The `ZSTD_isRLE()` check was incorrect. It wouldn't produce
  corruption, but it could waste CPU and not emit RLE even if the block
  was RLE
* One windowSize was `1 << windowLog`, not `1u << windowLog`

Thanks to @tansy for finding the issue, and giving us a reproducer!

Fixes Issue #3350.
---
 lib/compress/zstd_compress.c |  6 ++--
 tests/zstreamtest.c          | 61 ++++++++++++++++++++++++++++++++++--
 2 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0069a7b1b..d2761974c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2624,7 +2624,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
                                 void* entropyWorkspace, size_t entropyWkspSize,
                           const int bmi2)
 {
-    const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
+    const int longOffsets = cctxParams->cParams.windowLog >= STREAM_ACCUMULATOR_MIN;
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
     unsigned* count = (unsigned*)entropyWorkspace;
     FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
@@ -5897,7 +5897,7 @@ static size_t
 ZSTD_validateSequence(U32 offCode, U32 matchLength,
                       size_t posInSrc, U32 windowLog, size_t dictSize)
 {
-    U32 const windowSize = 1 << windowLog;
+    U32 const windowSize = 1u << windowLog;
     /* posInSrc represents the amount of data the decoder would decode up to this point.
      * As long as the amount of data decoded is less than or equal to window size, offsets may be
      * larger than the total length of output decoded in order to reference the dict, even larger than
@@ -6256,7 +6256,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
 
         if (!cctx->isFirstBlock &&
             ZSTD_maybeRLE(&cctx->seqStore) &&
-            ZSTD_isRLE((BYTE const*)src, srcSize)) {
+            ZSTD_isRLE(ip, blockSize)) {
             /* We don't want to emit our first block as a RLE even if it qualifies because
             * doing so will cause the decoder (cli only) to throw a "should consume all input error."
             * This is only an issue for zstd <= v1.4.3
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 348f72ed4..453c45409 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -259,7 +259,7 @@ static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
     return 0;
 }
 
-static int basicUnitTests(U32 seed, double compressibility)
+static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 {
     size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
     void* CNBuffer = malloc(CNBufferSize);
@@ -1565,6 +1565,63 @@ static int basicUnitTests(U32 seed, double compressibility)
     CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
+    {
+        int windowLog;
+        int const kMaxWindowLog = bigTests ? 29 : 26;
+        size_t const kNbSequences = 10000;
+        size_t const kMaxSrcSize = (1u << kMaxWindowLog) + 10 * kNbSequences;
+        char* src = calloc(kMaxSrcSize, 1);
+        ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+        for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
+            size_t const srcSize = ((size_t)1 << windowLog) + 10 * (kNbSequences - 1);
+
+            sequences[0].offset = 32;
+            sequences[0].litLength = 32;
+            sequences[0].matchLength = (1u << windowLog) - 32;
+            sequences[0].rep = 0;
+            {
+                size_t i;
+                for (i = 1; i < kNbSequences; ++i) {
+                    sequences[i].offset = (1u << windowLog) - (FUZ_rand(&seed) % 8);
+                    sequences[i].litLength = FUZ_rand(&seed) & 7;
+                    sequences[i].matchLength = 10 - sequences[i].litLength;
+                    sequences[i].rep = 0;
+                }
+            }
+
+            CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_minMatch, 3));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, 1));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowLog));
+            assert(srcSize <= kMaxSrcSize);
+            cSize = ZSTD_compressSequences(zc, compressedBuffer, compressedBufferSize, sequences, kNbSequences, src, srcSize);
+            CHECK_Z(cSize);
+            CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
+            CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, windowLog))
+            {
+                ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
+                size_t decompressedBytes = 0;
+                for (;;) {
+                    ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
+                    size_t const ret = ZSTD_decompressStream(zd, &out, &in);
+                    CHECK_Z(ret);
+                    CHECK(decompressedBytes + out.pos > srcSize, "Output too large");
+                    CHECK(memcmp(out.dst, src + decompressedBytes, out.pos), "Corrupted");
+                    decompressedBytes += out.pos;
+                    if (ret == 0) {
+                        break;
+                    }
+                }
+                CHECK(decompressedBytes != srcSize, "Output wrong size");
+            }
+        }
+        free(sequences);
+        free(src);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     /* Overlen overwriting window data bug */
     DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
     {   /* This test has a window size of 1024 bytes and consists of 3 blocks:
@@ -2677,7 +2734,7 @@ int main(int argc, const char** argv)
     if (nbTests<=0) nbTests=1;
 
     if (testNb==0) {
-        result = basicUnitTests(0, ((double)proba) / 100);  /* constant seed for predictability */
+        result = basicUnitTests(0, ((double)proba) / 100, bigTests);  /* constant seed for predictability */
     }
 
     if (!result) {

From 45ed0df18a24430875fb4e90378b150e15c200f7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 15 Dec 2022 15:23:15 -0800
Subject: [PATCH 026/197] check potential overflow of compressBound()

fixed #3323, reported by @nigeltao

Completed documentation around this risk
(which is largely theoretical,
I can't see that happening in any "real world" scenario,
but an erroneous @srcSize value could indeed trigger it).
---
 lib/compress/zstd_compress.c | 15 +++++++++------
 lib/zstd.h                   | 27 +++++++++++++++++++++++++--
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0069a7b1b..1eb8c99cf 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -59,14 +59,17 @@
 *  Helper functions
 ***************************************/
 /* ZSTD_compressBound()
- * Note that the result from this function is only compatible with the "normal"
- * full-block strategy.
- * When there are a lot of small blocks due to frequent flush in streaming mode
- * the overhead of headers can make the compressed data to be larger than the
- * return value of ZSTD_compressBound().
+ * Note that the result from this function is only valid for
+ * the one-pass compression functions.
+ * When employing the streaming mode,
+ * if flushes are frequently altering the size of blocks,
+ * the overhead from block headers can make the compressed data larger
+ * than the return value of ZSTD_compressBound().
  */
 size_t ZSTD_compressBound(size_t srcSize) {
-    return ZSTD_COMPRESSBOUND(srcSize);
+    size_t const r = ZSTD_COMPRESSBOUND(srcSize);
+    if (r==0) return ERROR(srcSize_wrong);
+    return r;
 }
 
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 1dff31b4e..dc635adae 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -201,8 +201,31 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
 
 
 /*======  Helper functions  ======*/
-#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * providing @dstCapacity >= ZSTD_compressBound(srcSize) guarantees success.
+ * Note that it's still allowed to provide a smaller @dstCapacity value,
+ * in which case, the caller must inspect the return value with ZSTD_isError(),
+ * to detect any potential compression failure.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize is too large.
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((unsigned long long)(srcSize) > ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
 ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
 ZSTDLIB_API int         ZSTD_minCLevel(void);               /*!< minimum negative compression level allowed, requires v1.4.0+ */

From e0ef09ddba8ebe0cde6786bc3305b43438de0baf Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Wed, 14 Dec 2022 17:23:24 -0500
Subject: [PATCH 027/197] meson: partial fix for building pzstd on MSVC

It uses non-portable compiler options unconditionally. Elsewhere, we
check the compiler ID and only add the right ones, globally. Do the same
here.

NDEBUG can actually be handled by a core option, so while we are moving
things around, do so.

Unfortunately, this doesn't fix things entirely. The remaining issue is
not Meson's issue though -- MSVC simply does not like this source code
and somehow chokes on innocent code with the inscrutable "syntax error"
and "illegal token".
---
 build/meson/contrib/pzstd/meson.build | 3 ++-
 build/meson/meson.build               | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/build/meson/contrib/pzstd/meson.build b/build/meson/contrib/pzstd/meson.build
index 2c47999fa..b95dc7964 100644
--- a/build/meson/contrib/pzstd/meson.build
+++ b/build/meson/contrib/pzstd/meson.build
@@ -18,7 +18,8 @@ pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
   join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
 pzstd = executable('pzstd',
   pzstd_sources,
-  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-Wno-deprecated-declarations' ],
+  cpp_args: pzstd_warning_flags,
   include_directories: pzstd_includes,
   dependencies: [ libzstd_dep, thread_dep ],
+  override_options: ['b_ndebug=true'],
   install: true)
diff --git a/build/meson/meson.build b/build/meson/meson.build
index 98d556aa3..2d15753d6 100644
--- a/build/meson/meson.build
+++ b/build/meson/meson.build
@@ -104,8 +104,10 @@ use_lz4 = lz4_dep.found()
 
 add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
 
+pzstd_warning_flags = []
 if [compiler_gcc, compiler_clang].contains(cc_id)
   common_warning_flags = [ '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
+  pzstd_warning_flags = ['-Wno-shadow', '-Wno-deprecated-declarations']
   if cc_id == compiler_clang
     common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
   endif

From 1bc9dfe46eda467203fb046cf36a8be17d4576a3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 15 Dec 2022 15:56:11 -0800
Subject: [PATCH 028/197] Update documentation link to html format

fix #3319
---
 doc/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/README.md b/doc/README.md
index 705adfe5b..8f3babcdb 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -5,8 +5,9 @@ This directory contains material defining the Zstandard format,
 as well as detailed instructions to use `zstd` library.
 
 __`zstd_manual.html`__ : Documentation of `zstd.h` API, in html format.
-Click on this link: [https://facebook.github.io/zstd/zstd_manual.html](https://facebook.github.io/zstd/zstd_manual.html)
-to display documentation of latest release in readable format within a browser.
+Unfortunately, Github doesn't display `html` files in parsed format, just as source code.
+For a readable display of html API documentation of latest release,
+use this link: [https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html](https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html) .
 
 __`zstd_compression_format.md`__ : This document defines the Zstandard compression format.
 Compliant decoders must adhere to this document,

From e8401e9e8d84c7dab40677cc1687138135e86901 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Thu, 15 Dec 2022 19:48:22 -0500
Subject: [PATCH 029/197] meson: fix broken commit that broke the build

In commit 031de3c69ccbf3282ed02fb49369b476730aeca8 some code was added
that returned a boolean, but was treated as if it returned a dependency
object. This wasn't tested and could not work. Moreover, zstd no longer
built at all unless the entire programs directory was disabled and not
even evaluated.

Fix the return type checking.
---
 build/meson/programs/meson.build | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build
index 8cee115da..e611dc337 100644
--- a/build/meson/programs/meson.build
+++ b/build/meson/programs/meson.build
@@ -51,8 +51,8 @@ endif
 
 export_dynamic_on_windows = false
 # explicit backtrace enable/disable for Linux & Darwin
-execinfo = cc.has_header('execinfo.h', required: use_backtrace)
-if not execinfo.found()
+have_execinfo = cc.has_header('execinfo.h', required: use_backtrace)
+if not have_execinfo
   zstd_c_args += '-DBACKTRACE_ENABLE=0'
 elif use_debug and host_machine_os == os_windows  # MinGW target
   zstd_c_args += '-DBACKTRACE_ENABLE=1'

From 626425dce0bfda5be67b96b61d09b11173c5e436 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Thu, 15 Dec 2022 19:34:25 -0500
Subject: [PATCH 030/197] meson: fix warning for using too-new features

In commit 031de3c69ccbf3282ed02fb49369b476730aeca8 a feature of Meson
0.50.0 was added, but the minimum specified version of Meson is 0.48.0.
Meson therefore emitted a warning:

WARNING: Project targets '>=0.48.0' but uses feature introduced in '0.50.0': required arg in compiler.has_header.

And if anyone actually used Meson 0.48.0 to build with, it would error
out with mysterious claims that the build file itself is invalid, rather
than telling the user to install a newer version of Meson.

Solve this by bumping the minimum version to align with reality. This
e.g. drops support for Debian oldstable (buster)'s packaged version of
Meson, but still works if backports are enabled, or if the user can
`pip install` a newer version.
---
 build/meson/meson.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/meson/meson.build b/build/meson/meson.build
index 2d15753d6..576dc44db 100644
--- a/build/meson/meson.build
+++ b/build/meson/meson.build
@@ -26,7 +26,7 @@ project('zstd',
   version: run_command(
     find_program('GetZstdLibraryVersion.py'), '../../lib/zstd.h',
     check: true).stdout().strip(),
-  meson_version: '>=0.48.0')
+  meson_version: '>=0.50.0')
 
 cc = meson.get_compiler('c')
 cxx = meson.get_compiler('cpp')

From ee6475cbbd04408e721c4daa87d993faf3b3c1cf Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 15 Dec 2022 16:22:26 -0800
Subject: [PATCH 031/197] Add missing parens around macro definition

Fixes #3301.
---
 lib/compress/huf_compress.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index e94b398da..9d2b521bc 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -505,8 +505,8 @@ typedef struct {
  * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing.
  */
 #define RANK_POSITION_MAX_COUNT_LOG 32
-#define RANK_POSITION_LOG_BUCKETS_BEGIN (RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */
-#define RANK_POSITION_DISTINCT_COUNT_CUTOFF RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */
+#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */)
+#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */)
 
 /* Return the appropriate bucket index for a given count. See definition of
  * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy.

From ef566c8d683212a04174124ef3c0d34cdfb583b5 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 15 Dec 2022 16:55:31 -0800
Subject: [PATCH 032/197] [circleci] Try to re-enable aarch64build

Update CircleCI to focal and try to re-enable aarch64build.

Fixes #2785 if it works.
---
 .circleci/config.yml                | 3 +--
 .circleci/images/primary/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index f2eab6807..621ac49ff 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -7,8 +7,6 @@ jobs:
     #       preinstalled to reduce installation time.
     docker:
       - image: fbopensource/zstd-circleci-primary:0.0.1
-    # TODO: Re-enable aarch64 build:
-    #       make aarch64build     && make clean
     steps:
       - checkout
       - run:
@@ -36,6 +34,7 @@ jobs:
             make ppc64build V=1; make clean
             make ppcbuild   V=1; make clean
             make armbuild   V=1; make clean
+            make aarch64build V=1; make clean
             make -C tests test-legacy test-longmatch; make clean
             make -C lib libzstd-nomt; make clean
   # This step should only be run in a cron job
diff --git a/.circleci/images/primary/Dockerfile b/.circleci/images/primary/Dockerfile
index dd8004152..bfcb6af69 100644
--- a/.circleci/images/primary/Dockerfile
+++ b/.circleci/images/primary/Dockerfile
@@ -1,4 +1,4 @@
-FROM circleci/buildpack-deps:bionic
+FROM circleci/buildpack-deps:focal
 
 RUN sudo dpkg --add-architecture i386
 RUN sudo apt-get -y -qq update

From 26134b4565a85e133f6f77b16b4fe9cd0c530a07 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Tue, 19 Apr 2022 20:58:33 -0400
Subject: [PATCH 033/197] meson: don't require valgrind tests

It's entirely possible some people don't have valgrind installed, but
still want to run the tests. If they don't have it installed, then they
probably don't intend to run those precise test targets anyway.

Also, this solves an error when running the tests in an automated
environment. The valgrind tests have a hard dependency on behavior such
as `./zstd` erroring out with the message "stdin is a console, aborting"
which does not work if the automated environment doesn't have a console.
As a rough heuristic, automated environments lacking a console will
*probably* also not have valgrind, so avoiding that test definition
neatly sidesteps the issue.

Also, valgrind is not easily installable on macOS, at least homebrew
says it isn't available there. This makes it needlessly hard to
enable the testsuite on macOS.
---
 build/meson/tests/meson.build | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index 22f43209a..09168de78 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -134,13 +134,15 @@ checkTag = executable('checkTag',
 # =============================================================================
 
 if tests_supported_oses.contains(host_machine_os)
-  valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
+  valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: false)
   valgrindTest_py = files('valgrindTest.py')
-  test('valgrindTest',
-    valgrindTest_py,
-    args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
-    depends: [zstd, datagen, fuzzer, fullbench],
-    timeout: 600) # Timeout should work on HDD drive
+  if valgrind_prog.found()
+    test('valgrindTest',
+      valgrindTest_py,
+      args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
+      depends: [zstd, datagen, fuzzer, fullbench],
+      timeout: 600) # Timeout should work on HDD drive
+  endif
 endif
 
 if host_machine_os != os_windows

From 9c3e18f7feff00b6d816a6c4cbea906e0ef1fd93 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Wed, 14 Dec 2022 21:10:41 -0500
Subject: [PATCH 034/197] meson: add support for running both fast and slow
 version of tests

playTests.sh has an option to run really slow tests. This is enabled by
default in Meson, but what we really want is to do like the Makefile,
and run the fast ones by default, but with an option to run the slow
ones instead.
---
 build/meson/tests/meson.build | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index 09168de78..f7ba53101 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -21,7 +21,6 @@ FUZZER_FLAGS = ['--no-big-tests']
 FUZZERTEST = '-T200s'
 ZSTREAM_TESTTIME = '-T90s'
 DECODECORPUS_TESTTIME = '-T30'
-ZSTDRTTEST = ['--test-large-data']
 
 # =============================================================================
 # Executables
@@ -147,13 +146,25 @@ endif
 
 if host_machine_os != os_windows
   playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
-  test('test-zstd',
-    playTests_sh,
-    args: ZSTDRTTEST,
-    env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
-    depends: [datagen],
-    workdir: meson.current_build_dir(),
-    timeout: 2800) # Timeout should work on HDD drive
+
+  # add slow tests only if the meson version is new enough to support
+  # test setups with default-excluded suites
+  if meson.version().version_compare('>=0.57.0')
+    matrix = {'fast': [], 'slow': ['--test-large-data']}
+  else
+    matrix = {'fast': []}
+  endif
+
+  foreach suite, opt: matrix
+    test('test-zstd-'+suite,
+      playTests_sh,
+      args: opt,
+      env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
+      depends: [datagen],
+      suite: suite,
+      workdir: meson.current_build_dir(),
+      timeout: 2800) # Timeout should work on HDD drive
+  endforeach
 endif
 
 test('test-fullbench-1',
@@ -192,3 +203,11 @@ test('test-decodecorpus',
   args: ['-t', DECODECORPUS_TESTTIME],
   timeout: 60)
 test('test-poolTests', poolTests) # should be fast
+
+if meson.version().version_compare('>=0.57.0')
+  add_test_setup('fast',
+    is_default: true,
+    exclude_suites: ['slow'])
+  add_test_setup('slow',
+    exclude_suites: ['fast'])
+endif

From 7f29c1847d5b36908f13592c31fa178072b4db0f Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Wed, 14 Dec 2022 21:46:09 -0500
Subject: [PATCH 035/197] meson: add Linux CI

Travis is no longer run, but this wasn't ported to something else.
---
 .github/workflows/dev-short-tests.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 08df1c07c..423134fc2 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -178,6 +178,30 @@ jobs:
           make clean
           CC=clang MOREFLAGS="-Werror -Wimplicit-fallthrough -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
 
+  meson-linux:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install packages
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev
+          pip install --pre meson
+      - name: Build with Meson
+        run: |
+          meson setup \
+            --buildtype=debugoptimized \
+            -Db_lundef=false \
+            -Dauto_features=enabled \
+            -Dbin_programs=true \
+            -Dbin_tests=true \
+            -Dbin_contrib=true \
+            -Ddefault_library=both \
+            build/meson builddir
+          ninja -C builddir/
+          meson test -C builddir/ --print-errorlogs
+          meson install -C builddir --destdir staging/
+
   cmake-visual-2019:
     runs-on: windows-2019
     strategy:

From 937e9d3b6257304e1533e9e6104ecff8e8dd5a30 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Wed, 14 Dec 2022 21:57:59 -0500
Subject: [PATCH 036/197] meson: add Windows CI

There are a couple of oddities here. We don't attempt to build e.g.
contrib, because that doesn't seem to work at the moment. Also notice
that each command is its own step. This happens because github actions
runs in powershell, which doesn't seem to let you abort on the first
failure.
---
 .github/workflows/dev-short-tests.yml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 423134fc2..d4d8f13fd 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -202,6 +202,27 @@ jobs:
           meson test -C builddir/ --print-errorlogs
           meson install -C builddir --destdir staging/
 
+  meson-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install packages
+        run: pip install --pre meson
+      - name: Initialize the MSVC dev command prompt
+        uses: ilammy/msvc-dev-cmd@674ff850cbd739c402260838fa45b7114f750570
+      - name: Configure with Meson
+        run: |
+          meson setup build/meson/ builddir/ -Dbin_tests=true
+      - name: Build with Meson
+        run: |
+          ninja -C builddir/
+      - name: Test with Meson
+        run: |
+          meson test -C builddir/ --print-errorlogs
+      - name: Install with Meson
+        run: |
+          meson install -C builddir --destdir staging/
+
   cmake-visual-2019:
     runs-on: windows-2019
     strategy:

From 6747ba4ef5c5ff10e567fe6becf41436a745d0a1 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Tue, 19 Apr 2022 21:08:02 -0400
Subject: [PATCH 037/197] meson: mark a known test failure on Windows

---
 build/meson/tests/meson.build | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index f7ba53101..30df7d931 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -192,6 +192,8 @@ test('test-zstream-1',
 test('test-zstream-3',
   zstreamtest,
   args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  # --newapi dies on Windows with "exit status 3221225477 or signal 3221225349 SIGinvalid"
+  should_fail: host_machine_os == os_windows,
   timeout: 120)
 test('test-longmatch', longmatch, timeout: 36)
 test('test-invalidDictionaries', invalidDictionaries) # should be fast

From 58508398f4121f2a84092ac771db0f2b0fbb3b1a Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 16 Dec 2022 10:33:38 -0800
Subject: [PATCH 038/197] [CI] Re-enable versions-test

It seems like with the deletion of Travis CI we didn't successfully transfer the
version compatibility test. Attempt to enable the version compatibility test.
---
 .github/workflows/dev-short-tests.yml |  19 ++---
 tests/test-zstd-versions.py           | 102 +++++++++++---------------
 2 files changed, 48 insertions(+), 73 deletions(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 08df1c07c..69277d06c 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -431,18 +431,13 @@ jobs:
           cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd)
           ./simple LICENSE
 
-
-# This test currently fails on Github Actions specifically.
-# Possible reason : TTY emulation.
-# Note that the same test works fine locally and on travisCI.
-# This will have to be fixed before transferring the test to GA.
-#  versions-compatibility:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: actions/checkout@v3
-#    - name: Versions Compatibility Test
-#      run: |
-#        make -C tests versionsTest
+  versions-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Versions Compatibility Test
+      run: |
+        make -C tests versionsTest
 
 
 # For reference : icc tests
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index 94e3b8c5b..5f4b1e725 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -23,7 +23,7 @@ from subprocess import Popen, PIPE
 repo_url = 'https://github.com/facebook/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
 make_cmd = 'make'
-make_args = ['-j','CFLAGS=-O1']
+make_args = ['-j','CFLAGS=-O0']
 git_cmd = 'git'
 test_dat_src = 'README.md'
 test_dat = 'test_dat'
@@ -86,41 +86,46 @@ def create_dict(tag, dict_source_path):
         if result == 0:
             print(dict_name + ' created')
         else:
-            print('ERROR: creating of ' + dict_name + ' failed')
+            raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
     else:
         print(dict_name + ' already exists')
 
 
+def zstd(tag, args, input_file, output_file):
+    """
+    Zstd compress input_file to output_file.
+    Need this helper because 0.5.0 is broken when stdout is not a TTY.
+    Throws an exception if the command returns non-zero.
+    """
+    with open(input_file, "rb") as i:
+        with open(output_file, "wb") as o:
+            cmd = ['./zstd.' + tag] + args
+            print("Running: '{}', input={}, output={}" .format(
+                ' '.join(cmd), input_file, output_file
+            ))
+            subprocess.check_call(cmd, stdin=i, stdout=o)
+
+
 def dict_compress_sample(tag, sample):
     dict_name = 'dict.' + tag
-    DEVNULL = open(os.devnull, 'wb')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-15'], sample, sample + '_15_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-18'], sample, sample + '_18_64_' + tag + '_dictio.zst')
     # zstdFiles = glob.glob("*.zst*")
     # print(zstdFiles)
     print(tag + " : dict compression completed")
 
 
 def compress_sample(tag, sample):
-    DEVNULL = open(os.devnull, 'wb')
-    if subprocess.call(['./zstd.' + tag, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-1'], sample, sample + '_01_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-3'], sample, sample + '_03_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-5'], sample, sample + '_05_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-9'], sample, sample + '_09_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-15'], sample, sample + '_15_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-18'], sample, sample + '_18_64_' + tag + '_nodict.zst')
     # zstdFiles = glob.glob("*.zst*")
     # print(zstdFiles)
     print(tag + " : compression completed")
@@ -150,23 +155,13 @@ def decompress_zst(tag):
     dec_error = 0
     list_zst = sorted(glob.glob('*_nodict.zst'))
     for file_zst in list_zst:
-        print(file_zst, end=' ')
-        print(tag, end=' ')
+        print(file_zst + ' ' + tag)
         file_dec = file_zst + '_d64_' + tag + '.dec'
-        if tag <= 'v0.5.0':
-            params = ['./zstd.' + tag, '-df', file_zst, file_dec]
+        zstd(tag, ['-d'], file_zst, file_dec)
+        if not filecmp.cmp(file_dec, test_dat):
+            raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
         else:
-            params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
-        if execute(params) == 0:
-            if not filecmp.cmp(file_dec, test_dat):
-                print('ERR !! ')
-                dec_error = 1
-            else:
-                print('OK     ')
-        else:
-            print('command does not work')
-            dec_error = 1
-    return dec_error
+            print('OK     ')
 
 
 def decompress_dict(tag):
@@ -181,22 +176,13 @@ def decompress_dict(tag):
         if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
             continue
         dict_name = 'dict.' + dict_tag
-        print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
+        print(file_zst + ' ' + tag + ' dict=' + dict_tag)
         file_dec = file_zst + '_d64_' + tag + '.dec'
-        if tag <= 'v0.5.0':
-            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
+        zstd(tag, ['-D', dict_name, '-d'], file_zst, file_dec)
+        if not filecmp.cmp(file_dec, test_dat):
+            raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
         else:
-            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
-        if execute(params) == 0:
-            if not filecmp.cmp(file_dec, test_dat):
-                print('ERR !! ')
-                dec_error = 1
-            else:
-                print('OK     ')
-        else:
-            print('command does not work')
-            dec_error = 1
-    return dec_error
+            print('OK     ')
 
 
 if __name__ == '__main__':
@@ -267,25 +253,19 @@ if __name__ == '__main__':
     print('Compress test.dat by all released zstd')
     print('-----------------------------------------------')
 
-    error_code = 0
     for tag in tags:
         print(tag)
         if tag >= 'v0.5.0':
             create_dict(tag, dict_source_path)
             dict_compress_sample(tag, test_dat)
             remove_duplicates()
-            error_code += decompress_dict(tag)
+            decompress_dict(tag)
         compress_sample(tag, test_dat)
         remove_duplicates()
-        error_code += decompress_zst(tag)
+        decompress_zst(tag)
 
     print('')
     print('Enumerate different compressed files')
     zstds = sorted(glob.glob('*.zst'))
     for zstd in zstds:
         print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
-
-    if error_code != 0:
-        print('======  ERROR !!!  =======')
-
-    sys.exit(error_code)

From 97f63ce2b558edb44c290b0d3a3385f7abe809c4 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 16 Dec 2022 12:28:26 -0800
Subject: [PATCH 039/197] added unit tests for compressBound()

and rephrased the code documentation, as suggested by @terrelln
---
 lib/zstd.h     |  11 ++-
 tests/fuzzer.c | 187 +++++++++++++++++++++++++++----------------------
 2 files changed, 109 insertions(+), 89 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index dc635adae..04c64a8e2 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -204,11 +204,10 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
 /* ZSTD_compressBound() :
  * maximum compressed size in worst case single-pass scenario.
  * When invoking `ZSTD_compress()` or any other one-pass compression function,
- * providing @dstCapacity >= ZSTD_compressBound(srcSize) guarantees success.
- * Note that it's still allowed to provide a smaller @dstCapacity value,
- * in which case, the caller must inspect the return value with ZSTD_isError(),
- * to detect any potential compression failure.
- * Note : ZSTD_compressBound() itself can fail, if @srcSize is too large.
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
  *        In which case, ZSTD_compressBound() will return an error code
  *        which can be tested using ZSTD_isError().
  *
@@ -219,7 +218,7 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
  * Will produce constant value 0 if srcSize too large.
  */
 #define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
-#define ZSTD_COMPRESSBOUND(srcSize)   (((unsigned long long)(srcSize) > ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) > ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
 ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
 /* ZSTD_isError() :
  * Most ZSTD_* functions returning a size_t value can be tested for error,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 879e537bc..e15cf0648 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -82,8 +82,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 void FUZ_bug976(void);
 void FUZ_bug976(void)
 {   /* these constants shall not depend on MIN() macro */
-    assert(ZSTD_HASHLOG_MAX < 31);
-    assert(ZSTD_CHAINLOG_MAX < 31);
+    DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
+    DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
 }
 
 
@@ -118,23 +118,24 @@ static U32 FUZ_highbit32(U32 v32)
 /*=============================================
 *   Test macros
 =============================================*/
-#define CHECK_Z(f) {                               \
-    size_t const err = f;                          \
-    if (ZSTD_isError(err)) {                       \
-        DISPLAY("Error => %s : %s ",               \
-                #f, ZSTD_getErrorName(err));       \
-        exit(1);                                   \
+#define CHECK(fn)  { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
+
+#define CHECK_Z(f) {                          \
+    size_t const err = f;                     \
+    if (ZSTD_isError(err)) {                  \
+        DISPLAY("Error => %s : %s ",          \
+                #f, ZSTD_getErrorName(err));  \
+        exit(1);                              \
 }   }
 
-#define CHECK_VAR(var, fn)  var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
+#define CHECK_VAR(var, fn)  var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
 #define CHECK_NEWV(var, fn)  size_t const CHECK_VAR(var, fn)
-#define CHECK(fn)  { CHECK_NEWV(__err, fn); }
 #define CHECKPLUS(var, fn, more)  { CHECK_NEWV(var, fn); more; }
 
 #define CHECK_OP(op, lhs, rhs) {                                  \
     if (!((lhs) op (rhs))) {                                      \
         DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs);  \
-        goto _output_error;                                       \
+         exit(1);                                                 \
     }                                                             \
 }
 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
@@ -338,6 +339,7 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
 }
 
 #ifdef ZSTD_MULTITHREAD
+
 typedef struct {
     ZSTD_CCtx* cctx;
     ZSTD_threadPool* pool;
@@ -461,6 +463,28 @@ _output_error:
 *   Unit tests
 =============================================*/
 
+static void test_compressBound(int tnb)
+{
+    DISPLAYLEVEL(3, "test%3i : compressBound : ", tnb);
+
+    /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
+     * for a large range of known valid values */
+    DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
+    {   int s;
+        for (s=0; s<30; s++) {
+            size_t const w = (size_t)1 << s;
+            CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
+    }   }
+
+    // Ensure error if srcSize too big
+    {   size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
+        CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
+        CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
+    }
+
+    DISPLAYLEVEL(3, "OK \n");
+}
+
 static int basicUnitTests(U32 const seed, double compressibility)
 {
     size_t const CNBuffSize = 5 MB;
@@ -507,6 +531,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "%u (OK) \n", vn);
     }
 
+    test_compressBound(testNb++);
+
     DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
     {
         ZSTD_compressionParameters params;
@@ -1579,7 +1605,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             DISPLAYLEVEL(3, "OK \n");
 
             DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
-            CHECK( ZSTD_compressBegin(staticCCtx, 1) );
+            CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
             DISPLAYLEVEL(3, "OK \n");
 
             DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
@@ -1647,8 +1673,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
             testResult = 1;
             goto _end;
         }
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
@@ -1678,9 +1704,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
         CHECK_VAR(cSize, ZSTD_compress2(mtctx,
                                 compressedBuffer, compressedBufferSize,
                                 CNBuffer, CNBuffSize) );
@@ -1699,11 +1725,11 @@ static int basicUnitTests(U32 const seed, double compressibility)
         ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         char out[32];
         if (cctx == NULL || dctx == NULL) goto _output_error;
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
         CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
         DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
 
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
         {   char const* outPtr = out;
             ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
             ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
@@ -1718,9 +1744,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
     {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
         cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
-        CHECK(cSize);
+        CHECK_Z(cSize);
         ZSTD_freeCCtx(cctx);
     }
     DISPLAYLEVEL(3, "OK \n");
@@ -1728,13 +1754,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
     DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
     {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
         size_t cSize1, cSize2;
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
         cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
-        CHECK(cSize1);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
+        CHECK_Z(cSize1);
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
         cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
-        CHECK(cSize2);
+        CHECK_Z(cSize2);
         CHECK_LT(cSize1, cSize2);
         ZSTD_freeCCtx(cctx);
     }
@@ -1745,10 +1771,10 @@ static int basicUnitTests(U32 const seed, double compressibility)
         /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
          * ZSTDMT is forced to not take the shortcut.
          */
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
-        CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
+        CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
         ZSTD_freeCCtx(cctx);
     }
     DISPLAYLEVEL(3, "OK \n");
@@ -1758,22 +1784,22 @@ static int basicUnitTests(U32 const seed, double compressibility)
         int const jobSize = 512 KB;
         int value;
         /* Check that the overlap log and job size are unset. */
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 0);
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, 0);
         /* Set and check the overlap log and job size. */
-        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
-        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
+        CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 5);
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, jobSize);
         /* Set the number of workers and check the overlap log and job size. */
-        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 5);
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, jobSize);
         ZSTD_freeCCtxParams(params);
     }
@@ -1884,8 +1910,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
-        CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
-        CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
+        CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
+        CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
@@ -1945,8 +1971,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
         {   size_t const testSize = CNBuffSize / 3;
-            CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
-            CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
+            CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
+            CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
 
             CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
                                           (const char*)CNBuffer + dictSize, testSize) );
@@ -2780,7 +2806,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const wrongSrcSize = (srcSize + 1000);
         ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
         params.fParams.contentSizeFlag = 1;
-        CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
+        CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
         {   size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
             if (!ZSTD_isError(result)) goto _output_error;
             if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
@@ -2800,7 +2826,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
                                                      CNBuffer, srcSize, compressionLevel);
             if (ZSTD_isError(cSize_1pass)) goto _output_error;
 
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
             {   size_t const compressionResult = ZSTD_compress2(cctx,
                                     compressedBuffer, compressedBufferSize,
                                     CNBuffer, srcSize);
@@ -2819,13 +2845,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
         {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
             DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
             assert(cctx != NULL);
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
             {   size_t const compressedSize = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
                                 CNBuffer, inputSize);
-                CHECK(compressedSize);
+                CHECK_Z(compressedSize);
                 cSize = compressedSize;
                 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
             }
@@ -2835,13 +2861,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
             DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
             {   size_t const result = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
                                 CNBuffer, inputSize);
-                CHECK(result);
+                CHECK_Z(result);
                 if (result != cSize) goto _output_error;   /* must result in same compressed result, hence same size */
                 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error;  /* must result in exactly same content, hence same hash */
                 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
@@ -2856,7 +2882,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
         {   ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
-            CHECK(bounds.error);
+            CHECK_Z(bounds.error);
         }
         DISPLAYLEVEL(3, "OK \n");
 
@@ -2890,7 +2916,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
             ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
             size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2908,7 +2934,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_frameHeader zfh;
             size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
             if (zfhrt != 0) goto _output_error;
@@ -2930,7 +2956,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_inBuffer in = { CNBuffer, 0, 0 };
             ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
             size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2942,7 +2968,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
         /* one shot */
         {   size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
             if (result != 0) goto _output_error;
@@ -2977,13 +3003,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
             int check;
             if (ZSTD_isError(bounds.error))
                 continue;
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
             value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
-            CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2));
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
+            CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
             if (check != value2) goto _output_error;
-            CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
+            CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
             if (check != value1) goto _output_error;
         }
         ZSTD_freeDCtx(dctx);
@@ -3000,21 +3026,21 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin(cctx, 5) );
-        CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
+        CHECK_Z( ZSTD_compressBegin(cctx, 5) );
+        CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
         CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
-        CHECK( ZSTD_decompressBegin(dctx) );
+        CHECK_Z( ZSTD_decompressBegin(dctx) );
         { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
           if (r != blockSize) goto _output_error; }
         DISPLAYLEVEL(3, "OK \n");
 
         /* very long stream of block compression */
         DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin(cctx, -199) );  /* we just want to quickly overflow internal U32 index */
-        CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
+        CHECK_Z( ZSTD_compressBegin(cctx, -199) );  /* we just want to quickly overflow internal U32 index */
+        CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
         {   U64 const toCompress = 5000000000ULL;   /* > 4 GB */
             U64 compressed = 0;
             while (compressed < toCompress) {
@@ -3027,7 +3053,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* dictionary block compression */
         DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
+        CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
         CHECK_VAR(cSize,  ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
         RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed);  /* create a non-compressible second block */
         { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) );  /* for cctx history consistency */
@@ -3038,7 +3064,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
-        CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
+        CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
         {   CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
             if (r != blockSize) {
                 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
@@ -3057,8 +3083,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
         {   ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
             if (cdict==NULL) goto _output_error;
-            CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) );
-            CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
+            CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
             ZSTD_freeCDict(cdict);
         }
         DISPLAYLEVEL(3, "OK \n");
@@ -3227,7 +3253,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
         size_t nbSeq = 1;
         while (nbSeq <= maxNbSeq) {
-          CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
+          CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
           /* Check every sequence for the first 100, then skip more rapidly. */
           if (nbSeq < 100) {
             ++nbSeq;
@@ -3256,7 +3282,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const bound = ZSTD_compressBound(CNBuffSize);
         size_t size = 1;
         while (size <= CNBuffSize) {
-          CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
+          CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
           /* Check every size for the first 100, then skip more rapidly. */
           if (size < 100) {
             ++size;
@@ -3291,7 +3317,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             void* const outputBuffer = malloc(outputSize);
             ZSTD_CCtx* const cctx = ZSTD_createCCtx();
             if (!outputBuffer || !cctx) goto _output_error;
-            CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
+            CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
             free(outputBuffer);
             ZSTD_freeCCtx(cctx);
         }
@@ -3628,7 +3654,7 @@ static int longUnitTests(U32 const seed, double compressibility)
             while (approxIndex <= (maxIndex / 4) * 3) {
                 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
                 approxIndex += in.pos;
-                CHECK(in.pos == in.size);
+                CHECK_Z(in.pos == in.size);
                 in.pos = 0;
                 out.pos = 0;
             }
@@ -3654,7 +3680,7 @@ static int longUnitTests(U32 const seed, double compressibility)
             while (approxIndex <= maxIndex) {
                 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
                 approxIndex += in.pos;
-                CHECK(in.pos == in.size);
+                CHECK_Z(in.pos == in.size);
                 in.pos = 0;
                 out.pos = 0;
             }
@@ -3737,7 +3763,7 @@ static int longUnitTests(U32 const seed, double compressibility)
         RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
         RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
 
-        CHECK(cctx_params != NULL);
+        CHECK_Z(cctx_params != NULL);
 
         for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
             DISPLAYLEVEL(3, "\n    Testing with dictSize %u ", (U32)dictSize);
@@ -3780,11 +3806,6 @@ _end:
     free(compressedBuffer);
     free(decodedBuffer);
     return testResult;
-
-_output_error:
-    testResult = 1;
-    DISPLAY("Error detected in Unit tests ! \n");
-    goto _end;
 }
 
 

From 358a2374848d6b9c4fb20a254c038e27bb1527ac Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 15 Dec 2022 15:46:34 -0800
Subject: [PATCH 040/197] [api][visibility] Make the visibility macros more
 consistent

1. Follow the scheme introduced in PR #2501 for both `zdict.h` and `zstd_errors.h`.
2. If the `*_VISIBLE` macro isn't set, but the `*_VISIBILITY` macro is, use that.
   Also make this change for `zstd.h`, since we probably shouldn't have changed
   that macro name without backward compatibility in the first place.
3. Change all references to `*_VISIBILITY` to `*_VISIBLE`.

Fixes #3359.
---
 contrib/linux-kernel/Makefile |  4 +--
 lib/README.md                 |  7 ++++
 lib/module.modulemap          | 16 ++++++++--
 lib/zdict.h                   | 60 ++++++++++++++++++++++++-----------
 lib/zstd.h                    | 14 ++++++--
 lib/zstd_errors.h             | 26 +++++++++++----
 6 files changed, 95 insertions(+), 32 deletions(-)

diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index f80ee8653..c467bf927 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -49,8 +49,8 @@ libzstd:
 		-UZSTD_MULTITHREAD \
 		-U_MSC_VER \
 		-U_WIN32 \
-		-RZSTDLIB_VISIBILITY= \
-		-RZSTDERRORLIB_VISIBILITY= \
+		-RZSTDLIB_VISIBLE= \
+		-RZSTDERRORLIB_VISIBLE= \
 		-RZSTD_FALLTHROUGH=fallthrough \
 		-DZSTD_HAVE_WEAK_SYMBOLS=0 \
 		-DZSTD_TRACE=0 \
diff --git a/lib/README.md b/lib/README.md
index 015cde567..c3b5d1817 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -161,6 +161,13 @@ The file structure is designed to make this selection manually achievable for an
   `ZSTD_DCtx` decompression contexts,
   but might also result in a small decompression speed cost.
 
+- The C compiler macros `ZSTDLIB_VISIBLE`, `ZSTDERRORLIB_VISIBLE` and `ZDICTLIB_VISIBLE`
+  can be overridden to control the visibility of zstd's API. Additionally,
+  `ZSTDLIB_STATIC_API` and `ZDICTLIB_STATIC_API` can be overridden to control the visibility
+  of zstd's static API. Specifically, it can be set to `ZSTDLIB_HIDDEN` to hide the symbols
+  from the shared library. These macros default to `ZSTDLIB_VISIBILITY`,
+  `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility
+  with the old macro names.
 
 #### Windows : using MinGW+MSYS to create DLL
 
diff --git a/lib/module.modulemap b/lib/module.modulemap
index bbb939782..eff98dfac 100644
--- a/lib/module.modulemap
+++ b/lib/module.modulemap
@@ -1,17 +1,27 @@
 module libzstd [extern_c] {
     header "zstd.h"
     export *
-    config_macros [exhaustive] /* zstd.h */ \
+    config_macros [exhaustive] \
+        /* zstd.h */ \
         ZSTD_STATIC_LINKING_ONLY, \
+        ZSTDLIB_VISIBILITY, \
         ZSTDLIB_VISIBLE, \
+        ZSTDLIB_HIDDEN, \
         ZSTD_DLL_EXPORT, \
         ZSTDLIB_STATIC_API, \
         ZSTD_DISABLE_DEPRECATE_WARNINGS, \
         ZSTD_CLEVEL_DEFAULT, \
-        /* zdict.h */ ZDICT_STATIC_LINKING_ONLY, \
+        /* zdict.h */ \
+        ZDICT_STATIC_LINKING_ONLY, \
+        ZDICTLIB_VISIBLE, \
+        ZDICTLIB_HIDDEN, \
         ZDICTLIB_VISIBILITY, \
+        ZDICTLIB_STATIC_API, \
         ZDICT_DISABLE_DEPRECATE_WARNINGS, \
-        /* zstd_errors.h */ ZSTDERRORLIB_VISIBILITY
+        /* zstd_errors.h */ \
+        ZSTDERRORLIB_VISIBLE, \
+        ZSTDERRORLIB_HIDDEN, \
+        ZSTDERRORLIB_VISIBILITY
 
     module dictbuilder [extern_c] {
         header "zdict.h"
diff --git a/lib/zdict.h b/lib/zdict.h
index 8e21ba0f7..8231f5a2b 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -21,19 +21,31 @@ extern "C" {
 
 
 /* =====   ZDICTLIB_API : control library symbols visibility   ===== */
-#ifndef ZDICTLIB_VISIBILITY
-#  if defined(__GNUC__) && (__GNUC__ >= 4)
-#    define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#ifndef ZDICTLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZDICTLIB_VISIBILITY
+#    define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
 #  else
-#    define ZDICTLIB_VISIBILITY
+#    define ZDICTLIB_VISIBLE
 #  endif
 #endif
+
+#ifndef ZDICTLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZDICTLIB_HIDDEN
+#  endif
+#endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define ZDICTLIB_API ZDICTLIB_VISIBILITY
+#  define ZDICTLIB_API ZDICTLIB_VISIBLE
 #endif
 
 /*******************************************************************************
@@ -264,6 +276,17 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
 
 #ifdef ZDICT_STATIC_LINKING_ONLY
 
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZDICTLIB_STATIC_API
+#  if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#    define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
+#  elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#    define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
+#  else
+#    define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
+#  endif
+#endif
+
 /* ====================================================================================
  * The definitions in this section are considered experimental.
  * They should never be used with a dynamic library, as they may change in the future.
@@ -318,7 +341,7 @@ typedef struct {
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
           void *dictBuffer, size_t dictBufferCapacity,
     const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
           ZDICT_cover_params_t parameters);
@@ -340,7 +363,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
  *          See ZDICT_trainFromBuffer() for details on failure modes.
  * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
  */
-ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
           void* dictBuffer, size_t dictBufferCapacity,
     const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
           ZDICT_cover_params_t* parameters);
@@ -361,7 +384,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
                     size_t dictBufferCapacity, const void *samplesBuffer,
                     const size_t *samplesSizes, unsigned nbSamples,
                     ZDICT_fastCover_params_t parameters);
@@ -384,7 +407,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
  *          See ZDICT_trainFromBuffer() for details on failure modes.
  * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
  */
-ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
                     size_t dictBufferCapacity, const void* samplesBuffer,
                     const size_t* samplesSizes, unsigned nbSamples,
                     ZDICT_fastCover_params_t* parameters);
@@ -409,7 +432,7 @@ typedef struct {
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  *  Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
     void* dictBuffer, size_t dictBufferCapacity,
     const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
     ZDICT_legacy_params_t parameters);
@@ -421,23 +444,24 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
    or _CRT_SECURE_NO_WARNINGS in Visual.
    Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
 #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
-#  define ZDICT_DEPRECATED(message) ZDICTLIB_API   /* disable deprecation warnings */
+#  define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
 #else
 #  define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
 #  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
-#    define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
+#    define ZDICT_DEPRECATED(message) [[deprecated(message)]]
 #  elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
+#    define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (ZDICT_GCC_VERSION >= 301)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
+#    define ZDICT_DEPRECATED(message) __attribute__((deprecated))
 #  elif defined(_MSC_VER)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
+#    define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
 #  else
 #    pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API
+#    define ZDICT_DEPRECATED(message)
 #  endif
 #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
 
+ZDICTLIB_STATIC_API
 ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
 size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
                                   const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
diff --git a/lib/zstd.h b/lib/zstd.h
index 1dff31b4e..404aa0800 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -21,14 +21,24 @@ extern "C" {
 
 /* =====   ZSTDLIB_API : control library symbols visibility   ===== */
 #ifndef ZSTDLIB_VISIBLE
-#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDLIB_VISIBILITY
+#    define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
 #    define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
-#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
 #  else
 #    define ZSTDLIB_VISIBLE
+#  endif
+#endif
+
+#ifndef ZSTDLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
 #    define ZSTDLIB_HIDDEN
 #  endif
 #endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
 #  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 2ec0b0ab1..68ce01cd1 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -20,19 +20,31 @@ extern "C" {
 
 
 /* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
-#ifndef ZSTDERRORLIB_VISIBILITY
-#  if defined(__GNUC__) && (__GNUC__ >= 4)
-#    define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#ifndef ZSTDERRORLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
 #  else
-#    define ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE
 #  endif
 #endif
+
+#ifndef ZSTDERRORLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDERRORLIB_HIDDEN
+#  endif
+#endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
 #endif
 
 /*-*********************************************

From 0c42424a1ed73555906cd10e9c40e8316c8584e2 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 15 Dec 2022 16:40:04 -0800
Subject: [PATCH 041/197] [build] Fix ZSTD_LIB_MINIFY build option

`ZSTD_LIB_MINIFY` broke in 8bf699aa59372d7c2bb4216bcf8037cab7dae51e.

This commit fixes the macro and the static library shrinks from ~600K to 324K
with ZSTD_LIB_MINIFY set.

Fixes #3066.
---
 .github/workflows/dev-short-tests.yml | 14 ++++++++++
 lib/libzstd.mk                        | 40 ++++++++++++++++-----------
 tests/check_size.py                   | 31 +++++++++++++++++++++
 3 files changed, 69 insertions(+), 16 deletions(-)
 create mode 100755 tests/check_size.py

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 69277d06c..50991ff4f 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -236,6 +236,20 @@ jobs:
 #        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140
 #        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
 
+  # This tests that we don't accidently grow the size too much.
+  # If the size grows intentionally, you can raise these numbers.
+  # But we do need to think about binary size, since it is a concern.
+  libzstd-size:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: libzstd size test
+      run: |
+        make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
+        make clean && make -j -C lib libzstd ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 && ./tests/check_size.py lib/libzstd.so 400000
+        make clean && make -j -C lib libzstd ZSTD_LIB_MINIFY=1 && ./tests/check_size.py lib/libzstd.so 300000
+        make clean && make -j -C lib libzstd ZSTD_LIB_MINIFY=1 ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 && ./tests/check_size.py lib/libzstd.so 80000
+
   minimal-decompressor-macros:
     runs-on: ubuntu-latest
     steps:
diff --git a/lib/libzstd.mk b/lib/libzstd.mk
index c97db5628..60cab8cc8 100644
--- a/lib/libzstd.mk
+++ b/lib/libzstd.mk
@@ -15,17 +15,34 @@
 # Zstd lib directory
 LIBZSTD ?= ./
 
+# ZSTD_LIB_MINIFY is a helper variable that
+# configures a bunch of other variables to space-optimized defaults.
+ZSTD_LIB_MINIFY ?= 0
+
 # Legacy support
-ZSTD_LEGACY_SUPPORT ?= 5
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  ZSTD_LEGACY_SUPPORT ?= 0
+else
+  ZSTD_LEGACY_SUPPORT ?= 5
+endif
 ZSTD_LEGACY_MULTITHREADED_API ?= 0
 
 # Build size optimizations
-HUF_FORCE_DECOMPRESS_X1 ?= 0
-HUF_FORCE_DECOMPRESS_X2 ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
-ZSTD_NO_INLINE ?= 0
-ZSTD_STRIP_ERROR_STRINGS ?= 0
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  HUF_FORCE_DECOMPRESS_X1 ?= 1
+  HUF_FORCE_DECOMPRESS_X2 ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
+  ZSTD_NO_INLINE ?= 1
+  ZSTD_STRIP_ERROR_STRINGS ?= 1
+else
+  HUF_FORCE_DECOMPRESS_X1 ?= 0
+  HUF_FORCE_DECOMPRESS_X2 ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
+  ZSTD_NO_INLINE ?= 0
+  ZSTD_STRIP_ERROR_STRINGS ?= 0
+endif
 
 # Assembly support
 ZSTD_NO_ASM ?= 0
@@ -61,17 +78,8 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
 CCVER := $(shell $(CC) --version)
 ZSTD_VERSION?= $(LIBVER)
 
-# ZSTD_LIB_MINIFY is a helper variable that
-# configures a bunch of other variables to space-optimized defaults.
-ZSTD_LIB_MINIFY ?= 0
 ifneq ($(ZSTD_LIB_MINIFY), 0)
   HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
-  ZSTD_LEGACY_SUPPORT ?= 0
-  ZSTD_LIB_DEPRECATED ?= 0
-  HUF_FORCE_DECOMPRESS_X1 ?= 1
-  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
-  ZSTD_NO_INLINE ?= 1
-  ZSTD_STRIP_ERROR_STRINGS ?= 1
 ifneq ($(HAVE_CC_OZ), 0)
     # Some compilers (clang) support an even more space-optimized setting.
     CFLAGS += -Oz
diff --git a/tests/check_size.py b/tests/check_size.py
new file mode 100755
index 000000000..408f0f795
--- /dev/null
+++ b/tests/check_size.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+import os
+import subprocess
+import sys
+
+if len(sys.argv) != 3:
+	print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT")
+	sys.exit(1)
+
+file = sys.argv[1]
+limit = int(sys.argv[2])
+
+if not os.path.exists(file):
+	print(f"{file} does not exist")
+	sys.exit(1)
+
+size = os.path.getsize(file)
+
+if size > limit:
+	print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes")
+	sys.exit(1)

From 2f7b8d47fb580ab931a67d2b323f0883395a0ab9 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 16 Dec 2022 12:04:50 -0800
Subject: [PATCH 042/197] [zdict] Fix static linking only include guards

Fix `zdict.h` static linking only section so if you include it twice it
still exposes the static linking only symbols. E.g. this pattern:

```
```

This can easily happen when a header you include includes `zdict.h`.
---
 lib/zdict.h | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/zdict.h b/lib/zdict.h
index 8231f5a2b..231bf8282 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -8,13 +8,12 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-#ifndef DICTBUILDER_H_001
-#define DICTBUILDER_H_001
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef ZSTD_ZDICT_H
+#define ZSTD_ZDICT_H
 
 /*======  Dependencies  ======*/
 #include <stddef.h>  /* size_t */
@@ -272,9 +271,10 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS
 ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
 ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
 
+#endif   /* ZSTD_ZDICT_H */
 
-
-#ifdef ZDICT_STATIC_LINKING_ONLY
+#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
+#define ZSTD_ZDICT_H_STATIC
 
 /* This can be overridden externally to hide static symbols. */
 #ifndef ZDICTLIB_STATIC_API
@@ -467,10 +467,8 @@ size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize
                                   const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
 
-#endif   /* ZDICT_STATIC_LINKING_ONLY */
+#endif   /* ZSTD_ZDICT_H_STATIC */
 
 #if defined (__cplusplus)
 }
 #endif
-
-#endif   /* DICTBUILDER_H_001 */

From ea24b886673a1e154a81d71222d130f65da7a0ed Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 16 Dec 2022 15:32:19 -0800
Subject: [PATCH 043/197] decompressBound() tests

fixed an overflow in an intermediate result on 32-bit platform.
Checked that the new test catch this bug in 32-bit mode.
---
 lib/decompress/zstd_decompress.c |  2 +-
 tests/fuzzer.c                   | 57 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e95b8822f..57f220d0c 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -785,7 +785,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
         frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
                                         ? zfh.frameContentSize
-                                        : nbBlocks * zfh.blockSizeMax;
+                                        : (unsigned long long)nbBlocks * zfh.blockSizeMax;
         return frameSizeInfo;
     }
 }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index e15cf0648..c33228523 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -485,6 +485,61 @@ static void test_compressBound(int tnb)
     DISPLAYLEVEL(3, "OK \n");
 }
 
+static void test_decompressBound(int tnb)
+{
+    DISPLAYLEVEL(3, "test%3i : decompressBound : ", tnb);
+
+    // Simple compression, with size : should provide size;
+    {   const char example[] = "abcd";
+        char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
+        size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
+        CHECK_Z(cSize);
+        {   size_t const dbSize = ZSTD_decompressBound(cBuffer, cSize);
+            CHECK_Z(dbSize);
+            CHECK_EQ(dbSize, sizeof(example));
+    }   }
+
+    // Simple small compression without size : should provide 1 block size
+    {   char cBuffer[ZSTD_COMPRESSBOUND(0)];
+        ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
+        ZSTD_inBuffer in = { NULL, 0, 0 };
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        assert(cctx);
+        CHECK_Z( ZSTD_initCStream(cctx, 0) );
+        CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
+        CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
+        CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
+        ZSTD_freeCCtx(cctx);
+    }
+
+    // Attempt to overflow 32-bit intermediate multiplication result
+    // This requires dBound >= 4 GB, aka 2^32.
+    // This requires 2^32 / 2^17 = 2^15 blocks
+    // => create 2^15 blocks (can be empty, or just 1 byte).
+    {   const char input[] = "a";
+        size_t const nbBlocks = (1 << 15) + 1;
+        size_t blockNb;
+        size_t const outCapacity = 1 << 18; // large margin
+        char* const outBuffer = malloc (outCapacity);
+        ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        assert(cctx);
+        assert(outBuffer);
+        CHECK_Z( ZSTD_initCStream(cctx, 0) );
+        for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+            ZSTD_inBuffer in = { input, sizeof(input), 0 };
+            CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
+            CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
+        }
+        CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
+        CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000LLU /* 4 GB */ );
+        ZSTD_freeCCtx(cctx);
+        free(outBuffer);
+    }
+
+    DISPLAYLEVEL(3, "OK \n");
+}
+
 static int basicUnitTests(U32 const seed, double compressibility)
 {
     size_t const CNBuffSize = 5 MB;
@@ -533,6 +588,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     test_compressBound(testNb++);
 
+    test_decompressBound(testNb++);
+
     DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
     {
         ZSTD_compressionParameters params;

From 2f4238e47ac6dcf923dcf95f129a283cb1cfa642 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 16 Dec 2022 15:58:25 -0800
Subject: [PATCH 044/197] make ZSTD_DECOMPRESSBOUND() compatible with input
 size 0

for environments with stringent compilation warnings.
---
 lib/zstd.h     | 2 +-
 tests/fuzzer.c | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 065a1d652..09a200af2 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -228,7 +228,7 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
  * Will produce constant value 0 if srcSize too large.
  */
 #define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
-#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) > ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
 ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
 /* ZSTD_isError() :
  * Most ZSTD_* functions returning a size_t value can be tested for error,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c33228523..9d1862077 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -494,10 +494,8 @@ static void test_decompressBound(int tnb)
         char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
         size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
         CHECK_Z(cSize);
-        {   size_t const dbSize = ZSTD_decompressBound(cBuffer, cSize);
-            CHECK_Z(dbSize);
-            CHECK_EQ(dbSize, sizeof(example));
-    }   }
+        CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
+    }
 
     // Simple small compression without size : should provide 1 block size
     {   char cBuffer[ZSTD_COMPRESSBOUND(0)];

From 3867c41552ee648bfe0cc63407ab39dc1d2adc1d Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <yoniko@gmail.com>
Date: Sat, 17 Dec 2022 10:33:49 -0800
Subject: [PATCH 045/197] meson: Fix Windows CI test by pulling a more recent
 version msvc-dev-cmd

---
 .github/workflows/dev-short-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index ffbf4f3cd..e1d4cc40e 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -209,7 +209,7 @@ jobs:
       - name: Install packages
         run: pip install --pre meson
       - name: Initialize the MSVC dev command prompt
-        uses: ilammy/msvc-dev-cmd@674ff850cbd739c402260838fa45b7114f750570
+        uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d
       - name: Configure with Meson
         run: |
           meson setup build/meson/ builddir/ -Dbin_tests=true

From 500f02eb66419d399a521e685825846c4da0acf2 Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <yoniko@gmail.com>
Date: Thu, 15 Dec 2022 16:11:56 -0800
Subject: [PATCH 046/197] Fixes two bugs in the Windows thread / pthread
 translation layer

1. If threads are resized the threads' `ZSTD_pthread_t` might move
while the worker still holds a pointer into it (see more details in #3120).
2. The join operation was waiting for a thread and then return its `thread.arg`
as a return value, but since the `ZSTD_pthread_t thread` was passed by value it
would have a stale `arg` that wouldn't match the thread's actual return value.

This fix changes the `ZSTD_pthread_join` API and removes support for returning
a value. This means that we are diverging from the `pthread_join` API and this
is no longer just an alias.
In the future, if needed, we could return a Windows thread's return value using
`GetExitCodeThread`, but as this path wouldn't be excised in any case, it's
preferable to not add it right now.
---
 lib/common/pool.c      | 2 +-
 lib/common/threading.c | 5 ++---
 lib/common/threading.h | 6 +++---
 tests/fuzzer.c         | 4 ++--
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index bf21c57ed..63dc9d377 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -173,7 +173,7 @@ static void POOL_join(POOL_ctx* ctx) {
     /* Join all of the threads */
     {   size_t i;
         for (i = 0; i < ctx->threadCapacity; ++i) {
-            ZSTD_pthread_join(ctx->threads[i], NULL);  /* note : could fail */
+            ZSTD_pthread_join(ctx->threads[i]);  /* note : could fail */
     }   }
 }
 
diff --git a/lib/common/threading.c b/lib/common/threading.c
index c62c65d57..6bbb14937 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -37,7 +37,7 @@ int g_ZSTD_threading_useless_symbol;
 static unsigned __stdcall worker(void *arg)
 {
     ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
-    thread->arg = thread->start_routine(thread->arg);
+    thread->start_routine(thread->arg);
     return 0;
 }
 
@@ -55,7 +55,7 @@ int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
         return 0;
 }
 
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
+int ZSTD_pthread_join(ZSTD_pthread_t thread)
 {
     DWORD result;
 
@@ -66,7 +66,6 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
 
     switch (result) {
     case WAIT_OBJECT_0:
-        if (value_ptr) *value_ptr = thread.arg;
         return 0;
     case WAIT_ABANDONED:
         return EINVAL;
diff --git a/lib/common/threading.h b/lib/common/threading.h
index b1458054e..603d479c7 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -70,7 +70,7 @@ typedef struct {
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
                    void* (*start_routine) (void*), void* arg);
 
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
+int ZSTD_pthread_join(ZSTD_pthread_t thread);
 
 /**
  * add here more wrappers as required
@@ -98,7 +98,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
 
 #define ZSTD_pthread_t                  pthread_t
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+#define ZSTD_pthread_join(a)         pthread_join((a),NULL)
 
 #else /* DEBUGLEVEL >= 1 */
 
@@ -123,7 +123,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
 
 #define ZSTD_pthread_t                  pthread_t
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+#define ZSTD_pthread_join(a)         pthread_join((a),NULL)
 
 #endif
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9d1862077..d58751a5c 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -431,8 +431,8 @@ static int threadPoolTests(void) {
 
         ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
         ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
-        ZSTD_pthread_join(t1, NULL);
-        ZSTD_pthread_join(t2, NULL);
+        ZSTD_pthread_join(t1);
+        ZSTD_pthread_join(t2);
 
         assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
         free(decodedBuffer2);

From ec42c92aaa13815164bb0ecb6e2483aff2fc78d9 Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <yoniko@gmail.com>
Date: Fri, 16 Dec 2022 18:24:02 -0800
Subject: [PATCH 047/197] Fix race condition in the Windows thread / pthread
 translation layer

When spawning a Windows thread we have small worker wrapper function that translates
between the interfaces of Windows and POSIX threads.
This wrapper is given a pointer that might get stale before the worker starts running,
resulting in UB and crashes.
This commit adds synchronization so that we know the wrapper has finished reading the data
it needs before we allow the main thread to resume execution.
---
 lib/common/threading.c | 59 ++++++++++++++++++++++++++++++++++--------
 lib/common/threading.h |  6 +----
 2 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/lib/common/threading.c b/lib/common/threading.c
index 6bbb14937..825826500 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -34,35 +34,72 @@ int g_ZSTD_threading_useless_symbol;
 
 /* ===  Implementation  === */
 
+typedef struct {
+    void* (*start_routine)(void*);
+    void* arg;
+    int initialized;
+    ZSTD_pthread_cond_t initialized_cond;
+    ZSTD_pthread_mutex_t initialized_mutex;
+} ZSTD_thread_params_t;
+
 static unsigned __stdcall worker(void *arg)
 {
-    ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
-    thread->start_routine(thread->arg);
+    ZSTD_thread_params_t* const thread_param = (ZSTD_thread_params_t*)arg;
+    void* (*start_routine)(void*) = thread_param->start_routine;
+    void* thread_arg = thread_param->arg;
+
+    /* Signal main thread that we are running and do not depend on its memory anymore */
+    ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex);
+    thread_param->initialized = 1;
+    ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex);
+    ZSTD_pthread_cond_signal(&thread_param->initialized_cond);
+
+    start_routine(thread_arg);
+
     return 0;
 }
 
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
             void* (*start_routine) (void*), void* arg)
 {
+    ZSTD_thread_params_t thread_param;
+    int error = 0;
     (void)unused;
-    thread->arg = arg;
-    thread->start_routine = start_routine;
-    thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
+    thread_param.start_routine = start_routine;
+    thread_param.arg = arg;
+    thread_param.initialized = 0;
 
-    if (!thread->handle)
+    /* Setup thread initialization synchronization */
+    error |= ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL);
+    error |= ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL);
+    if(error)
+        return -1;
+    ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex);
+
+    /* Spawn thread */
+    *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL);
+    if (!thread)
         return errno;
-    else
-        return 0;
+
+    /* Wait for thread to be initialized */
+    while(!thread_param.initialized) {
+        ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex);
+    }
+    ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex);
+    ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
+    ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
+
+    return 0;
 }
 
 int ZSTD_pthread_join(ZSTD_pthread_t thread)
 {
     DWORD result;
 
-    if (!thread.handle) return 0;
+    if (!thread) return 0;
 
-    result = WaitForSingleObject(thread.handle, INFINITE);
-    CloseHandle(thread.handle);
+    result = WaitForSingleObject(thread, INFINITE);
+    CloseHandle(thread);
 
     switch (result) {
     case WAIT_OBJECT_0:
diff --git a/lib/common/threading.h b/lib/common/threading.h
index 603d479c7..fb5c1c878 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -61,11 +61,7 @@ extern "C" {
 #define ZSTD_pthread_cond_broadcast(a)  WakeAllConditionVariable((a))
 
 /* ZSTD_pthread_create() and ZSTD_pthread_join() */
-typedef struct {
-    HANDLE handle;
-    void* (*start_routine)(void*);
-    void* arg;
-} ZSTD_pthread_t;
+typedef HANDLE ZSTD_pthread_t;
 
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
                    void* (*start_routine) (void*), void* arg);

From aaa38b29bbd91592c866917aefca10eb71c645c7 Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <yoniko@gmail.com>
Date: Fri, 16 Dec 2022 18:33:49 -0800
Subject: [PATCH 048/197] meson: zstreamtests should now pass on Windows

---
 build/meson/tests/meson.build | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index 30df7d931..f7ba53101 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -192,8 +192,6 @@ test('test-zstream-1',
 test('test-zstream-3',
   zstreamtest,
   args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
-  # --newapi dies on Windows with "exit status 3221225477 or signal 3221225349 SIGinvalid"
-  should_fail: host_machine_os == os_windows,
   timeout: 120)
 test('test-longmatch', longmatch, timeout: 36)
 test('test-invalidDictionaries', invalidDictionaries) # should be fast

From 832c1a6a1c4dfea4c5dd78074226e656b2cd3d56 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Sun, 18 Dec 2022 04:12:32 -0800
Subject: [PATCH 049/197] minor reformatting

and minor reliability and maintenance changes
---
 lib/compress/zstd_compress.c          | 409 ++++++++++++++------------
 lib/compress/zstd_compress_internal.h |  13 +-
 2 files changed, 230 insertions(+), 192 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index cb4963669..ecb21f268 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2496,11 +2496,13 @@ typedef struct {
  * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
-                        const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
-                              BYTE* dst, const BYTE* const dstEnd,
-                              ZSTD_strategy strategy, unsigned* countWorkspace,
-                              void* entropyWorkspace, size_t entropyWkspSize) {
+ZSTD_buildSequencesStatistics(
+                const seqStore_t* seqStorePtr, size_t nbSeq,
+                const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
+                      BYTE* dst, const BYTE* const dstEnd,
+                      ZSTD_strategy strategy, unsigned* countWorkspace,
+                      void* entropyWorkspace, size_t entropyWkspSize)
+{
     BYTE* const ostart = dst;
     const BYTE* const oend = dstEnd;
     BYTE* op = ostart;
@@ -2619,13 +2621,14 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
  */
 #define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
-                          const ZSTD_entropyCTables_t* prevEntropy,
-                                ZSTD_entropyCTables_t* nextEntropy,
-                          const ZSTD_CCtx_params* cctxParams,
-                                void* dst, size_t dstCapacity,
-                                void* entropyWorkspace, size_t entropyWkspSize,
-                          const int bmi2)
+ZSTD_entropyCompressSeqStore_internal(
+                        const seqStore_t* seqStorePtr,
+                        const ZSTD_entropyCTables_t* prevEntropy,
+                              ZSTD_entropyCTables_t* nextEntropy,
+                        const ZSTD_CCtx_params* cctxParams,
+                              void* dst, size_t dstCapacity,
+                              void* entropyWorkspace, size_t entropyWkspSize,
+                        const int bmi2)
 {
     const int longOffsets = cctxParams->cParams.windowLog >= STREAM_ACCUMULATOR_MIN;
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
@@ -2634,7 +2637,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
     FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
     FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
     const seqDef* const sequences = seqStorePtr->sequencesStart;
-    const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
     const BYTE* const llCodeTable = seqStorePtr->llCode;
     const BYTE* const mlCodeTable = seqStorePtr->mlCode;
@@ -2652,8 +2655,8 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
 
     /* Compress literals */
     {   const BYTE* const literals = seqStorePtr->litStart;
-        size_t const numSequences = seqStorePtr->sequences - seqStorePtr->sequencesStart;
-        size_t const numLiterals = seqStorePtr->lit - seqStorePtr->litStart;
+        size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+        size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
         /* Base suspicion of uncompressibility on ratio of literals to sequences */
         unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
         size_t const litSize = (size_t)(seqStorePtr->lit - literals);
@@ -2738,14 +2741,15 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
 }
 
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
-                       const ZSTD_entropyCTables_t* prevEntropy,
-                             ZSTD_entropyCTables_t* nextEntropy,
-                       const ZSTD_CCtx_params* cctxParams,
-                             void* dst, size_t dstCapacity,
-                             size_t srcSize,
-                             void* entropyWorkspace, size_t entropyWkspSize,
-                             int bmi2)
+ZSTD_entropyCompressSeqStore(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          void* dst, size_t dstCapacity,
+                          size_t srcSize,
+                          void* entropyWorkspace, size_t entropyWkspSize,
+                          int bmi2)
 {
     size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
                             seqStorePtr, prevEntropy, nextEntropy, cctxParams,
@@ -3058,19 +3062,17 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) {
     const size_t unrollMask = unrollSize - 1;
     const size_t prefixLength = length & unrollMask;
     size_t i;
-    size_t u;
     if (length == 1) return 1;
     /* Check if prefix is RLE first before using unrolled loop */
     if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
         return 0;
     }
     for (i = prefixLength; i != length; i += unrollSize) {
+        size_t u;
         for (u = 0; u < unrollSize; u += sizeof(size_t)) {
             if (MEM_readST(ip + i + u) != valueST) {
                 return 0;
-            }
-        }
-    }
+    }   }   }
     return 1;
 }
 
@@ -3086,7 +3088,8 @@ static int ZSTD_maybeRLE(seqStore_t const* seqStore)
     return nbSeqs < 4 && nbLits < 10;
 }
 
-static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
+static void
+ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
 {
     ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock;
     bs->prevCBlock = bs->nextCBlock;
@@ -3094,7 +3097,9 @@ static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* c
 }
 
 /* Writes the block header */
-static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) {
+static void
+writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
+{
     U32 const cBlockHeader = cSize == 1 ?
                         lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
                         lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
@@ -3107,13 +3112,16 @@ static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastB
  *  Stores literals block type (raw, rle, compressed, repeat) and
  *  huffman description table to hufMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE workspace
- *  @return : size of huffman description table or error code */
-static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
-                                            const ZSTD_hufCTables_t* prevHuf,
-                                                  ZSTD_hufCTables_t* nextHuf,
-                                                  ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                  const int literalsCompressionIsDisabled,
-                                                  void* workspace, size_t wkspSize, HUF_depth_mode depthMode)
+ * @return : size of huffman description table, or an error code
+ */
+static size_t
+ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
+                               const ZSTD_hufCTables_t* prevHuf,
+                                     ZSTD_hufCTables_t* nextHuf,
+                                     ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               const int literalsCompressionIsDisabled,
+                                     void* workspace, size_t wkspSize,
+                                     HUF_depth_mode depthMode)
 {
     BYTE* const wkspStart = (BYTE*)workspace;
     BYTE* const wkspEnd = wkspStart + wkspSize;
@@ -3138,33 +3146,37 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
 
     /* small ? don't even attempt compression (speed opt) */
 #ifndef COMPRESS_LITERALS_SIZE_MIN
-#define COMPRESS_LITERALS_SIZE_MIN 63
+# define COMPRESS_LITERALS_SIZE_MIN 63  /* heuristic */
 #endif
     {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
         if (srcSize <= minLitSize) {
             DEBUGLOG(5, "set_basic - too small");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Scan input and build symbol stats */
-    {   size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
+    {   size_t const largest =
+            HIST_count_wksp (countWksp, &maxSymbolValue,
+                            (const BYTE*)src, srcSize,
+                            workspace, wkspSize);
         FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
         if (largest == srcSize) {
+            /* only one literal symbol */
             DEBUGLOG(5, "set_rle");
             hufMetadata->hType = set_rle;
             return 0;
         }
         if (largest <= (srcSize >> 7)+4) {
+            /* heuristic: likely not compressible */
             DEBUGLOG(5, "set_basic - no gain");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Validate the previous Huffman table */
-    if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
+    if (repeat == HUF_repeat_check
+      && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
         repeat = HUF_repeat_none;
     }
 
@@ -3177,35 +3189,34 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
                                                     nodeWksp, nodeWkspSize);
         FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
         huffLog = (U32)maxBits;
-        {   /* Build and write the CTable */
-            size_t const newCSize = HUF_estimateCompressedSize(
-                    (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
-            size_t const hSize = HUF_writeCTable_wksp(
-                    hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
-                    (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
-                    nodeWksp, nodeWkspSize);
-            /* Check against repeating the previous CTable */
-            if (repeat != HUF_repeat_none) {
-                size_t const oldCSize = HUF_estimateCompressedSize(
-                        (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
-                if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
-                    DEBUGLOG(5, "set_repeat - smaller");
-                    ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                    hufMetadata->hType = set_repeat;
-                    return 0;
-                }
-            }
-            if (newCSize + hSize >= srcSize) {
-                DEBUGLOG(5, "set_basic - no gains");
+    }
+    {   /* Build and write the CTable */
+        size_t const newCSize = HUF_estimateCompressedSize(
+                (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
+        size_t const hSize = HUF_writeCTable_wksp(
+                hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
+                (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
+                nodeWksp, nodeWkspSize);
+        /* Check against repeating the previous CTable */
+        if (repeat != HUF_repeat_none) {
+            size_t const oldCSize = HUF_estimateCompressedSize(
+                    (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
+            if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
+                DEBUGLOG(5, "set_repeat - smaller");
                 ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                hufMetadata->hType = set_basic;
+                hufMetadata->hType = set_repeat;
                 return 0;
-            }
-            DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
-            hufMetadata->hType = set_compressed;
-            nextHuf->repeatMode = HUF_repeat_check;
-            return hSize;
+        }   }
+        if (newCSize + hSize >= srcSize) {
+            DEBUGLOG(5, "set_basic - no gains");
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            hufMetadata->hType = set_basic;
+            return 0;
         }
+        DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
+        hufMetadata->hType = set_compressed;
+        nextHuf->repeatMode = HUF_repeat_check;
+        return hSize;
     }
 }
 
@@ -3215,7 +3226,8 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
  * and updates nextEntropy to the appropriate repeatMode.
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
+ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
+{
     ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0};
     nextEntropy->litlength_repeatMode = FSE_repeat_none;
     nextEntropy->offcode_repeatMode = FSE_repeat_none;
@@ -3227,16 +3239,18 @@ ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
  *  Builds entropy for the sequences.
  *  Stores symbol compression modes and fse table to fseMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE wksp.
- *  @return : size of fse tables or error code */
-static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
-                                              const ZSTD_fseCTables_t* prevEntropy,
-                                                    ZSTD_fseCTables_t* nextEntropy,
-                                              const ZSTD_CCtx_params* cctxParams,
-                                                    ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                    void* workspace, size_t wkspSize)
+ * @return : size of fse tables or error code */
+static size_t
+ZSTD_buildBlockEntropyStats_sequences(
+                const seqStore_t* seqStorePtr,
+                const ZSTD_fseCTables_t* prevEntropy,
+                      ZSTD_fseCTables_t* nextEntropy,
+                const ZSTD_CCtx_params* cctxParams,
+                      ZSTD_fseCTablesMetadata_t* fseMetadata,
+                      void* workspace, size_t wkspSize)
 {
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
-    size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     BYTE* const ostart = fseMetadata->fseTablesBuffer;
     BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
     BYTE* op = ostart;
@@ -3263,18 +3277,20 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
 /** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  Requires workspace size ENTROPY_WORKSPACE_SIZE
- *
- *  @return : 0 on success or error code
+ * @return : 0 on success, or an error code
+ *  Note : also employed in superblock
  */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize)
+size_t ZSTD_buildBlockEntropyStats(
+            const seqStore_t* seqStorePtr,
+            const ZSTD_entropyCTables_t* prevEntropy,
+                  ZSTD_entropyCTables_t* nextEntropy,
+            const ZSTD_CCtx_params* cctxParams,
+                  ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                  void* workspace, size_t wkspSize)
 {
-    size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
-    HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
+    size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+    int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
+    HUF_depth_mode const depthMode = huf_useOptDepth ? HUF_depth_optimal : HUF_depth_fast;
 
     entropyMetadata->hufMetadata.hufDesSize =
         ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
@@ -3295,11 +3311,12 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
 }
 
 /* Returns the size estimate for the literals section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
-                                                const ZSTD_hufCTables_t* huf,
-                                                const ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                void* workspace, size_t wkspSize,
-                                                int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
+                               const ZSTD_hufCTables_t* huf,
+                               const ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               void* workspace, size_t wkspSize,
+                               int writeEntropy)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
@@ -3321,12 +3338,13 @@ static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSiz
 }
 
 /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
-static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
-                        const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
-                        const FSE_CTable* fseCTable,
-                        const U8* additionalBits,
-                        short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
-                        void* workspace, size_t wkspSize)
+static size_t
+ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
+                    const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
+                    const FSE_CTable* fseCTable,
+                    const U8* additionalBits,
+                    short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+                    void* workspace, size_t wkspSize)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     const BYTE* ctp = codeTable;
@@ -3358,99 +3376,107 @@ static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
 }
 
 /* Returns the size estimate for the sequences section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
-                                                  const BYTE* llCodeTable,
-                                                  const BYTE* mlCodeTable,
-                                                  size_t nbSeq,
-                                                  const ZSTD_fseCTables_t* fseTables,
-                                                  const ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                  void* workspace, size_t wkspSize,
-                                                  int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
+                                 const BYTE* llCodeTable,
+                                 const BYTE* mlCodeTable,
+                                 size_t nbSeq,
+                                 const ZSTD_fseCTables_t* fseTables,
+                                 const ZSTD_fseCTablesMetadata_t* fseMetadata,
+                                 void* workspace, size_t wkspSize,
+                                 int writeEntropy)
 {
     size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ);
     size_t cSeqSizeEstimate = 0;
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff,
-                                         fseTables->offcodeCTable, NULL,
-                                         OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
-                                         workspace, wkspSize);
+                                    fseTables->offcodeCTable, NULL,
+                                    OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL,
-                                         fseTables->litlengthCTable, LL_bits,
-                                         LL_defaultNorm, LL_defaultNormLog, MaxLL,
-                                         workspace, wkspSize);
+                                    fseTables->litlengthCTable, LL_bits,
+                                    LL_defaultNorm, LL_defaultNormLog, MaxLL,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML,
-                                         fseTables->matchlengthCTable, ML_bits,
-                                         ML_defaultNorm, ML_defaultNormLog, MaxML,
-                                         workspace, wkspSize);
+                                    fseTables->matchlengthCTable, ML_bits,
+                                    ML_defaultNorm, ML_defaultNormLog, MaxML,
+                                    workspace, wkspSize);
     if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
     return cSeqSizeEstimate + sequencesSectionHeaderSize;
 }
 
 /* Returns the size estimate for a given stream of literals, of, ll, ml */
-static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
-                                     const BYTE* ofCodeTable,
-                                     const BYTE* llCodeTable,
-                                     const BYTE* mlCodeTable,
-                                     size_t nbSeq,
-                                     const ZSTD_entropyCTables_t* entropy,
-                                     const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                     void* workspace, size_t wkspSize,
-                                     int writeLitEntropy, int writeSeqEntropy) {
+static size_t
+ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
+                       const BYTE* ofCodeTable,
+                       const BYTE* llCodeTable,
+                       const BYTE* mlCodeTable,
+                       size_t nbSeq,
+                       const ZSTD_entropyCTables_t* entropy,
+                       const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                       void* workspace, size_t wkspSize,
+                       int writeLitEntropy, int writeSeqEntropy)
+{
     size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize,
-                                                         &entropy->huf, &entropyMetadata->hufMetadata,
-                                                         workspace, wkspSize, writeLitEntropy);
+                                    &entropy->huf, &entropyMetadata->hufMetadata,
+                                    workspace, wkspSize, writeLitEntropy);
     size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
-                                                         nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
-                                                         workspace, wkspSize, writeSeqEntropy);
+                                    nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+                                    workspace, wkspSize, writeSeqEntropy);
     return seqSize + literalsSize + ZSTD_blockHeaderSize;
 }
 
 /* Builds entropy statistics and uses them for blocksize estimation.
  *
- * Returns the estimated compressed size of the seqStore, or a zstd error.
+ * @return: estimated compressed size of the seqStore, or a zstd error.
  */
-static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) {
-    ZSTD_entropyCTablesMetadata_t* entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
+static size_t
+ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc)
+{
+    ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
     DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
     FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
                     &zc->blockState.prevCBlock->entropy,
                     &zc->blockState.nextCBlock->entropy,
                     &zc->appliedParams,
                     entropyMetadata,
-                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
-    return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), "");
+    return ZSTD_estimateBlockSize(
+                    seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
                     seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
                     (size_t)(seqStore->sequences - seqStore->sequencesStart),
-                    &zc->blockState.nextCBlock->entropy, entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
+                    &zc->blockState.nextCBlock->entropy,
+                    entropyMetadata,
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
                     (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
 }
 
 /* Returns literals bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore)
+{
     size_t literalsBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
-        seqDef seq = seqStore->sequencesStart[i];
+        seqDef const seq = seqStore->sequencesStart[i];
         literalsBytes += seq.litLength;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
             literalsBytes += 0x10000;
-        }
-    }
+    }   }
     return literalsBytes;
 }
 
 /* Returns match bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore)
+{
     size_t matchBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
         seqDef seq = seqStore->sequencesStart[i];
         matchBytes += seq.mlBase + MINMATCH;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
             matchBytes += 0x10000;
-        }
-    }
+    }   }
     return matchBytes;
 }
 
@@ -3499,6 +3525,7 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, c
     U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0;  /* [ 0 - 3 ] */
     assert(OFFBASE_IS_REPCODE(offBase));
     if (adjustedRepCode == ZSTD_REP_NUM) {
+        assert(ll0);
         /* litlength == 0 and offCode == 2 implies selection of first repcode - 1
          * This is only valid if it results in a valid offset value, aka > 0.
          * Note : it may happen that `rep[0]==1` in exceptional circumstances.
@@ -3524,14 +3551,16 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, c
  *        1-3 : repcode 1-3
  *        4+ : real_offset+3
  */
-static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
-                                          seqStore_t* const seqStore, U32 const nbSeq) {
+static void
+ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
+                        const seqStore_t* const seqStore, U32 const nbSeq)
+{
     U32 idx = 0;
     for (; idx < nbSeq; ++idx) {
         seqDef* const seq = seqStore->sequencesStart + idx;
         U32 const ll0 = (seq->litLength == 0);
         U32 const offBase = seq->offBase;
-        assert(seq->offBase > 0);
+        assert(offBase > 0);
         if (OFFBASE_IS_REPCODE(offBase)) {
             U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0);
             U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0);
@@ -3540,7 +3569,7 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
              * repcode history.
              */
             if (dRawOffset != cRawOffset) {
-                seq->offBase = cRawOffset + ZSTD_REP_NUM;
+                seq->offBase = OFFSET_TO_OFFBASE(cRawOffset);
             }
         }
         /* Compression repcode history is always updated with values directly from the unmodified seqStore.
@@ -3557,10 +3586,11 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
  * Returns the total size of that block (including header) or a ZSTD error code.
  */
 static size_t
-ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore,
+ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
+                            const seqStore_t* const seqStore,
                                   repcodes_t* const dRep, repcodes_t* const cRep,
                                   void* dst, size_t dstCapacity,
-                                  const void* src, size_t srcSize,
+                            const void* src, size_t srcSize,
                                   U32 lastBlock, U32 isPartition)
 {
     const U32 rleMaxLength = 25;
@@ -3634,10 +3664,11 @@ typedef struct {
 
 /* Helper function to perform the recursive search for block splits.
  * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half.
- * If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then
- * we do not recurse.
+ * If advantageous to split, then we recurse down the two sub-blocks.
+ * If not, or if an error occurred in estimation, then we do not recurse.
  *
- * Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING.
+ * Note: The recursion depth is capped by a heuristic minimum number of sequences,
+ * defined by MIN_SEQUENCES_BLOCK_SPLITTING.
  * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING).
  * In practice, recursion depth usually doesn't go beyond 4.
  *
@@ -3649,16 +3680,17 @@ static void
 ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
                              ZSTD_CCtx* zc, const seqStore_t* origSeqStore)
 {
-    seqStore_t* fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
-    seqStore_t* firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
-    seqStore_t* secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
+    seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
+    seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
+    seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
     size_t estimatedOriginalSize;
     size_t estimatedFirstHalfSize;
     size_t estimatedSecondHalfSize;
     size_t midIdx = (startIdx + endIdx)/2;
 
+    assert(endIdx >= startIdx);
     if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) {
-        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences");
+        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx);
         return;
     }
     DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
@@ -3682,15 +3714,16 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end
     }
 }
 
-/* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio.
+/* Base recursive function.
+ * Populates a table with intra-block partition indices that can improve compression ratio.
  *
- * Returns the number of splits made (which equals the size of the partition table - 1).
+ * @return: number of splits made (which equals the size of the partition table - 1).
  */
 static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
 {
     seqStoreSplits splits = {partitions, 0};
     if (nbSeq <= 4) {
-        DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split");
+        DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq);
         /* Refuse to try and split anything with less than 4 sequences */
         return 0;
     }
@@ -3706,18 +3739,20 @@ static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
  * Returns combined size of all blocks (which includes headers), or a ZSTD error code.
  */
 static size_t
-ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity,
-                                       const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq)
+ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
+                                    void* dst, size_t dstCapacity,
+                              const void* src, size_t blockSize,
+                                    U32 lastBlock, U32 nbSeq)
 {
     size_t cSize = 0;
     const BYTE* ip = (const BYTE*)src;
     BYTE* op = (BYTE*)dst;
     size_t i = 0;
     size_t srcBytesTotal = 0;
-    U32* partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
-    seqStore_t* nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
-    seqStore_t* currSeqStore = &zc->blockSplitCtx.currSeqStore;
-    size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
+    U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
+    seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
+    seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
+    size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
 
     /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
      * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
@@ -3739,19 +3774,21 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
     ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
     ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t));
 
-    DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+    DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
                 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
                 (unsigned)zc->blockState.matchState.nextToUpdate);
 
     if (numSplits == 0) {
-        size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
-                                                                   &dRep, &cRep,
-                                                                    op, dstCapacity,
-                                                                    ip, blockSize,
-                                                                    lastBlock, 0 /* isPartition */);
+        size_t cSizeSingleBlock =
+            ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
+                                            &dRep, &cRep,
+                                            op, dstCapacity,
+                                            ip, blockSize,
+                                            lastBlock, 0 /* isPartition */);
         FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
         DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
-        assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX);
+        assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize);
         return cSizeSingleBlock;
     }
 
@@ -3776,7 +3813,8 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
                                                        op, dstCapacity,
                                                        ip, srcBytes,
                                                        lastBlockEntireSrc, 1 /* isPartition */);
-        DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
+        DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size",
+                    ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
         FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
 
         ip += srcBytes;
@@ -3784,10 +3822,10 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
         dstCapacity -= cSizeChunk;
         cSize += cSizeChunk;
         *currSeqStore = *nextSeqStore;
-        assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize);
     }
-    /* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes
-     * for the next block.
+    /* cRep and dRep may have diverged during the compression.
+     * If so, we use the dRep repcodes for the next block.
      */
     ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
     return cSize;
@@ -3798,8 +3836,6 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
                               const void* src, size_t srcSize, U32 lastBlock)
 {
-    const BYTE* ip = (const BYTE*)src;
-    BYTE* op = (BYTE*)dst;
     U32 nbSeq;
     size_t cSize;
     DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
@@ -3810,7 +3846,7 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
         if (bss == ZSTDbss_noCompress) {
             if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
                 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
-            cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
+            cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
             FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
             DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
             return cSize;
@@ -3922,10 +3958,11 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
          *   * cSize >= blockBound(srcSize): We have expanded the block too much so
          *     emit an uncompressed block.
          */
-        {
-            size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
+        {   size_t const cSize =
+                ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
             if (cSize != ERROR(dstSize_tooSmall)) {
-                size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
+                size_t const maxCSize =
+                    srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
                 FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
                 if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
                     ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
@@ -3933,7 +3970,7 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
                 }
             }
         }
-    }
+    } /* if (bss == ZSTDbss_compress)*/
 
     DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
     /* Superblock compression failed, attempt to emit a single no compress block.
@@ -3991,7 +4028,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
 *   All blocks will be terminated, all input will be consumed.
 *   Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
 *   Frame is supposed already started (header already produced)
-*   @return : compressed size, or an error code
+*  @return : compressed size, or an error code
 */
 static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                                      void* dst, size_t dstCapacity,
@@ -4056,7 +4093,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                     MEM_writeLE24(op, cBlockHeader);
                     cSize += ZSTD_blockHeaderSize;
                 }
-            }
+            }  /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
 
 
             ip += blockSize;
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index baa726f7d..627d441af 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -118,12 +118,13 @@ typedef struct {
 /** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  @return : 0 on success or error code */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize);
+size_t ZSTD_buildBlockEntropyStats(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                          void* workspace, size_t wkspSize);
 
 /*********************************
 *  Compression internals structs *

From 65545969d56d8eaad448732becd36e23a4ccd425 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Dec 2022 05:07:09 +0000
Subject: [PATCH 050/197] Bump ossf/scorecard-action from 2.0.6 to 2.1.0

Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.0.6 to 2.1.0.
- [Release notes](https://github.com/ossf/scorecard-action/releases)
- [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md)
- [Commits](https://github.com/ossf/scorecard-action/compare/99c53751e09b9529366343771cc321ec74e9bd3d...937ffa90d79c7d720498178154ad4c7ba1e4ad8c)

---
updated-dependencies:
- dependency-name: ossf/scorecard-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/scorecards.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 1c19b48b2..d020c8781 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -32,7 +32,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6
+        uses: ossf/scorecard-action@937ffa90d79c7d720498178154ad4c7ba1e4ad8c # tag=v2.1.0
         with:
           results_file: results.sarif
           results_format: sarif

From 79a00f8dcfb7a1e528f026b5e08a071a2b965574 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Dec 2022 05:07:15 +0000
Subject: [PATCH 051/197] Bump github/codeql-action from 1.0.26 to 2.1.37

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 1.0.26 to 2.1.37.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/5f532563584d71fdef14ee64d17bafb34f751ce5...959cbb7472c4d4ad70cdfe6f4976053fe48ab394)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/scorecards.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 1c19b48b2..7ed09ef7a 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -59,6 +59,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@5f532563584d71fdef14ee64d17bafb34f751ce5 # tag=v1.0.26
+        uses: github/codeql-action/upload-sarif@959cbb7472c4d4ad70cdfe6f4976053fe48ab394 # tag=v2.1.37
         with:
           sarif_file: results.sarif

From 3cee69a1cc536a5ff88fd81288e6eab3b2140049 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Sat, 17 Dec 2022 21:09:48 -0500
Subject: [PATCH 052/197] CI: build programs on meson-windows too

---
 .github/workflows/dev-short-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index e1d4cc40e..459eb362f 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -212,7 +212,7 @@ jobs:
         uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d
       - name: Configure with Meson
         run: |
-          meson setup build/meson/ builddir/ -Dbin_tests=true
+          meson setup build/meson/ builddir -Dbin_tests=true -Dbin_programs=true
       - name: Build with Meson
         run: |
           ninja -C builddir/

From 67cd24b25b997f6e9f3e1aac7e38579b932d9d47 Mon Sep 17 00:00:00 2001
From: Eli Schwartz <eschwartz@archlinux.org>
Date: Sat, 17 Dec 2022 21:24:31 -0500
Subject: [PATCH 053/197] CI: build contrib directory on meson-windows

---
 .github/workflows/dev-short-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 459eb362f..d0d0940fa 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -212,7 +212,7 @@ jobs:
         uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d
       - name: Configure with Meson
         run: |
-          meson setup build/meson/ builddir -Dbin_tests=true -Dbin_programs=true
+          meson setup build/meson/ builddir -Dbin_tests=true -Dbin_programs=true -Dbin_contrib=true
       - name: Build with Meson
         run: |
           ninja -C builddir/

From e9797b5dc5c4a703fd13cf2e86d278b275825bb3 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 19 Dec 2022 12:23:29 -0800
Subject: [PATCH 054/197] [pzstd] Fixes for Windows build

* Add `Portability.h` to fix min/max issues.
* Fix conversion warnings
* Assert that windowLog <= 23, which is currently always the case.
  This could be loosened, but we aren't looking to add new functionality.

Fixes on top of PR #3375 by @eli-schwartz, which added Windows CI for contrib & programs.
---
 contrib/pzstd/Pzstd.cpp           |  9 ++++++++-
 contrib/pzstd/utils/FileSystem.h  |  6 ++++--
 contrib/pzstd/utils/Portability.h | 16 ++++++++++++++++
 contrib/pzstd/utils/Range.h       |  2 ++
 4 files changed, 30 insertions(+), 3 deletions(-)
 create mode 100644 contrib/pzstd/utils/Portability.h

diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 2c09bda7a..788cd8fbe 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -10,11 +10,13 @@
 #include "Pzstd.h"
 #include "SkippableFrame.h"
 #include "utils/FileSystem.h"
+#include "utils/Portability.h"
 #include "utils/Range.h"
 #include "utils/ScopeGuard.h"
 #include "utils/ThreadPool.h"
 #include "utils/WorkQueue.h"
 
+#include <algorithm>
 #include <chrono>
 #include <cinttypes>
 #include <cstddef>
@@ -336,6 +338,10 @@ static size_t calculateStep(
     const ZSTD_parameters &params) {
   (void)size;
   (void)numThreads;
+  // Not validated to work correctly for window logs > 23.
+  // It will definitely fail if windowLog + 2 is >= 4GB because
+  // the skippable frame can only store sizes up to 4GB.
+  assert(params.cParams.windowLog <= 23);
   return size_t{1} << (params.cParams.windowLog + 2);
 }
 
@@ -587,7 +593,8 @@ std::uint64_t writeFile(
       // start writing before compression is done because we need to know the
       // compressed size.
       // Wait for the compressed size to be available and write skippable frame
-      SkippableFrame frame(out->size());
+      assert(uint64_t(out->size()) < uint64_t(1) << 32);
+      SkippableFrame frame(uint32_t(out->size()));
       if (!writeData(frame.data(), outputFd)) {
         errorHolder.setError("Failed to write output");
         return bytesWritten;
diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h
index 8898b0357..05c64b879 100644
--- a/contrib/pzstd/utils/FileSystem.h
+++ b/contrib/pzstd/utils/FileSystem.h
@@ -8,11 +8,13 @@
  */
 #pragma once
 
+#include "utils/Portability.h"
 #include "utils/Range.h"
 
 #include <sys/stat.h>
 #include <cerrno>
 #include <cstdint>
+#include <limits>
 #include <system_error>
 
 // A small subset of `std::filesystem`.
@@ -82,11 +84,11 @@ inline std::uintmax_t file_size(
     std::error_code& ec) noexcept {
   auto stat = status(path, ec);
   if (ec) {
-    return -1;
+    return std::numeric_limits<uintmax_t>::max();
   }
   if (!is_regular_file(stat)) {
     ec.assign(ENOTSUP, std::generic_category());
-    return -1;
+    return std::numeric_limits<uintmax_t>::max();
   }
   ec.clear();
   return stat.st_size;
diff --git a/contrib/pzstd/utils/Portability.h b/contrib/pzstd/utils/Portability.h
new file mode 100644
index 000000000..27dbcf707
--- /dev/null
+++ b/contrib/pzstd/utils/Portability.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+#pragma once
+
+#include <algorithm>
+
+// Required for windows, which defines min/max, but we want the std:: version.
+#undef min
+#undef max
diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h
index 6a850ad4e..c78a8935a 100644
--- a/contrib/pzstd/utils/Range.h
+++ b/contrib/pzstd/utils/Range.h
@@ -14,7 +14,9 @@
 #pragma once
 
 #include "utils/Likely.h"
+#include "utils/Portability.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <cstring>
 #include <stdexcept>

From 26f1bf7d70f8cb484e8f68c0410818767cfc4f8c Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <yoniko@gmail.com>
Date: Mon, 19 Dec 2022 13:22:34 -0800
Subject: [PATCH 055/197] CR fixes

---
 lib/common/threading.c | 45 +++++++++++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/lib/common/threading.c b/lib/common/threading.c
index 825826500..f2341105a 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -44,15 +44,23 @@ typedef struct {
 
 static unsigned __stdcall worker(void *arg)
 {
-    ZSTD_thread_params_t* const thread_param = (ZSTD_thread_params_t*)arg;
-    void* (*start_routine)(void*) = thread_param->start_routine;
-    void* thread_arg = thread_param->arg;
+    void* (*start_routine)(void*);
+    void* thread_arg;
 
-    /* Signal main thread that we are running and do not depend on its memory anymore */
-    ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex);
-    thread_param->initialized = 1;
-    ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex);
-    ZSTD_pthread_cond_signal(&thread_param->initialized_cond);
+    /* Inialized thread_arg and start_routine and signal main thread that we don't need it
+     * to wait any longer.
+     */
+    {
+        ZSTD_thread_params_t*  thread_param = (ZSTD_thread_params_t*)arg;
+        thread_arg = thread_param->arg;
+        start_routine = thread_param->start_routine;
+
+        /* Signal main thread that we are running and do not depend on its memory anymore */
+        ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex);
+        thread_param->initialized = 1;
+        ZSTD_pthread_cond_signal(&thread_param->initialized_cond);
+        ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex);
+    }
 
     start_routine(thread_arg);
 
@@ -63,25 +71,34 @@ int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
             void* (*start_routine) (void*), void* arg)
 {
     ZSTD_thread_params_t thread_param;
-    int error = 0;
     (void)unused;
+
     thread_param.start_routine = start_routine;
     thread_param.arg = arg;
     thread_param.initialized = 0;
+    *thread = NULL;
 
     /* Setup thread initialization synchronization */
-    error |= ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL);
-    error |= ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL);
-    if(error)
+    if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) {
+        /* Should never happen on Windows */
         return -1;
-    ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex);
+    }
+    if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) {
+        /* Should never happen on Windows */
+        ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
+        return -1;
+    }
 
     /* Spawn thread */
     *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL);
-    if (!thread)
+    if (!thread) {
+        ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
+        ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
         return errno;
+    }
 
     /* Wait for thread to be initialized */
+    ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex);
     while(!thread_param.initialized) {
         ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex);
     }

From a05cca62082ba0c655a0dad3f7994253c011d483 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 21 Jun 2022 11:54:13 -0400
Subject: [PATCH 056/197] Manually Update VS Code Copyright Definitions

---
 build/VS2010/libzstd-dll/libzstd-dll.rc | 4 ++--
 build/VS2010/zstd/zstd.rc               | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build/VS2010/libzstd-dll/libzstd-dll.rc b/build/VS2010/libzstd-dll/libzstd-dll.rc
index ee9f56280..05c8fbd41 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.rc
+++ b/build/VS2010/libzstd-dll/libzstd-dll.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "CompanyName", "Meta Platforms, Inc."
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "libzstd.dll"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+            VALUE "LegalCopyright", "Copyright (c) 2013-present, Meta Platforms, Inc."
             VALUE "OriginalFilename", "libzstd.dll"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/build/VS2010/zstd/zstd.rc b/build/VS2010/zstd/zstd.rc
index f5e404730..c1df78a13 100644
--- a/build/VS2010/zstd/zstd.rc
+++ b/build/VS2010/zstd/zstd.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "CompanyName", "Meta Platforms, Inc."
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+            VALUE "LegalCopyright", "Copyright (c) 2013-present, Meta Platforms, Inc."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING

From 8927f985ffed45101f9c6c95ac26cd1f6018ecb7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 20 Dec 2022 12:37:57 -0500
Subject: [PATCH 057/197] Update Copyright Headers 'Facebook' -> 'Meta
 Platforms'

```
for f in $(find . \( -path ./.git -o -path ./tests/fuzz/corpora \) -prune -o -type f);
do
  sed -i 's/Facebook, Inc\./Meta Platforms, Inc. and affiliates./' $f;
done
```
---
 LICENSE                                                       | 2 +-
 Makefile                                                      | 2 +-
 build/cmake/CMakeLists.txt                                    | 2 +-
 build/cmake/contrib/CMakeLists.txt                            | 2 +-
 build/cmake/contrib/gen_html/CMakeLists.txt                   | 2 +-
 build/cmake/contrib/pzstd/CMakeLists.txt                      | 2 +-
 build/cmake/lib/CMakeLists.txt                                | 2 +-
 build/cmake/programs/CMakeLists.txt                           | 2 +-
 build/single_file_libs/zstd-in.c                              | 2 +-
 build/single_file_libs/zstddeclib-in.c                        | 2 +-
 contrib/diagnose_corruption/Makefile                          | 2 +-
 contrib/diagnose_corruption/check_flipped_bits.c              | 2 +-
 contrib/freestanding_lib/freestanding.py                      | 2 +-
 contrib/gen_html/Makefile                                     | 2 +-
 contrib/gen_html/gen_html.cpp                                 | 2 +-
 contrib/largeNbDicts/Makefile                                 | 2 +-
 contrib/largeNbDicts/largeNbDicts.c                           | 2 +-
 contrib/linux-kernel/Makefile                                 | 2 +-
 contrib/linux-kernel/decompress_sources.h                     | 2 +-
 contrib/linux-kernel/linux.mk                                 | 2 +-
 contrib/linux-kernel/linux_zstd.h                             | 2 +-
 contrib/linux-kernel/mem.h                                    | 2 +-
 contrib/linux-kernel/test/Makefile                            | 2 +-
 contrib/linux-kernel/test/include/linux/compiler.h            | 2 +-
 contrib/linux-kernel/test/include/linux/errno.h               | 2 +-
 contrib/linux-kernel/test/include/linux/kernel.h              | 2 +-
 contrib/linux-kernel/test/include/linux/limits.h              | 2 +-
 contrib/linux-kernel/test/include/linux/math64.h              | 2 +-
 contrib/linux-kernel/test/include/linux/module.h              | 2 +-
 contrib/linux-kernel/test/include/linux/printk.h              | 2 +-
 contrib/linux-kernel/test/include/linux/stddef.h              | 2 +-
 contrib/linux-kernel/test/include/linux/swab.h                | 2 +-
 contrib/linux-kernel/test/include/linux/types.h               | 2 +-
 contrib/linux-kernel/test/static_test.c                       | 2 +-
 contrib/linux-kernel/test/test.c                              | 2 +-
 contrib/linux-kernel/zstd_common_module.c                     | 2 +-
 contrib/linux-kernel/zstd_compress_module.c                   | 2 +-
 contrib/linux-kernel/zstd_decompress_module.c                 | 2 +-
 contrib/linux-kernel/zstd_deps.h                              | 2 +-
 contrib/match_finders/zstd_edist.c                            | 2 +-
 contrib/match_finders/zstd_edist.h                            | 2 +-
 contrib/pzstd/ErrorHolder.h                                   | 2 +-
 contrib/pzstd/Logging.h                                       | 2 +-
 contrib/pzstd/Makefile                                        | 2 +-
 contrib/pzstd/Options.cpp                                     | 2 +-
 contrib/pzstd/Options.h                                       | 2 +-
 contrib/pzstd/Pzstd.cpp                                       | 2 +-
 contrib/pzstd/Pzstd.h                                         | 2 +-
 contrib/pzstd/SkippableFrame.cpp                              | 2 +-
 contrib/pzstd/SkippableFrame.h                                | 2 +-
 contrib/pzstd/main.cpp                                        | 2 +-
 contrib/pzstd/test/OptionsTest.cpp                            | 2 +-
 contrib/pzstd/test/PzstdTest.cpp                              | 2 +-
 contrib/pzstd/test/RoundTrip.h                                | 2 +-
 contrib/pzstd/test/RoundTripTest.cpp                          | 2 +-
 contrib/pzstd/utils/Buffer.h                                  | 2 +-
 contrib/pzstd/utils/FileSystem.h                              | 2 +-
 contrib/pzstd/utils/Likely.h                                  | 2 +-
 contrib/pzstd/utils/Portability.h                             | 2 +-
 contrib/pzstd/utils/Range.h                                   | 2 +-
 contrib/pzstd/utils/ResourcePool.h                            | 2 +-
 contrib/pzstd/utils/ScopeGuard.h                              | 2 +-
 contrib/pzstd/utils/ThreadPool.h                              | 2 +-
 contrib/pzstd/utils/WorkQueue.h                               | 2 +-
 contrib/pzstd/utils/test/BufferTest.cpp                       | 2 +-
 contrib/pzstd/utils/test/RangeTest.cpp                        | 2 +-
 contrib/pzstd/utils/test/ResourcePoolTest.cpp                 | 2 +-
 contrib/pzstd/utils/test/ScopeGuardTest.cpp                   | 2 +-
 contrib/pzstd/utils/test/ThreadPoolTest.cpp                   | 2 +-
 contrib/pzstd/utils/test/WorkQueueTest.cpp                    | 2 +-
 contrib/recovery/Makefile                                     | 2 +-
 contrib/recovery/recover_directory.c                          | 2 +-
 contrib/seekable_format/examples/Makefile                     | 2 +-
 contrib/seekable_format/examples/parallel_compression.c       | 2 +-
 contrib/seekable_format/examples/parallel_processing.c        | 2 +-
 contrib/seekable_format/examples/seekable_compression.c       | 2 +-
 contrib/seekable_format/examples/seekable_decompression.c     | 2 +-
 contrib/seekable_format/examples/seekable_decompression_mem.c | 2 +-
 contrib/seekable_format/tests/Makefile                        | 2 +-
 contrib/seekable_format/zstd_seekable_compression_format.md   | 2 +-
 contrib/seekable_format/zstdseek_compress.c                   | 2 +-
 contrib/seekable_format/zstdseek_decompress.c                 | 2 +-
 contrib/seqBench/Makefile                                     | 2 +-
 doc/educational_decoder/Makefile                              | 2 +-
 doc/educational_decoder/harness.c                             | 2 +-
 doc/educational_decoder/zstd_decompress.c                     | 2 +-
 doc/educational_decoder/zstd_decompress.h                     | 2 +-
 doc/zstd_compression_format.md                                | 2 +-
 examples/Makefile                                             | 2 +-
 examples/common.h                                             | 2 +-
 examples/dictionary_compression.c                             | 2 +-
 examples/dictionary_decompression.c                           | 2 +-
 examples/multiple_simple_compression.c                        | 2 +-
 examples/multiple_streaming_compression.c                     | 2 +-
 examples/simple_compression.c                                 | 2 +-
 examples/simple_decompression.c                               | 2 +-
 examples/streaming_compression.c                              | 2 +-
 examples/streaming_compression_thread_pool.c                  | 2 +-
 examples/streaming_decompression.c                            | 2 +-
 examples/streaming_memory_usage.c                             | 2 +-
 lib/Makefile                                                  | 2 +-
 lib/common/bits.h                                             | 2 +-
 lib/common/bitstream.h                                        | 2 +-
 lib/common/compiler.h                                         | 2 +-
 lib/common/cpu.h                                              | 2 +-
 lib/common/debug.c                                            | 2 +-
 lib/common/debug.h                                            | 2 +-
 lib/common/entropy_common.c                                   | 2 +-
 lib/common/error_private.c                                    | 2 +-
 lib/common/error_private.h                                    | 2 +-
 lib/common/fse.h                                              | 2 +-
 lib/common/fse_decompress.c                                   | 2 +-
 lib/common/huf.h                                              | 2 +-
 lib/common/mem.h                                              | 2 +-
 lib/common/pool.c                                             | 2 +-
 lib/common/pool.h                                             | 2 +-
 lib/common/portability_macros.h                               | 2 +-
 lib/common/xxhash.c                                           | 2 +-
 lib/common/xxhash.h                                           | 2 +-
 lib/common/zstd_common.c                                      | 2 +-
 lib/common/zstd_deps.h                                        | 2 +-
 lib/common/zstd_internal.h                                    | 2 +-
 lib/common/zstd_trace.h                                       | 2 +-
 lib/compress/clevels.h                                        | 2 +-
 lib/compress/fse_compress.c                                   | 2 +-
 lib/compress/hist.c                                           | 2 +-
 lib/compress/hist.h                                           | 2 +-
 lib/compress/huf_compress.c                                   | 2 +-
 lib/compress/zstd_compress.c                                  | 2 +-
 lib/compress/zstd_compress_internal.h                         | 2 +-
 lib/compress/zstd_compress_literals.c                         | 2 +-
 lib/compress/zstd_compress_literals.h                         | 2 +-
 lib/compress/zstd_compress_sequences.c                        | 2 +-
 lib/compress/zstd_compress_sequences.h                        | 2 +-
 lib/compress/zstd_compress_superblock.c                       | 2 +-
 lib/compress/zstd_compress_superblock.h                       | 2 +-
 lib/compress/zstd_cwksp.h                                     | 2 +-
 lib/compress/zstd_double_fast.c                               | 2 +-
 lib/compress/zstd_double_fast.h                               | 2 +-
 lib/compress/zstd_fast.c                                      | 2 +-
 lib/compress/zstd_fast.h                                      | 2 +-
 lib/compress/zstd_lazy.c                                      | 2 +-
 lib/compress/zstd_lazy.h                                      | 2 +-
 lib/compress/zstd_ldm.c                                       | 2 +-
 lib/compress/zstd_ldm.h                                       | 2 +-
 lib/compress/zstd_ldm_geartab.h                               | 2 +-
 lib/compress/zstd_opt.c                                       | 2 +-
 lib/compress/zstd_opt.h                                       | 2 +-
 lib/compress/zstdmt_compress.c                                | 2 +-
 lib/compress/zstdmt_compress.h                                | 2 +-
 lib/decompress/huf_decompress.c                               | 2 +-
 lib/decompress/huf_decompress_amd64.S                         | 2 +-
 lib/decompress/zstd_ddict.c                                   | 2 +-
 lib/decompress/zstd_ddict.h                                   | 2 +-
 lib/decompress/zstd_decompress.c                              | 2 +-
 lib/decompress/zstd_decompress_block.c                        | 2 +-
 lib/decompress/zstd_decompress_block.h                        | 2 +-
 lib/decompress/zstd_decompress_internal.h                     | 2 +-
 lib/deprecated/zbuff.h                                        | 2 +-
 lib/deprecated/zbuff_common.c                                 | 2 +-
 lib/deprecated/zbuff_compress.c                               | 2 +-
 lib/deprecated/zbuff_decompress.c                             | 2 +-
 lib/dictBuilder/cover.c                                       | 2 +-
 lib/dictBuilder/cover.h                                       | 2 +-
 lib/dictBuilder/fastcover.c                                   | 2 +-
 lib/dictBuilder/zdict.c                                       | 2 +-
 lib/dll/example/Makefile                                      | 2 +-
 lib/legacy/zstd_legacy.h                                      | 2 +-
 lib/legacy/zstd_v01.c                                         | 2 +-
 lib/legacy/zstd_v01.h                                         | 2 +-
 lib/legacy/zstd_v02.c                                         | 2 +-
 lib/legacy/zstd_v02.h                                         | 2 +-
 lib/legacy/zstd_v03.c                                         | 2 +-
 lib/legacy/zstd_v03.h                                         | 2 +-
 lib/legacy/zstd_v04.c                                         | 2 +-
 lib/legacy/zstd_v04.h                                         | 2 +-
 lib/legacy/zstd_v05.c                                         | 2 +-
 lib/legacy/zstd_v05.h                                         | 2 +-
 lib/legacy/zstd_v06.c                                         | 2 +-
 lib/legacy/zstd_v06.h                                         | 2 +-
 lib/legacy/zstd_v07.c                                         | 2 +-
 lib/legacy/zstd_v07.h                                         | 2 +-
 lib/libzstd.mk                                                | 2 +-
 lib/zdict.h                                                   | 2 +-
 lib/zstd.h                                                    | 2 +-
 lib/zstd_errors.h                                             | 2 +-
 programs/Makefile                                             | 2 +-
 programs/benchfn.c                                            | 2 +-
 programs/benchfn.h                                            | 2 +-
 programs/benchzstd.c                                          | 2 +-
 programs/benchzstd.h                                          | 2 +-
 programs/datagen.c                                            | 2 +-
 programs/datagen.h                                            | 2 +-
 programs/dibio.c                                              | 2 +-
 programs/dibio.h                                              | 2 +-
 programs/fileio.c                                             | 2 +-
 programs/fileio.h                                             | 2 +-
 programs/fileio_asyncio.c                                     | 2 +-
 programs/fileio_asyncio.h                                     | 2 +-
 programs/fileio_common.h                                      | 2 +-
 programs/fileio_types.h                                       | 2 +-
 programs/platform.h                                           | 2 +-
 programs/timefn.c                                             | 2 +-
 programs/timefn.h                                             | 2 +-
 programs/util.c                                               | 2 +-
 programs/util.h                                               | 2 +-
 programs/windres/verrsrc.h                                    | 2 +-
 programs/windres/zstd.rc                                      | 4 ++--
 programs/zstdcli.c                                            | 2 +-
 programs/zstdcli_trace.c                                      | 2 +-
 programs/zstdcli_trace.h                                      | 2 +-
 tests/DEPRECATED-test-zstd-speed.py                           | 2 +-
 tests/Makefile                                                | 2 +-
 tests/automated_benchmarking.py                               | 2 +-
 tests/bigdict.c                                               | 2 +-
 tests/checkTag.c                                              | 2 +-
 tests/check_size.py                                           | 2 +-
 tests/cli-tests/run.py                                        | 2 +-
 tests/datagencli.c                                            | 2 +-
 tests/decodecorpus.c                                          | 2 +-
 tests/fullbench.c                                             | 2 +-
 tests/fuzz/Makefile                                           | 2 +-
 tests/fuzz/block_decompress.c                                 | 2 +-
 tests/fuzz/block_round_trip.c                                 | 2 +-
 tests/fuzz/decompress_dstSize_tooSmall.c                      | 2 +-
 tests/fuzz/dictionary_decompress.c                            | 2 +-
 tests/fuzz/dictionary_loader.c                                | 2 +-
 tests/fuzz/dictionary_round_trip.c                            | 2 +-
 tests/fuzz/dictionary_stream_round_trip.c                     | 2 +-
 tests/fuzz/fse_read_ncount.c                                  | 2 +-
 tests/fuzz/fuzz.h                                             | 2 +-
 tests/fuzz/fuzz.py                                            | 2 +-
 tests/fuzz/fuzz_data_producer.c                               | 2 +-
 tests/fuzz/fuzz_data_producer.h                               | 2 +-
 tests/fuzz/fuzz_helpers.c                                     | 2 +-
 tests/fuzz/fuzz_helpers.h                                     | 2 +-
 tests/fuzz/huf_decompress.c                                   | 2 +-
 tests/fuzz/huf_round_trip.c                                   | 2 +-
 tests/fuzz/raw_dictionary_round_trip.c                        | 2 +-
 tests/fuzz/regression_driver.c                                | 2 +-
 tests/fuzz/seekable_roundtrip.c                               | 2 +-
 tests/fuzz/sequence_compression_api.c                         | 2 +-
 tests/fuzz/simple_compress.c                                  | 2 +-
 tests/fuzz/simple_decompress.c                                | 2 +-
 tests/fuzz/simple_round_trip.c                                | 2 +-
 tests/fuzz/stream_decompress.c                                | 2 +-
 tests/fuzz/stream_round_trip.c                                | 2 +-
 tests/fuzz/zstd_frame_info.c                                  | 2 +-
 tests/fuzz/zstd_helpers.c                                     | 2 +-
 tests/fuzz/zstd_helpers.h                                     | 2 +-
 tests/fuzzer.c                                                | 2 +-
 tests/gzip/Makefile                                           | 2 +-
 tests/invalidDictionaries.c                                   | 2 +-
 tests/legacy.c                                                | 2 +-
 tests/longmatch.c                                             | 2 +-
 tests/paramgrill.c                                            | 2 +-
 tests/poolTests.c                                             | 2 +-
 tests/rateLimiter.py                                          | 2 +-
 tests/regression/Makefile                                     | 2 +-
 tests/regression/config.c                                     | 2 +-
 tests/regression/config.h                                     | 2 +-
 tests/regression/data.c                                       | 2 +-
 tests/regression/data.h                                       | 2 +-
 tests/regression/levels.h                                     | 2 +-
 tests/regression/method.c                                     | 2 +-
 tests/regression/method.h                                     | 2 +-
 tests/regression/result.c                                     | 2 +-
 tests/regression/result.h                                     | 2 +-
 tests/regression/test.c                                       | 2 +-
 tests/roundTripCrash.c                                        | 2 +-
 tests/seqgen.c                                                | 2 +-
 tests/seqgen.h                                                | 2 +-
 tests/test-license.py                                         | 2 +-
 tests/test-zstd-versions.py                                   | 2 +-
 tests/zstreamtest.c                                           | 2 +-
 zlibWrapper/examples/zwrapbench.c                             | 2 +-
 zlibWrapper/gzcompatibility.h                                 | 2 +-
 zlibWrapper/zstd_zlibwrapper.c                                | 2 +-
 zlibWrapper/zstd_zlibwrapper.h                                | 2 +-
 279 files changed, 280 insertions(+), 280 deletions(-)

diff --git a/LICENSE b/LICENSE
index a793a8028..493d0db22 100644
--- a/LICENSE
+++ b/LICENSE
@@ -2,7 +2,7 @@ BSD License
 
 For Zstandard software
 
-Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
diff --git a/Makefile b/Makefile
index 429c90ff1..dcd872f5c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2021, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index f1eac9ada..193f60d77 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt
index f7631d08c..3f253cf22 100644
--- a/build/cmake/contrib/CMakeLists.txt
+++ b/build/cmake/contrib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt
index 8fdd61131..96975421b 100644
--- a/build/cmake/contrib/gen_html/CMakeLists.txt
+++ b/build/cmake/contrib/gen_html/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt
index 27af86c88..e6f99c7d2 100644
--- a/build/cmake/contrib/pzstd/CMakeLists.txt
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index cf7af0f8c..9f1e7db49 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index 28b1e1d16..650b8c571 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c
index 59c6b5f19..89296179d 100644
--- a/build/single_file_libs/zstd-in.c
+++ b/build/single_file_libs/zstd-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/build/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c
index 5a58589c9..9e7d36ec9 100644
--- a/build/single_file_libs/zstddeclib-in.c
+++ b/build/single_file_libs/zstddeclib-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/Makefile b/contrib/diagnose_corruption/Makefile
index a21a00212..ff17dd894 100644
--- a/contrib/diagnose_corruption/Makefile
+++ b/contrib/diagnose_corruption/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Facebook, Inc.
+# Copyright (c) 2019-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/check_flipped_bits.c b/contrib/diagnose_corruption/check_flipped_bits.c
index cc40ab84b..342d9baba 100644
--- a/contrib/diagnose_corruption/check_flipped_bits.c
+++ b/contrib/diagnose_corruption/check_flipped_bits.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-present, Facebook, Inc.
+ * Copyright (c) 2019-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index 4e0a944f1..12eb52ab0 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) 2021-2021, Facebook, Inc.
+# Copyright (c) 2021-2021, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/Makefile b/contrib/gen_html/Makefile
index 425f266c4..ebb13fa0e 100644
--- a/contrib/gen_html/Makefile
+++ b/contrib/gen_html/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp
index 90d5b21a3..9ee807c22 100644
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc.
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile
index 4c055b0ed..611ec8e63 100644
--- a/contrib/largeNbDicts/Makefile
+++ b/contrib/largeNbDicts/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2018-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index cc489000f..ad0522918 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2018-present, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index c467bf927..61a52ce20 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/decompress_sources.h b/contrib/linux-kernel/decompress_sources.h
index a06ca187a..8a47eb2a4 100644
--- a/contrib/linux-kernel/decompress_sources.h
+++ b/contrib/linux-kernel/decompress_sources.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/linux.mk b/contrib/linux-kernel/linux.mk
index 20f08c644..464c410b2 100644
--- a/contrib/linux-kernel/linux.mk
+++ b/contrib/linux-kernel/linux.mk
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/linux_zstd.h b/contrib/linux-kernel/linux_zstd.h
index 113408eef..716079adf 100644
--- a/contrib/linux-kernel/linux_zstd.h
+++ b/contrib/linux-kernel/linux_zstd.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/mem.h b/contrib/linux-kernel/mem.h
index 1d9cc0392..8dfd20684 100644
--- a/contrib/linux-kernel/mem.h
+++ b/contrib/linux-kernel/mem.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile
index 53b0c2a65..67b55e665 100644
--- a/contrib/linux-kernel/test/Makefile
+++ b/contrib/linux-kernel/test/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h
index de43edb69..ed3062d8f 100644
--- a/contrib/linux-kernel/test/include/linux/compiler.h
+++ b/contrib/linux-kernel/test/include/linux/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h
index b24752257..4115f6091 100644
--- a/contrib/linux-kernel/test/include/linux/errno.h
+++ b/contrib/linux-kernel/test/include/linux/errno.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h
index 1f702abac..19d5f26cc 100644
--- a/contrib/linux-kernel/test/include/linux/kernel.h
+++ b/contrib/linux-kernel/test/include/linux/kernel.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h
index db9c09904..9dbcc42a5 100644
--- a/contrib/linux-kernel/test/include/linux/limits.h
+++ b/contrib/linux-kernel/test/include/linux/limits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h
index 8eefa2d5c..60b05bd68 100644
--- a/contrib/linux-kernel/test/include/linux/math64.h
+++ b/contrib/linux-kernel/test/include/linux/math64.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h
index 63a28d57b..0e8c5563d 100644
--- a/contrib/linux-kernel/test/include/linux/module.h
+++ b/contrib/linux-kernel/test/include/linux/module.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h
index eab08e0c4..7661c0b37 100644
--- a/contrib/linux-kernel/test/include/linux/printk.h
+++ b/contrib/linux-kernel/test/include/linux/printk.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h
index 8538eb3e4..689edb0e7 100644
--- a/contrib/linux-kernel/test/include/linux/stddef.h
+++ b/contrib/linux-kernel/test/include/linux/stddef.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h
index 783046b42..36c25550c 100644
--- a/contrib/linux-kernel/test/include/linux/swab.h
+++ b/contrib/linux-kernel/test/include/linux/swab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h
index 459a45700..1c194a8ac 100644
--- a/contrib/linux-kernel/test/include/linux/types.h
+++ b/contrib/linux-kernel/test/include/linux/types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/static_test.c b/contrib/linux-kernel/test/static_test.c
index d2b8b5a32..ba4a420d4 100644
--- a/contrib/linux-kernel/test/static_test.c
+++ b/contrib/linux-kernel/test/static_test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c
index 67d248e0c..0f4ba3f45 100644
--- a/contrib/linux-kernel/test/test.c
+++ b/contrib/linux-kernel/test/test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_common_module.c b/contrib/linux-kernel/zstd_common_module.c
index 22686e367..2fead39eb 100644
--- a/contrib/linux-kernel/zstd_common_module.c
+++ b/contrib/linux-kernel/zstd_common_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_compress_module.c b/contrib/linux-kernel/zstd_compress_module.c
index 04e1b5c01..8ecf43226 100644
--- a/contrib/linux-kernel/zstd_compress_module.c
+++ b/contrib/linux-kernel/zstd_compress_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_decompress_module.c b/contrib/linux-kernel/zstd_decompress_module.c
index f4ed952ed..eb1c49e69 100644
--- a/contrib/linux-kernel/zstd_decompress_module.c
+++ b/contrib/linux-kernel/zstd_decompress_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_deps.h b/contrib/linux-kernel/zstd_deps.h
index 7a5bf4483..925161416 100644
--- a/contrib/linux-kernel/zstd_deps.h
+++ b/contrib/linux-kernel/zstd_deps.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.c b/contrib/match_finders/zstd_edist.c
index d63a7cf8d..73a7f6d77 100644
--- a/contrib/match_finders/zstd_edist.c
+++ b/contrib/match_finders/zstd_edist.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.h b/contrib/match_finders/zstd_edist.h
index a947649b2..b60a5dc05 100644
--- a/contrib/match_finders/zstd_edist.h
+++ b/contrib/match_finders/zstd_edist.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/ErrorHolder.h b/contrib/pzstd/ErrorHolder.h
index 829651c59..d144e0fcc 100644
--- a/contrib/pzstd/ErrorHolder.h
+++ b/contrib/pzstd/ErrorHolder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h
index beb160b64..652eae824 100644
--- a/contrib/pzstd/Logging.h
+++ b/contrib/pzstd/Logging.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index 3d930cca9..8604bac0c 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index 90f9d571f..c278b1977 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h
index 924543abf..5bf6a6492 100644
--- a/contrib/pzstd/Options.h
+++ b/contrib/pzstd/Options.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 788cd8fbe..794d2c180 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h
index c667c887d..46c74cc3f 100644
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.cpp b/contrib/pzstd/SkippableFrame.cpp
index 769866dfc..058667a06 100644
--- a/contrib/pzstd/SkippableFrame.cpp
+++ b/contrib/pzstd/SkippableFrame.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.h b/contrib/pzstd/SkippableFrame.h
index 60deed040..389f5304f 100644
--- a/contrib/pzstd/SkippableFrame.h
+++ b/contrib/pzstd/SkippableFrame.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp
index b93f043b1..dea12fac7 100644
--- a/contrib/pzstd/main.cpp
+++ b/contrib/pzstd/main.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp
index e60114825..dd9a0113b 100644
--- a/contrib/pzstd/test/OptionsTest.cpp
+++ b/contrib/pzstd/test/OptionsTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp
index 5c7d66310..60c03e8c3 100644
--- a/contrib/pzstd/test/PzstdTest.cpp
+++ b/contrib/pzstd/test/PzstdTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h
index c6364ecb4..4990501ae 100644
--- a/contrib/pzstd/test/RoundTrip.h
+++ b/contrib/pzstd/test/RoundTrip.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp
index 36af0673a..af0e8149e 100644
--- a/contrib/pzstd/test/RoundTripTest.cpp
+++ b/contrib/pzstd/test/RoundTripTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Buffer.h b/contrib/pzstd/utils/Buffer.h
index d17ad2f2c..5a640d60b 100644
--- a/contrib/pzstd/utils/Buffer.h
+++ b/contrib/pzstd/utils/Buffer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h
index 05c64b879..2615c4e00 100644
--- a/contrib/pzstd/utils/FileSystem.h
+++ b/contrib/pzstd/utils/FileSystem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Likely.h b/contrib/pzstd/utils/Likely.h
index 7cea8da27..ba01c840d 100644
--- a/contrib/pzstd/utils/Likely.h
+++ b/contrib/pzstd/utils/Likely.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Portability.h b/contrib/pzstd/utils/Portability.h
index 27dbcf707..9410d5b24 100644
--- a/contrib/pzstd/utils/Portability.h
+++ b/contrib/pzstd/utils/Portability.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h
index c78a8935a..6e217eaa1 100644
--- a/contrib/pzstd/utils/Range.h
+++ b/contrib/pzstd/utils/Range.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h
index 8dfcdd765..9bdc32ff2 100644
--- a/contrib/pzstd/utils/ResourcePool.h
+++ b/contrib/pzstd/utils/ResourcePool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ScopeGuard.h b/contrib/pzstd/utils/ScopeGuard.h
index c26f911bf..3ae81da88 100644
--- a/contrib/pzstd/utils/ScopeGuard.h
+++ b/contrib/pzstd/utils/ScopeGuard.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h
index 8ece8e0da..50eed836c 100644
--- a/contrib/pzstd/utils/ThreadPool.h
+++ b/contrib/pzstd/utils/ThreadPool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h
index 1d14d922c..a0bb58c85 100644
--- a/contrib/pzstd/utils/WorkQueue.h
+++ b/contrib/pzstd/utils/WorkQueue.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/BufferTest.cpp b/contrib/pzstd/utils/test/BufferTest.cpp
index fbba74e82..c1a67a57e 100644
--- a/contrib/pzstd/utils/test/BufferTest.cpp
+++ b/contrib/pzstd/utils/test/BufferTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/RangeTest.cpp b/contrib/pzstd/utils/test/RangeTest.cpp
index 755b50fa6..bc2a8e73d 100644
--- a/contrib/pzstd/utils/test/RangeTest.cpp
+++ b/contrib/pzstd/utils/test/RangeTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
index 6fe145180..542c265ec 100644
--- a/contrib/pzstd/utils/test/ResourcePoolTest.cpp
+++ b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ScopeGuardTest.cpp b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
index 7bc624da7..b5e0d00cb 100644
--- a/contrib/pzstd/utils/test/ScopeGuardTest.cpp
+++ b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
index 703fd4c9c..b4656d7bd 100644
--- a/contrib/pzstd/utils/test/ThreadPoolTest.cpp
+++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp
index 14cf77304..70f923e5d 100644
--- a/contrib/pzstd/utils/test/WorkQueueTest.cpp
+++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/Makefile b/contrib/recovery/Makefile
index 9a9f4f2e8..fa500af19 100644
--- a/contrib/recovery/Makefile
+++ b/contrib/recovery/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Facebook, Inc.
+# Copyright (c) 2019-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/recover_directory.c b/contrib/recovery/recover_directory.c
index 13f83fd10..8f6b12abe 100644
--- a/contrib/recovery/recover_directory.c
+++ b/contrib/recovery/recover_directory.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile
index 9df6b75fb..07ec19070 100644
--- a/contrib/seekable_format/examples/Makefile
+++ b/contrib/seekable_format/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Facebook, Inc.
+# Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_compression.c b/contrib/seekable_format/examples/parallel_compression.c
index 4118b0ad7..04347138b 100644
--- a/contrib/seekable_format/examples/parallel_compression.c
+++ b/contrib/seekable_format/examples/parallel_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_processing.c b/contrib/seekable_format/examples/parallel_processing.c
index 36226b49f..8f3096aa2 100644
--- a/contrib/seekable_format/examples/parallel_processing.c
+++ b/contrib/seekable_format/examples/parallel_processing.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c
index 9a331a895..9e1588a3d 100644
--- a/contrib/seekable_format/examples/seekable_compression.c
+++ b/contrib/seekable_format/examples/seekable_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression.c b/contrib/seekable_format/examples/seekable_decompression.c
index e9e201333..19c997041 100644
--- a/contrib/seekable_format/examples/seekable_decompression.c
+++ b/contrib/seekable_format/examples/seekable_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression_mem.c b/contrib/seekable_format/examples/seekable_decompression_mem.c
index e7b1c6505..f1bdb9d33 100644
--- a/contrib/seekable_format/examples/seekable_decompression_mem.c
+++ b/contrib/seekable_format/examples/seekable_decompression_mem.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile
index d51deb3ea..ab5dda696 100644
--- a/contrib/seekable_format/tests/Makefile
+++ b/contrib/seekable_format/tests/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Facebook, Inc.
+# Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstd_seekable_compression_format.md b/contrib/seekable_format/zstd_seekable_compression_format.md
index 55aebfd2e..8cb06af79 100644
--- a/contrib/seekable_format/zstd_seekable_compression_format.md
+++ b/contrib/seekable_format/zstd_seekable_compression_format.md
@@ -2,7 +2,7 @@
 
 ### Notices
 
-Copyright (c) 2017-present Facebook, Inc.
+Copyright (c) 2017-present Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c
index 7ec9bb577..02f19a129 100644
--- a/contrib/seekable_format/zstdseek_compress.c
+++ b/contrib/seekable_format/zstdseek_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index 24083aa52..73c54271d 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seqBench/Makefile b/contrib/seqBench/Makefile
index 0782961eb..63c872097 100644
--- a/contrib/seqBench/Makefile
+++ b/contrib/seqBench/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2018-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index a9c601ebc..7560e36fc 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index 935f60da8..12c5a801b 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index 54c059352..9ade76502 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h
index d89c83523..c13c8134d 100644
--- a/doc/educational_decoder/zstd_decompress.h
+++ b/doc/educational_decoder/zstd_decompress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 20c4a10e3..a8f7f4ff5 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -3,7 +3,7 @@ Zstandard Compression Format
 
 ### Notices
 
-Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
+Copyright (c) 2016-2021 Yann Collet, Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/examples/Makefile b/examples/Makefile
index 8d7361dd8..5fb4c7138 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/common.h b/examples/common.h
index 9b09030eb..621332e11 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c
index 7c7d34288..a878498a5 100644
--- a/examples/dictionary_compression.c
+++ b/examples/dictionary_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c
index 107cfc1ee..77c406645 100644
--- a/examples/dictionary_decompression.c
+++ b/examples/dictionary_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index 5d2a28fcd..cf9113838 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index d4efc8e57..ca9ef5a5d 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_compression.c b/examples/simple_compression.c
index 27a65b17f..5b1c3b301 100644
--- a/examples/simple_compression.c
+++ b/examples/simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index 59c1fd414..1b0aca6db 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index ff1875829..4fb7f282b 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression_thread_pool.c b/examples/streaming_compression_thread_pool.c
index 21cb3d549..a1a024129 100644
--- a/examples/streaming_compression_thread_pool.c
+++ b/examples/streaming_compression_thread_pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Martin Liska, SUSE, Facebook, Inc.
+ * Copyright (c) Martin Liska, SUSE, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 6dc4c2267..ad630e92c 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c
index a5219ef1e..aac62668c 100644
--- a/examples/streaming_memory_usage.c
+++ b/examples/streaming_memory_usage.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/Makefile b/lib/Makefile
index 74c6f1baf..f5b2d652a 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/bits.h b/lib/common/bits.h
index c0e917750..7939f3d0f 100644
--- a/lib/common/bits.h
+++ b/lib/common/bits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index 841778626..c27eb09ab 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * bitstream
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 42f289e0b..bbc26a98d 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/cpu.h b/lib/common/cpu.h
index 8acd33be3..8bc34a36d 100644
--- a/lib/common/cpu.h
+++ b/lib/common/cpu.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/debug.c b/lib/common/debug.c
index bb863c9ea..b98168d12 100644
--- a/lib/common/debug.c
+++ b/lib/common/debug.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/debug.h b/lib/common/debug.h
index 3b2a320a1..5805a9280 100644
--- a/lib/common/debug.h
+++ b/lib/common/debug.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index 98bd4238d..b3d9d0b1b 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Common functions of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 1b67500f3..bbd380188 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index 007d81066..f82775add 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/fse.h b/lib/common/fse.h
index 466a07281..212f4697d 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
index 7034fd97b..fb641e71d 100644
--- a/lib/common/fse_decompress.c
+++ b/lib/common/fse_decompress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 595b2f6db..2905fe8a4 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/mem.h b/lib/common/mem.h
index 493782f6f..7a55f6fc1 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/pool.c b/lib/common/pool.c
index 63dc9d377..c37d7077a 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/pool.h b/lib/common/pool.h
index b86a3452e..b5078f3cb 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/portability_macros.h b/lib/common/portability_macros.h
index 1650fa3d8..bf3e2b7b1 100644
--- a/lib/common/portability_macros.h
+++ b/lib/common/portability_macros.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index c2d61ca64..efd010803 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -1,6 +1,6 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Facebook, Inc.
+ *  Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - xxHash homepage: https://cyan4973.github.io/xxHash/
diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index 901bc0832..6889cad14 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -1,6 +1,6 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Facebook, Inc.
+ *  Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - xxHash homepage: https://cyan4973.github.io/xxHash/
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 3d7e35b30..fce41243d 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_deps.h b/lib/common/zstd_deps.h
index 14211344a..4d767ae9b 100644
--- a/lib/common/zstd_deps.h
+++ b/lib/common/zstd_deps.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index e89226702..8463334ce 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_trace.h b/lib/common/zstd_trace.h
index 6215f1e70..da20534eb 100644
--- a/lib/common/zstd_trace.h
+++ b/lib/common/zstd_trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/clevels.h b/lib/compress/clevels.h
index 7ed2e0049..99172c3c7 100644
--- a/lib/compress/clevels.h
+++ b/lib/compress/clevels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index c2271f6ee..065620bc8 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.c b/lib/compress/hist.c
index 073c57e75..9338276f0 100644
--- a/lib/compress/hist.c
+++ b/lib/compress/hist.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.h b/lib/compress/hist.h
index 228ed48a7..8b6a1b696 100644
--- a/lib/compress/hist.h
+++ b/lib/compress/hist.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 9d2b521bc..a0fe97c67 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ecb21f268..2686711a6 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 627d441af..4d138993b 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index ea80a45c8..939bcaa96 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h
index bb260db9b..b6befb24c 100644
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c
index 2c1eee567..cfd7e5f32 100644
--- a/lib/compress/zstd_compress_sequences.c
+++ b/lib/compress/zstd_compress_sequences.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.h b/lib/compress/zstd_compress_sequences.h
index 7991364c2..5f4421adf 100644
--- a/lib/compress/zstd_compress_sequences.h
+++ b/lib/compress/zstd_compress_sequences.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index eed58e7cf..fa3bf02de 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.h b/lib/compress/zstd_compress_superblock.h
index 176f9b106..6d16e2b33 100644
--- a/lib/compress/zstd_compress_superblock.h
+++ b/lib/compress/zstd_compress_superblock.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 47afe3dc7..cab928bc6 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index c2dbd54c1..82f41af69 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h
index 6d8ee8c65..66c8badf6 100644
--- a/lib/compress/zstd_double_fast.h
+++ b/lib/compress/zstd_double_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 291173449..bddd15997 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h
index 3bfeb2c5f..2032081bb 100644
--- a/lib/compress/zstd_fast.h
+++ b/lib/compress/zstd_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index d5a7b5cbf..8605cf02c 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index 150f7b390..e2f9d6c99 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index c14c62454..698f166d6 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 4e68dbf52..a2d2deef7 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm_geartab.h b/lib/compress/zstd_ldm_geartab.h
index 647f865be..8087c54cd 100644
--- a/lib/compress/zstd_ldm_geartab.h
+++ b/lib/compress/zstd_ldm_geartab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 800f87e9e..668212df9 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 627255f53..e690e5d2d 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 0c10eb603..e3fc54685 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 271eb1ac7..ff1fa0ef4 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index c6fd92860..3ee35c470 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/decompress/huf_decompress_amd64.S b/lib/decompress/huf_decompress_amd64.S
index 3f0e5c26c..8d7c7a644 100644
--- a/lib/decompress/huf_decompress_amd64.S
+++ b/lib/decompress/huf_decompress_amd64.S
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index 6ffa35f6e..ec451d37b 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h
index bd03268b5..b3f394b9c 100644
--- a/lib/decompress/zstd_ddict.h
+++ b/lib/decompress/zstd_ddict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 57f220d0c..373733373 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index e1ff21582..d8b2814fa 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_block.h b/lib/decompress/zstd_decompress_block.h
index c61a9d0c4..e59f9d3e2 100644
--- a/lib/decompress/zstd_decompress_block.h
+++ b/lib/decompress/zstd_decompress_block.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 91e9dceb5..2252c7d5d 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index b83ea0fed..69933ebda 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_common.c b/lib/deprecated/zbuff_common.c
index e7d01a081..1d4310511 100644
--- a/lib/deprecated/zbuff_common.c
+++ b/lib/deprecated/zbuff_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_compress.c b/lib/deprecated/zbuff_compress.c
index 51cf158c4..bf1051ff9 100644
--- a/lib/deprecated/zbuff_compress.c
+++ b/lib/deprecated/zbuff_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
index d73c0f35f..7795fc369 100644
--- a/lib/deprecated/zbuff_decompress.c
+++ b/lib/deprecated/zbuff_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 724675d30..92852c1f0 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index 1aacdddd6..252624bde 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 63a2cee72..46bba0120 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index a932276ff..375a17649 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dll/example/Makefile b/lib/dll/example/Makefile
index 03b034dd5..be996fad6 100644
--- a/lib/dll/example/Makefile
+++ b/lib/dll/example/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index a6f1174b8..cc2165b31 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index c9e4f632f..8c504c792 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index f777eb6e4..6ac876954 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index d80481784..dfaed7bb8 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 1b371953b..dab0260ee 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 20ca5681a..5b1fd7175 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 7a00d4304..9bf3cce64 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 4f7eb9277..23735443a 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 66b97ab8e..640240d62 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 9e73a5daa..5d2ebb2b5 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index bd423bfc1..2dcffc923 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 3839a5f63..53aaec468 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index 9e32b76e0..633891010 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index efe6fbefe..c2b88eb3e 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index bc35cfa6a..1ff39041f 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/libzstd.mk b/lib/libzstd.mk
index 60cab8cc8..152e538db 100644
--- a/lib/libzstd.mk
+++ b/lib/libzstd.mk
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zdict.h b/lib/zdict.h
index 231bf8282..8f52e2a86 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd.h b/lib/zstd.h
index 09a200af2..af9e2e2d7 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 68ce01cd1..2c97a48d7 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/Makefile b/programs/Makefile
index 1cdec4a64..c626aae5d 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.c b/programs/benchfn.c
index 1aadbdd91..d94377734 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.h b/programs/benchfn.h
index 99d13ac47..65409b4f5 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 6ceca020c..bf5279013 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 11ac85da7..c19d03ce9 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.c b/programs/datagen.c
index 3b4f9e5c7..8baf40a5e 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.h b/programs/datagen.h
index b76ae2a22..c088e71d9 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.c b/programs/dibio.c
index 8643bc378..a45c65dad 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.h b/programs/dibio.h
index 666c1e661..b8e859f5a 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.c b/programs/fileio.c
index 313a530ce..3d532ae33 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.h b/programs/fileio.h
index b848934bc..c3264534d 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_asyncio.c b/programs/fileio_asyncio.c
index 92c9a5b1d..8f12fe1f9 100644
--- a/programs/fileio_asyncio.c
+++ b/programs/fileio_asyncio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_asyncio.h b/programs/fileio_asyncio.h
index 30db44b6e..34dad6f4d 100644
--- a/programs/fileio_asyncio.h
+++ b/programs/fileio_asyncio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_common.h b/programs/fileio_common.h
index aec2e8d56..827a5a06b 100644
--- a/programs/fileio_common.h
+++ b/programs/fileio_common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_types.h b/programs/fileio_types.h
index a1fac2ca7..c1f42f1ad 100644
--- a/programs/fileio_types.h
+++ b/programs/fileio_types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/platform.h b/programs/platform.h
index 7006f97f2..f7d558595 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.c b/programs/timefn.c
index 64577b0e9..d6f4749b2 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.h b/programs/timefn.h
index 8ba8ed787..b92b5605e 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/util.c b/programs/util.c
index bdb651074..6f73348b3 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/util.h b/programs/util.h
index cf1738772..90b7afe91 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/windres/verrsrc.h b/programs/windres/verrsrc.h
index c1b60e90f..9d06960b1 100644
--- a/programs/windres/verrsrc.h
+++ b/programs/windres/verrsrc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc
index f5e404730..76de608d8 100644
--- a/programs/windres/zstd.rc
+++ b/programs/windres/zstd.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "CompanyName", "Yann Collet, Meta Platforms, Inc. and affiliates."
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index cc6bbb935..f177ee23b 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/zstdcli_trace.c b/programs/zstdcli_trace.c
index b3b977feb..35075a52c 100644
--- a/programs/zstdcli_trace.c
+++ b/programs/zstdcli_trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/zstdcli_trace.h b/programs/zstdcli_trace.h
index 38c27dc04..9c135d3ca 100644
--- a/programs/zstdcli_trace.h
+++ b/programs/zstdcli_trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/DEPRECATED-test-zstd-speed.py b/tests/DEPRECATED-test-zstd-speed.py
index 665e0a7f9..568b059b7 100755
--- a/tests/DEPRECATED-test-zstd-speed.py
+++ b/tests/DEPRECATED-test-zstd-speed.py
@@ -2,7 +2,7 @@
 # THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
 
 # ################################################################
-# Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+# Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/Makefile b/tests/Makefile
index afea6475a..a8f2401ee 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,6 @@
 
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py
index c98391193..153e7db4c 100644
--- a/tests/automated_benchmarking.py
+++ b/tests/automated_benchmarking.py
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/bigdict.c b/tests/bigdict.c
index fb0892586..b8fe92162 100644
--- a/tests/bigdict.c
+++ b/tests/bigdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/checkTag.c b/tests/checkTag.c
index f6c5e9766..38e09ddc9 100644
--- a/tests/checkTag.c
+++ b/tests/checkTag.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/check_size.py b/tests/check_size.py
index 408f0f795..028b0a9c0 100755
--- a/tests/check_size.py
+++ b/tests/check_size.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/cli-tests/run.py b/tests/cli-tests/run.py
index d726fba7f..2696d8b64 100755
--- a/tests/cli-tests/run.py
+++ b/tests/cli-tests/run.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/datagencli.c b/tests/datagencli.c
index 7300fdb76..0d291952f 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index d2b126f39..c53e8a3fd 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fullbench.c b/tests/fullbench.c
index b55ff767f..097c97f13 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 912348c38..35a19dd89 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c
index bdbf76916..8307e6f3c 100644
--- a/tests/fuzz/block_decompress.c
+++ b/tests/fuzz/block_decompress.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c
index 46a84c7b9..54012a1da 100644
--- a/tests/fuzz/block_round_trip.c
+++ b/tests/fuzz/block_round_trip.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/decompress_dstSize_tooSmall.c b/tests/fuzz/decompress_dstSize_tooSmall.c
index 3f7607b29..8ad3ea1c9 100644
--- a/tests/fuzz/decompress_dstSize_tooSmall.c
+++ b/tests/fuzz/decompress_dstSize_tooSmall.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c
index 33c58c8bd..8dea3e560 100644
--- a/tests/fuzz/dictionary_decompress.c
+++ b/tests/fuzz/dictionary_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_loader.c b/tests/fuzz/dictionary_loader.c
index 5b60bc484..1ac22740f 100644
--- a/tests/fuzz/dictionary_loader.c
+++ b/tests/fuzz/dictionary_loader.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c
index 0b20e8d67..6dd78c3f3 100644
--- a/tests/fuzz/dictionary_round_trip.c
+++ b/tests/fuzz/dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_stream_round_trip.c b/tests/fuzz/dictionary_stream_round_trip.c
index 9af712f0e..3ebbd843b 100644
--- a/tests/fuzz/dictionary_stream_round_trip.c
+++ b/tests/fuzz/dictionary_stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fse_read_ncount.c b/tests/fuzz/fse_read_ncount.c
index c32386099..29e19442a 100644
--- a/tests/fuzz/fse_read_ncount.c
+++ b/tests/fuzz/fse_read_ncount.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h
index 6f3fb2994..d1e439fef 100644
--- a/tests/fuzz/fuzz.h
+++ b/tests/fuzz/fuzz.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 17eac4f6f..03ffeeefd 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c
index eae8ee4b6..a93e8ba95 100644
--- a/tests/fuzz/fuzz_data_producer.c
+++ b/tests/fuzz/fuzz_data_producer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h
index 62771a9f8..8ca501f9b 100644
--- a/tests/fuzz/fuzz_data_producer.h
+++ b/tests/fuzz/fuzz_data_producer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_helpers.c b/tests/fuzz/fuzz_helpers.c
index 61c0debe5..1b6ad9736 100644
--- a/tests/fuzz/fuzz_helpers.c
+++ b/tests/fuzz/fuzz_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h
index c180478e9..aaf4c1df4 100644
--- a/tests/fuzz/fuzz_helpers.h
+++ b/tests/fuzz/fuzz_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/huf_decompress.c b/tests/fuzz/huf_decompress.c
index fea09fc93..13f040a17 100644
--- a/tests/fuzz/huf_decompress.c
+++ b/tests/fuzz/huf_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/huf_round_trip.c b/tests/fuzz/huf_round_trip.c
index 32b08d6d5..0993ecd14 100644
--- a/tests/fuzz/huf_round_trip.c
+++ b/tests/fuzz/huf_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/raw_dictionary_round_trip.c b/tests/fuzz/raw_dictionary_round_trip.c
index 0e6517654..7ceab2be6 100644
--- a/tests/fuzz/raw_dictionary_round_trip.c
+++ b/tests/fuzz/raw_dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c
index e6d2dec64..550c65d86 100644
--- a/tests/fuzz/regression_driver.c
+++ b/tests/fuzz/regression_driver.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/seekable_roundtrip.c b/tests/fuzz/seekable_roundtrip.c
index dcdcaae11..6f0aa288f 100644
--- a/tests/fuzz/seekable_roundtrip.c
+++ b/tests/fuzz/seekable_roundtrip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c
index cc872a081..ec8fef488 100644
--- a/tests/fuzz/sequence_compression_api.c
+++ b/tests/fuzz/sequence_compression_api.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c
index 8e6980b35..c9fea22d3 100644
--- a/tests/fuzz/simple_compress.c
+++ b/tests/fuzz/simple_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c
index dfff11cbc..ce5f9f098 100644
--- a/tests/fuzz/simple_decompress.c
+++ b/tests/fuzz/simple_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 9da986bc9..23a805af2 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c
index 86a39b8c9..0254d06ec 100644
--- a/tests/fuzz/stream_decompress.c
+++ b/tests/fuzz/stream_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index 719eac48d..8a28907b6 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c
index 9ce645d15..95dbdd49a 100644
--- a/tests/fuzz/zstd_frame_info.c
+++ b/tests/fuzz/zstd_frame_info.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 08ce70dd7..d5210627e 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h
index 9fbefdc72..a4cfe321b 100644
--- a/tests/fuzz/zstd_helpers.h
+++ b/tests/fuzz/zstd_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index d58751a5c..edbb613ec 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
index a50350f8d..cca310967 100644
--- a/tests/gzip/Makefile
+++ b/tests/gzip/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c
index b71f7419f..0841cca59 100644
--- a/tests/invalidDictionaries.c
+++ b/tests/invalidDictionaries.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/legacy.c b/tests/legacy.c
index ac4938fe5..e0a360101 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/longmatch.c b/tests/longmatch.c
index a171c0e4a..499cb5541 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 756a43340..224643c84 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 08f31c069..3d2d4ce03 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py
index 26293728e..3e65081e0 100755
--- a/tests/rateLimiter.py
+++ b/tests/rateLimiter.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/Makefile b/tests/regression/Makefile
index a440c6c94..ba8b43024 100644
--- a/tests/regression/Makefile
+++ b/tests/regression/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.c b/tests/regression/config.c
index 30d0ca5e2..1d86fff23 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.h b/tests/regression/config.h
index dd8893743..a4b542a90 100644
--- a/tests/regression/config.h
+++ b/tests/regression/config.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.c b/tests/regression/data.c
index 341b02d3b..43f085f2c 100644
--- a/tests/regression/data.c
+++ b/tests/regression/data.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.h b/tests/regression/data.h
index e54e6a1dc..a4ee92072 100644
--- a/tests/regression/data.h
+++ b/tests/regression/data.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/levels.h b/tests/regression/levels.h
index e98209d80..d15b12046 100644
--- a/tests/regression/levels.h
+++ b/tests/regression/levels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.c b/tests/regression/method.c
index 55b1154eb..f84a15ef3 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.h b/tests/regression/method.h
index 1a36a9399..8efdd33a0 100644
--- a/tests/regression/method.h
+++ b/tests/regression/method.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.c b/tests/regression/result.c
index 1f879c140..8ccb8751e 100644
--- a/tests/regression/result.c
+++ b/tests/regression/result.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.h b/tests/regression/result.h
index 197fa905d..8a761ea4d 100644
--- a/tests/regression/result.h
+++ b/tests/regression/result.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/test.c b/tests/regression/test.c
index 1de6be86c..07600be57 100644
--- a/tests/regression/test.c
+++ b/tests/regression/test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c
index 9aa208cff..bfcd8cfe2 100644
--- a/tests/roundTripCrash.c
+++ b/tests/roundTripCrash.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.c b/tests/seqgen.c
index 1e340c851..0d8a766c8 100644
--- a/tests/seqgen.c
+++ b/tests/seqgen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.h b/tests/seqgen.h
index cea3f552b..df17398ef 100644
--- a/tests/seqgen.h
+++ b/tests/seqgen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/test-license.py b/tests/test-license.py
index f27cb6774..317c459d1 100755
--- a/tests/test-license.py
+++ b/tests/test-license.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index 5f4b1e725..4995e32d4 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -2,7 +2,7 @@
 """Test zstd interoperability between versions"""
 
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 453c45409..2830f25f7 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 8673ca348..0ca4c3595 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/gzcompatibility.h b/zlibWrapper/gzcompatibility.h
index 3a46127b8..969bfdbad 100644
--- a/zlibWrapper/gzcompatibility.h
+++ b/zlibWrapper/gzcompatibility.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 386c0f0ac..bec433e8f 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index c39cf6379..22ff1a722 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

From 36d5c2f32621c7e434640006960c77dac3e83eb0 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 20 Dec 2022 12:42:50 -0500
Subject: [PATCH 058/197] Update Copyright Year ('2021' -> 'present')

```
for f in $(find . \( -path ./.git -o -path ./tests/fuzz/corpora -o -path ./tests/regression/data-cache -o -path ./tests/regression/cache \) -prune -o -type f);
do
  sed -i 's/\-2021/-present/' $f;
done

g co HEAD -- .github/workflows/dev-short-tests.yml # fix bad match
```
---
 Makefile                                           | 2 +-
 contrib/freestanding_lib/freestanding.py           | 2 +-
 contrib/linux-kernel/test/include/linux/compiler.h | 2 +-
 contrib/linux-kernel/test/include/linux/errno.h    | 2 +-
 contrib/linux-kernel/test/include/linux/kernel.h   | 2 +-
 contrib/linux-kernel/test/include/linux/limits.h   | 2 +-
 contrib/linux-kernel/test/include/linux/math64.h   | 2 +-
 contrib/linux-kernel/test/include/linux/module.h   | 2 +-
 contrib/linux-kernel/test/include/linux/printk.h   | 2 +-
 contrib/linux-kernel/test/include/linux/stddef.h   | 2 +-
 contrib/linux-kernel/test/include/linux/swab.h     | 2 +-
 contrib/linux-kernel/test/include/linux/types.h    | 2 +-
 contrib/recovery/recover_directory.c               | 2 +-
 doc/zstd_compression_format.md                     | 2 +-
 zlibWrapper/gzcompatibility.h                      | 2 +-
 zlibWrapper/zstd_zlibwrapper.c                     | 2 +-
 zlibWrapper/zstd_zlibwrapper.h                     | 2 +-
 17 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/Makefile b/Makefile
index dcd872f5c..65d213061 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2021, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index 12eb52ab0..33a1ead38 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) 2021-2021, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2021-present, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h
index ed3062d8f..f882850c1 100644
--- a/contrib/linux-kernel/test/include/linux/compiler.h
+++ b/contrib/linux-kernel/test/include/linux/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h
index 4115f6091..fe643adf5 100644
--- a/contrib/linux-kernel/test/include/linux/errno.h
+++ b/contrib/linux-kernel/test/include/linux/errno.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h
index 19d5f26cc..0543b4ad3 100644
--- a/contrib/linux-kernel/test/include/linux/kernel.h
+++ b/contrib/linux-kernel/test/include/linux/kernel.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h
index 9dbcc42a5..e9a818ac4 100644
--- a/contrib/linux-kernel/test/include/linux/limits.h
+++ b/contrib/linux-kernel/test/include/linux/limits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h
index 60b05bd68..f8821528b 100644
--- a/contrib/linux-kernel/test/include/linux/math64.h
+++ b/contrib/linux-kernel/test/include/linux/math64.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h
index 0e8c5563d..0821f296c 100644
--- a/contrib/linux-kernel/test/include/linux/module.h
+++ b/contrib/linux-kernel/test/include/linux/module.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h
index 7661c0b37..1989aeb53 100644
--- a/contrib/linux-kernel/test/include/linux/printk.h
+++ b/contrib/linux-kernel/test/include/linux/printk.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h
index 689edb0e7..94f1f36ee 100644
--- a/contrib/linux-kernel/test/include/linux/stddef.h
+++ b/contrib/linux-kernel/test/include/linux/stddef.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h
index 36c25550c..aeeb84c03 100644
--- a/contrib/linux-kernel/test/include/linux/swab.h
+++ b/contrib/linux-kernel/test/include/linux/swab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h
index 1c194a8ac..917fb428f 100644
--- a/contrib/linux-kernel/test/include/linux/types.h
+++ b/contrib/linux-kernel/test/include/linux/types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/recover_directory.c b/contrib/recovery/recover_directory.c
index 8f6b12abe..749ca7df6 100644
--- a/contrib/recovery/recover_directory.c
+++ b/contrib/recovery/recover_directory.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index a8f7f4ff5..4b2330994 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -3,7 +3,7 @@ Zstandard Compression Format
 
 ### Notices
 
-Copyright (c) 2016-2021 Yann Collet, Meta Platforms, Inc. and affiliates.
+Copyright (c) 2016-present Yann Collet, Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/zlibWrapper/gzcompatibility.h b/zlibWrapper/gzcompatibility.h
index 969bfdbad..68d29090c 100644
--- a/zlibWrapper/gzcompatibility.h
+++ b/zlibWrapper/gzcompatibility.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index bec433e8f..e001629bf 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index 22ff1a722..2a34763e4 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

From 4c999cf968f6c5cf1bec76c806f4558d51fb0d54 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 21 Jun 2022 12:11:22 -0400
Subject: [PATCH 059/197] Update test-license.py

---
 tests/test-license.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test-license.py b/tests/test-license.py
index 317c459d1..d54c16419 100755
--- a/tests/test-license.py
+++ b/tests/test-license.py
@@ -83,8 +83,8 @@ def valid_copyright(lines):
             continue
         if "present" in line:
             return (False, f"Copyright line '{line}' contains 'present'!")
-        if "Facebook, Inc" not in line:
-            return (False, f"Copyright line '{line}' does not contain 'Facebook, Inc'")
+        if "Meta Platforms, Inc" not in line:
+            return (False, f"Copyright line '{line}' does not contain 'Meta Platforms, Inc'")
         year = YEAR_REGEX.search(line)
         if year is not None:
             return (False, f"Copyright line '{line}' contains {year.group(0)}; it should be yearless")

From 7f12f24cf47416fee0f1e84e4c57feb60e1bed0d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 20 Dec 2022 12:44:56 -0500
Subject: [PATCH 060/197] Rewrite Copyright Date Ranges from `-present` to
 `-2022`

Apparently it's better. Somehow.

```
for f in $(find . \( -path ./.git -o -path ./tests/fuzz/corpora -o -path ./tests/regression/data-cache -o -path ./tests/regression/cache \) -prune -o -type f); do echo $f; sed -i 's/\-present/-2022/' $f; done

g co HEAD -- build/meson/
```
---
 LICENSE                                                       | 2 +-
 Makefile                                                      | 2 +-
 build/VS2010/libzstd-dll/libzstd-dll.rc                       | 2 +-
 build/VS2010/zstd/zstd.rc                                     | 2 +-
 build/cmake/CMakeLists.txt                                    | 2 +-
 build/cmake/contrib/CMakeLists.txt                            | 2 +-
 build/cmake/contrib/gen_html/CMakeLists.txt                   | 2 +-
 build/cmake/contrib/pzstd/CMakeLists.txt                      | 2 +-
 build/cmake/lib/CMakeLists.txt                                | 2 +-
 build/cmake/programs/CMakeLists.txt                           | 2 +-
 build/cmake/tests/CMakeLists.txt                              | 2 +-
 build/single_file_libs/zstd-in.c                              | 2 +-
 build/single_file_libs/zstddeclib-in.c                        | 2 +-
 contrib/diagnose_corruption/Makefile                          | 2 +-
 contrib/diagnose_corruption/check_flipped_bits.c              | 2 +-
 contrib/freestanding_lib/freestanding.py                      | 2 +-
 contrib/gen_html/Makefile                                     | 2 +-
 contrib/gen_html/gen_html.cpp                                 | 2 +-
 contrib/largeNbDicts/Makefile                                 | 2 +-
 contrib/largeNbDicts/largeNbDicts.c                           | 2 +-
 contrib/linux-kernel/test/include/linux/compiler.h            | 2 +-
 contrib/linux-kernel/test/include/linux/errno.h               | 2 +-
 contrib/linux-kernel/test/include/linux/kernel.h              | 2 +-
 contrib/linux-kernel/test/include/linux/limits.h              | 2 +-
 contrib/linux-kernel/test/include/linux/math64.h              | 2 +-
 contrib/linux-kernel/test/include/linux/module.h              | 2 +-
 contrib/linux-kernel/test/include/linux/printk.h              | 2 +-
 contrib/linux-kernel/test/include/linux/stddef.h              | 2 +-
 contrib/linux-kernel/test/include/linux/swab.h                | 2 +-
 contrib/linux-kernel/test/include/linux/types.h               | 2 +-
 contrib/match_finders/zstd_edist.c                            | 2 +-
 contrib/match_finders/zstd_edist.h                            | 2 +-
 contrib/pzstd/ErrorHolder.h                                   | 2 +-
 contrib/pzstd/Logging.h                                       | 2 +-
 contrib/pzstd/Makefile                                        | 2 +-
 contrib/pzstd/Options.cpp                                     | 2 +-
 contrib/pzstd/Options.h                                       | 2 +-
 contrib/pzstd/Pzstd.cpp                                       | 2 +-
 contrib/pzstd/Pzstd.h                                         | 2 +-
 contrib/pzstd/SkippableFrame.cpp                              | 2 +-
 contrib/pzstd/SkippableFrame.h                                | 2 +-
 contrib/pzstd/main.cpp                                        | 2 +-
 contrib/pzstd/test/OptionsTest.cpp                            | 2 +-
 contrib/pzstd/test/PzstdTest.cpp                              | 2 +-
 contrib/pzstd/test/RoundTrip.h                                | 2 +-
 contrib/pzstd/test/RoundTripTest.cpp                          | 2 +-
 contrib/pzstd/utils/Buffer.h                                  | 2 +-
 contrib/pzstd/utils/FileSystem.h                              | 2 +-
 contrib/pzstd/utils/Likely.h                                  | 2 +-
 contrib/pzstd/utils/Portability.h                             | 2 +-
 contrib/pzstd/utils/Range.h                                   | 2 +-
 contrib/pzstd/utils/ResourcePool.h                            | 2 +-
 contrib/pzstd/utils/ScopeGuard.h                              | 2 +-
 contrib/pzstd/utils/ThreadPool.h                              | 2 +-
 contrib/pzstd/utils/WorkQueue.h                               | 2 +-
 contrib/pzstd/utils/test/BufferTest.cpp                       | 2 +-
 contrib/pzstd/utils/test/RangeTest.cpp                        | 2 +-
 contrib/pzstd/utils/test/ResourcePoolTest.cpp                 | 2 +-
 contrib/pzstd/utils/test/ScopeGuardTest.cpp                   | 2 +-
 contrib/pzstd/utils/test/ThreadPoolTest.cpp                   | 2 +-
 contrib/pzstd/utils/test/WorkQueueTest.cpp                    | 2 +-
 contrib/recovery/Makefile                                     | 2 +-
 contrib/recovery/recover_directory.c                          | 2 +-
 contrib/seekable_format/examples/Makefile                     | 2 +-
 contrib/seekable_format/examples/parallel_compression.c       | 2 +-
 contrib/seekable_format/examples/parallel_processing.c        | 2 +-
 contrib/seekable_format/examples/seekable_compression.c       | 2 +-
 contrib/seekable_format/examples/seekable_decompression.c     | 2 +-
 contrib/seekable_format/examples/seekable_decompression_mem.c | 2 +-
 contrib/seekable_format/tests/Makefile                        | 2 +-
 contrib/seekable_format/zstd_seekable_compression_format.md   | 2 +-
 contrib/seekable_format/zstdseek_compress.c                   | 2 +-
 contrib/seekable_format/zstdseek_decompress.c                 | 2 +-
 contrib/seqBench/Makefile                                     | 2 +-
 doc/zstd_compression_format.md                                | 2 +-
 programs/windres/zstd.rc                                      | 2 +-
 zlibWrapper/examples/zwrapbench.c                             | 2 +-
 zlibWrapper/gzcompatibility.h                                 | 2 +-
 zlibWrapper/zstd_zlibwrapper.c                                | 2 +-
 zlibWrapper/zstd_zlibwrapper.h                                | 2 +-
 80 files changed, 80 insertions(+), 80 deletions(-)

diff --git a/LICENSE b/LICENSE
index 493d0db22..d2bbd709b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -2,7 +2,7 @@ BSD License
 
 For Zstandard software
 
-Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates. All rights reserved.
+Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
diff --git a/Makefile b/Makefile
index 65d213061..641feb869 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.rc b/build/VS2010/libzstd-dll/libzstd-dll.rc
index 05c8fbd41..80d3a306c 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.rc
+++ b/build/VS2010/libzstd-dll/libzstd-dll.rc
@@ -36,7 +36,7 @@ BEGIN
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "libzstd.dll"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Meta Platforms, Inc."
+            VALUE "LegalCopyright", "Copyright (c) 2013-2022, Meta Platforms, Inc."
             VALUE "OriginalFilename", "libzstd.dll"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/build/VS2010/zstd/zstd.rc b/build/VS2010/zstd/zstd.rc
index c1df78a13..57b150d92 100644
--- a/build/VS2010/zstd/zstd.rc
+++ b/build/VS2010/zstd/zstd.rc
@@ -36,7 +36,7 @@ BEGIN
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Meta Platforms, Inc."
+            VALUE "LegalCopyright", "Copyright (c) 2013-2022, Meta Platforms, Inc."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 193f60d77..c41861ad0 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt
index 3f253cf22..c4684007f 100644
--- a/build/cmake/contrib/CMakeLists.txt
+++ b/build/cmake/contrib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt
index 96975421b..1f945a477 100644
--- a/build/cmake/contrib/gen_html/CMakeLists.txt
+++ b/build/cmake/contrib/gen_html/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt
index e6f99c7d2..fa419a6cd 100644
--- a/build/cmake/contrib/pzstd/CMakeLists.txt
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 9f1e7db49..13f21dafd 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index 650b8c571..b94dad3e7 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 6db8dd729..7f6477abc 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -1,6 +1,6 @@
 # ################################################################
 # zstd - Makefile
-# Copyright (C) Yann Collet 2014-present
+# Copyright (C) Yann Collet 2014-2022
 # All rights reserved.
 #
 # BSD license
diff --git a/build/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c
index 89296179d..c22b8cd25 100644
--- a/build/single_file_libs/zstd-in.c
+++ b/build/single_file_libs/zstd-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/build/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c
index 9e7d36ec9..218e5a698 100644
--- a/build/single_file_libs/zstddeclib-in.c
+++ b/build/single_file_libs/zstddeclib-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/Makefile b/contrib/diagnose_corruption/Makefile
index ff17dd894..7fee63284 100644
--- a/contrib/diagnose_corruption/Makefile
+++ b/contrib/diagnose_corruption/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2019-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/check_flipped_bits.c b/contrib/diagnose_corruption/check_flipped_bits.c
index 342d9baba..e76e99d72 100644
--- a/contrib/diagnose_corruption/check_flipped_bits.c
+++ b/contrib/diagnose_corruption/check_flipped_bits.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2019-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index 33a1ead38..6a26d1d14 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) 2021-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2021-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/Makefile b/contrib/gen_html/Makefile
index ebb13fa0e..64e1e2cac 100644
--- a/contrib/gen_html/Makefile
+++ b/contrib/gen_html/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp
index 9ee807c22..83c5994e6 100644
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Przemyslaw Skibinski, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile
index 611ec8e63..2f514e406 100644
--- a/contrib/largeNbDicts/Makefile
+++ b/contrib/largeNbDicts/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2018-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index ad0522918..13216035a 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2018-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h
index f882850c1..859d5d860 100644
--- a/contrib/linux-kernel/test/include/linux/compiler.h
+++ b/contrib/linux-kernel/test/include/linux/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h
index fe643adf5..c55da480c 100644
--- a/contrib/linux-kernel/test/include/linux/errno.h
+++ b/contrib/linux-kernel/test/include/linux/errno.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h
index 0543b4ad3..f3c42d440 100644
--- a/contrib/linux-kernel/test/include/linux/kernel.h
+++ b/contrib/linux-kernel/test/include/linux/kernel.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h
index e9a818ac4..da9f5d528 100644
--- a/contrib/linux-kernel/test/include/linux/limits.h
+++ b/contrib/linux-kernel/test/include/linux/limits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h
index f8821528b..f834c0c41 100644
--- a/contrib/linux-kernel/test/include/linux/math64.h
+++ b/contrib/linux-kernel/test/include/linux/math64.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h
index 0821f296c..77285dae6 100644
--- a/contrib/linux-kernel/test/include/linux/module.h
+++ b/contrib/linux-kernel/test/include/linux/module.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h
index 1989aeb53..ca84b2868 100644
--- a/contrib/linux-kernel/test/include/linux/printk.h
+++ b/contrib/linux-kernel/test/include/linux/printk.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h
index 94f1f36ee..8760df1c8 100644
--- a/contrib/linux-kernel/test/include/linux/stddef.h
+++ b/contrib/linux-kernel/test/include/linux/stddef.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h
index aeeb84c03..d6f48db66 100644
--- a/contrib/linux-kernel/test/include/linux/swab.h
+++ b/contrib/linux-kernel/test/include/linux/swab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h
index 917fb428f..8610dcaed 100644
--- a/contrib/linux-kernel/test/include/linux/types.h
+++ b/contrib/linux-kernel/test/include/linux/types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.c b/contrib/match_finders/zstd_edist.c
index 73a7f6d77..ab89145ad 100644
--- a/contrib/match_finders/zstd_edist.c
+++ b/contrib/match_finders/zstd_edist.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.h b/contrib/match_finders/zstd_edist.h
index b60a5dc05..328a02ccc 100644
--- a/contrib/match_finders/zstd_edist.h
+++ b/contrib/match_finders/zstd_edist.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/ErrorHolder.h b/contrib/pzstd/ErrorHolder.h
index d144e0fcc..0b0456b32 100644
--- a/contrib/pzstd/ErrorHolder.h
+++ b/contrib/pzstd/ErrorHolder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h
index 652eae824..2c797b927 100644
--- a/contrib/pzstd/Logging.h
+++ b/contrib/pzstd/Logging.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index 8604bac0c..d6bd19050 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index c278b1977..2462736ae 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h
index 5bf6a6492..1220984dc 100644
--- a/contrib/pzstd/Options.h
+++ b/contrib/pzstd/Options.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 794d2c180..5f90e5a8c 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h
index 46c74cc3f..5c47f4998 100644
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.cpp b/contrib/pzstd/SkippableFrame.cpp
index 058667a06..c866da4ce 100644
--- a/contrib/pzstd/SkippableFrame.cpp
+++ b/contrib/pzstd/SkippableFrame.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.h b/contrib/pzstd/SkippableFrame.h
index 389f5304f..78b859173 100644
--- a/contrib/pzstd/SkippableFrame.h
+++ b/contrib/pzstd/SkippableFrame.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp
index dea12fac7..bc8f8caf6 100644
--- a/contrib/pzstd/main.cpp
+++ b/contrib/pzstd/main.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp
index dd9a0113b..1ac50b803 100644
--- a/contrib/pzstd/test/OptionsTest.cpp
+++ b/contrib/pzstd/test/OptionsTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp
index 60c03e8c3..5aa329aec 100644
--- a/contrib/pzstd/test/PzstdTest.cpp
+++ b/contrib/pzstd/test/PzstdTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h
index 4990501ae..2834e9a1e 100644
--- a/contrib/pzstd/test/RoundTrip.h
+++ b/contrib/pzstd/test/RoundTrip.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp
index af0e8149e..22c59f4fb 100644
--- a/contrib/pzstd/test/RoundTripTest.cpp
+++ b/contrib/pzstd/test/RoundTripTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Buffer.h b/contrib/pzstd/utils/Buffer.h
index 5a640d60b..3b1bbd6ec 100644
--- a/contrib/pzstd/utils/Buffer.h
+++ b/contrib/pzstd/utils/Buffer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h
index 2615c4e00..a33813f91 100644
--- a/contrib/pzstd/utils/FileSystem.h
+++ b/contrib/pzstd/utils/FileSystem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Likely.h b/contrib/pzstd/utils/Likely.h
index ba01c840d..cc5a363ae 100644
--- a/contrib/pzstd/utils/Likely.h
+++ b/contrib/pzstd/utils/Likely.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Portability.h b/contrib/pzstd/utils/Portability.h
index 9410d5b24..55db32383 100644
--- a/contrib/pzstd/utils/Portability.h
+++ b/contrib/pzstd/utils/Portability.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h
index 6e217eaa1..33b6f4bce 100644
--- a/contrib/pzstd/utils/Range.h
+++ b/contrib/pzstd/utils/Range.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h
index 9bdc32ff2..b0dce8eed 100644
--- a/contrib/pzstd/utils/ResourcePool.h
+++ b/contrib/pzstd/utils/ResourcePool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ScopeGuard.h b/contrib/pzstd/utils/ScopeGuard.h
index 3ae81da88..83f1f9d6a 100644
--- a/contrib/pzstd/utils/ScopeGuard.h
+++ b/contrib/pzstd/utils/ScopeGuard.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h
index 50eed836c..658508473 100644
--- a/contrib/pzstd/utils/ThreadPool.h
+++ b/contrib/pzstd/utils/ThreadPool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h
index a0bb58c85..d2818f83b 100644
--- a/contrib/pzstd/utils/WorkQueue.h
+++ b/contrib/pzstd/utils/WorkQueue.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/BufferTest.cpp b/contrib/pzstd/utils/test/BufferTest.cpp
index c1a67a57e..029e5b1f7 100644
--- a/contrib/pzstd/utils/test/BufferTest.cpp
+++ b/contrib/pzstd/utils/test/BufferTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/RangeTest.cpp b/contrib/pzstd/utils/test/RangeTest.cpp
index bc2a8e73d..bd8c2f31d 100644
--- a/contrib/pzstd/utils/test/RangeTest.cpp
+++ b/contrib/pzstd/utils/test/RangeTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
index 542c265ec..b4f9329f2 100644
--- a/contrib/pzstd/utils/test/ResourcePoolTest.cpp
+++ b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ScopeGuardTest.cpp b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
index b5e0d00cb..c7b2017d8 100644
--- a/contrib/pzstd/utils/test/ScopeGuardTest.cpp
+++ b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
index b4656d7bd..df64bb56f 100644
--- a/contrib/pzstd/utils/test/ThreadPoolTest.cpp
+++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp
index 70f923e5d..4973729b6 100644
--- a/contrib/pzstd/utils/test/WorkQueueTest.cpp
+++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/Makefile b/contrib/recovery/Makefile
index fa500af19..aa46e85c3 100644
--- a/contrib/recovery/Makefile
+++ b/contrib/recovery/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2019-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/recover_directory.c b/contrib/recovery/recover_directory.c
index 749ca7df6..3d1104c29 100644
--- a/contrib/recovery/recover_directory.c
+++ b/contrib/recovery/recover_directory.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile
index 07ec19070..3177b3a02 100644
--- a/contrib/seekable_format/examples/Makefile
+++ b/contrib/seekable_format/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_compression.c b/contrib/seekable_format/examples/parallel_compression.c
index 04347138b..50358ecca 100644
--- a/contrib/seekable_format/examples/parallel_compression.c
+++ b/contrib/seekable_format/examples/parallel_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_processing.c b/contrib/seekable_format/examples/parallel_processing.c
index 8f3096aa2..e6e9d9688 100644
--- a/contrib/seekable_format/examples/parallel_processing.c
+++ b/contrib/seekable_format/examples/parallel_processing.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c
index 9e1588a3d..9b1205e3b 100644
--- a/contrib/seekable_format/examples/seekable_compression.c
+++ b/contrib/seekable_format/examples/seekable_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression.c b/contrib/seekable_format/examples/seekable_decompression.c
index 19c997041..154c3bd51 100644
--- a/contrib/seekable_format/examples/seekable_decompression.c
+++ b/contrib/seekable_format/examples/seekable_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression_mem.c b/contrib/seekable_format/examples/seekable_decompression_mem.c
index f1bdb9d33..1ecaa766c 100644
--- a/contrib/seekable_format/examples/seekable_decompression_mem.c
+++ b/contrib/seekable_format/examples/seekable_decompression_mem.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile
index ab5dda696..64394519b 100644
--- a/contrib/seekable_format/tests/Makefile
+++ b/contrib/seekable_format/tests/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstd_seekable_compression_format.md b/contrib/seekable_format/zstd_seekable_compression_format.md
index 8cb06af79..f86735c15 100644
--- a/contrib/seekable_format/zstd_seekable_compression_format.md
+++ b/contrib/seekable_format/zstd_seekable_compression_format.md
@@ -2,7 +2,7 @@
 
 ### Notices
 
-Copyright (c) 2017-present Meta Platforms, Inc. and affiliates.
+Copyright (c) 2017-2022 Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c
index 02f19a129..f5e7d4854 100644
--- a/contrib/seekable_format/zstdseek_compress.c
+++ b/contrib/seekable_format/zstdseek_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index 73c54271d..43d59a781 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seqBench/Makefile b/contrib/seqBench/Makefile
index 63c872097..6e5f735f3 100644
--- a/contrib/seqBench/Makefile
+++ b/contrib/seqBench/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) 2018-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 4b2330994..1eff9f684 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -3,7 +3,7 @@ Zstandard Compression Format
 
 ### Notices
 
-Copyright (c) 2016-present Yann Collet, Meta Platforms, Inc. and affiliates.
+Copyright (c) 2016-2022 Yann Collet, Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc
index 76de608d8..56e92e8d8 100644
--- a/programs/windres/zstd.rc
+++ b/programs/windres/zstd.rc
@@ -36,7 +36,7 @@ BEGIN
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Meta Platforms, Inc. and affiliates."
+            VALUE "LegalCopyright", "Copyright (c) 2013-2022, Yann Collet, Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 0ca4c3595..7512f8e97 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/gzcompatibility.h b/zlibWrapper/gzcompatibility.h
index 68d29090c..fffa4bbc9 100644
--- a/zlibWrapper/gzcompatibility.h
+++ b/zlibWrapper/gzcompatibility.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index e001629bf..fdddbb774 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index 2a34763e4..3ec483ba9 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

From 5d693cc38cd7866c525ef6101c35f20e15514a6b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 20 Dec 2022 12:49:47 -0500
Subject: [PATCH 061/197] Coalesce Almost All Copyright Notices to Standard
 Phrasing

```
for f in $(find . \( -path ./.git -o -path ./tests/fuzz/corpora -o -path ./tests/regression/data-cache -o -path ./tests/regression/cache \) -prune -o -type f); do sed -i '/Copyright .* \(Yann Collet\)\|\(Meta Platforms\)/ s/Copyright .*/Copyright (c) Meta Platforms, Inc. and affiliates./' $f; done

git checkout HEAD -- build/VS2010/libzstd-dll/libzstd-dll.rc build/VS2010/zstd/zstd.rc tests/test-license.py contrib/linux-kernel/test/include/linux/xxhash.h examples/streaming_compression_thread_pool.c lib/legacy/zstd_v0*.c lib/legacy/zstd_v0*.h
nano ./programs/windres/zstd.rc
nano ./build/VS2010/zstd/zstd.rc
nano ./build/VS2010/libzstd-dll/libzstd-dll.rc
```
---
 LICENSE                                                   | 8 ++++----
 Makefile                                                  | 2 +-
 build/VS2010/libzstd-dll/libzstd-dll.rc                   | 2 +-
 build/VS2010/zstd/zstd.rc                                 | 2 +-
 build/cmake/CMakeLists.txt                                | 2 +-
 build/cmake/contrib/CMakeLists.txt                        | 2 +-
 build/cmake/contrib/gen_html/CMakeLists.txt               | 2 +-
 build/cmake/contrib/pzstd/CMakeLists.txt                  | 2 +-
 build/cmake/lib/CMakeLists.txt                            | 2 +-
 build/cmake/programs/CMakeLists.txt                       | 2 +-
 build/cmake/tests/CMakeLists.txt                          | 2 +-
 build/single_file_libs/zstd-in.c                          | 2 +-
 build/single_file_libs/zstddeclib-in.c                    | 2 +-
 contrib/diagnose_corruption/Makefile                      | 2 +-
 contrib/diagnose_corruption/check_flipped_bits.c          | 2 +-
 contrib/freestanding_lib/freestanding.py                  | 2 +-
 contrib/gen_html/Makefile                                 | 2 +-
 contrib/gen_html/gen_html.cpp                             | 2 +-
 contrib/largeNbDicts/Makefile                             | 2 +-
 contrib/largeNbDicts/largeNbDicts.c                       | 2 +-
 contrib/linux-kernel/linux_zstd.h                         | 2 +-
 contrib/linux-kernel/mem.h                                | 2 +-
 contrib/linux-kernel/test/include/linux/compiler.h        | 2 +-
 contrib/linux-kernel/test/include/linux/errno.h           | 2 +-
 contrib/linux-kernel/test/include/linux/kernel.h          | 2 +-
 contrib/linux-kernel/test/include/linux/limits.h          | 2 +-
 contrib/linux-kernel/test/include/linux/math64.h          | 2 +-
 contrib/linux-kernel/test/include/linux/module.h          | 2 +-
 contrib/linux-kernel/test/include/linux/printk.h          | 2 +-
 contrib/linux-kernel/test/include/linux/stddef.h          | 2 +-
 contrib/linux-kernel/test/include/linux/swab.h            | 2 +-
 contrib/linux-kernel/test/include/linux/types.h           | 2 +-
 contrib/match_finders/zstd_edist.c                        | 2 +-
 contrib/match_finders/zstd_edist.h                        | 2 +-
 contrib/pzstd/ErrorHolder.h                               | 2 +-
 contrib/pzstd/Logging.h                                   | 2 +-
 contrib/pzstd/Makefile                                    | 2 +-
 contrib/pzstd/Options.cpp                                 | 2 +-
 contrib/pzstd/Options.h                                   | 2 +-
 contrib/pzstd/Pzstd.cpp                                   | 2 +-
 contrib/pzstd/Pzstd.h                                     | 2 +-
 contrib/pzstd/SkippableFrame.cpp                          | 2 +-
 contrib/pzstd/SkippableFrame.h                            | 2 +-
 contrib/pzstd/main.cpp                                    | 2 +-
 contrib/pzstd/test/OptionsTest.cpp                        | 2 +-
 contrib/pzstd/test/PzstdTest.cpp                          | 2 +-
 contrib/pzstd/test/RoundTrip.h                            | 2 +-
 contrib/pzstd/test/RoundTripTest.cpp                      | 2 +-
 contrib/pzstd/utils/Buffer.h                              | 2 +-
 contrib/pzstd/utils/FileSystem.h                          | 2 +-
 contrib/pzstd/utils/Likely.h                              | 2 +-
 contrib/pzstd/utils/Portability.h                         | 2 +-
 contrib/pzstd/utils/Range.h                               | 2 +-
 contrib/pzstd/utils/ResourcePool.h                        | 2 +-
 contrib/pzstd/utils/ScopeGuard.h                          | 2 +-
 contrib/pzstd/utils/ThreadPool.h                          | 2 +-
 contrib/pzstd/utils/WorkQueue.h                           | 2 +-
 contrib/pzstd/utils/test/BufferTest.cpp                   | 2 +-
 contrib/pzstd/utils/test/RangeTest.cpp                    | 2 +-
 contrib/pzstd/utils/test/ResourcePoolTest.cpp             | 2 +-
 contrib/pzstd/utils/test/ScopeGuardTest.cpp               | 2 +-
 contrib/pzstd/utils/test/ThreadPoolTest.cpp               | 2 +-
 contrib/pzstd/utils/test/WorkQueueTest.cpp                | 2 +-
 contrib/recovery/Makefile                                 | 2 +-
 contrib/recovery/recover_directory.c                      | 2 +-
 contrib/seekable_format/examples/Makefile                 | 2 +-
 contrib/seekable_format/examples/parallel_compression.c   | 2 +-
 contrib/seekable_format/examples/parallel_processing.c    | 2 +-
 contrib/seekable_format/examples/seekable_compression.c   | 2 +-
 contrib/seekable_format/examples/seekable_decompression.c | 2 +-
 .../seekable_format/examples/seekable_decompression_mem.c | 2 +-
 contrib/seekable_format/tests/Makefile                    | 2 +-
 .../seekable_format/zstd_seekable_compression_format.md   | 2 +-
 contrib/seekable_format/zstdseek_compress.c               | 2 +-
 contrib/seekable_format/zstdseek_decompress.c             | 2 +-
 contrib/seqBench/Makefile                                 | 2 +-
 doc/educational_decoder/Makefile                          | 2 +-
 doc/zstd_compression_format.md                            | 2 +-
 examples/Makefile                                         | 2 +-
 examples/common.h                                         | 2 +-
 examples/dictionary_compression.c                         | 2 +-
 examples/dictionary_decompression.c                       | 2 +-
 examples/multiple_simple_compression.c                    | 2 +-
 examples/multiple_streaming_compression.c                 | 2 +-
 examples/simple_compression.c                             | 2 +-
 examples/simple_decompression.c                           | 2 +-
 examples/streaming_compression.c                          | 2 +-
 examples/streaming_decompression.c                        | 2 +-
 examples/streaming_memory_usage.c                         | 2 +-
 lib/Makefile                                              | 2 +-
 lib/common/bitstream.h                                    | 2 +-
 lib/common/compiler.h                                     | 2 +-
 lib/common/debug.c                                        | 2 +-
 lib/common/debug.h                                        | 2 +-
 lib/common/entropy_common.c                               | 2 +-
 lib/common/error_private.c                                | 2 +-
 lib/common/error_private.h                                | 2 +-
 lib/common/fse.h                                          | 2 +-
 lib/common/fse_decompress.c                               | 2 +-
 lib/common/huf.h                                          | 2 +-
 lib/common/mem.h                                          | 2 +-
 lib/common/pool.c                                         | 2 +-
 lib/common/pool.h                                         | 2 +-
 lib/common/xxhash.c                                       | 2 +-
 lib/common/xxhash.h                                       | 2 +-
 lib/common/zstd_common.c                                  | 2 +-
 lib/common/zstd_internal.h                                | 2 +-
 lib/compress/clevels.h                                    | 2 +-
 lib/compress/fse_compress.c                               | 2 +-
 lib/compress/hist.c                                       | 2 +-
 lib/compress/hist.h                                       | 2 +-
 lib/compress/huf_compress.c                               | 2 +-
 lib/compress/zstd_compress.c                              | 2 +-
 lib/compress/zstd_compress_internal.h                     | 2 +-
 lib/compress/zstd_compress_literals.c                     | 2 +-
 lib/compress/zstd_compress_literals.h                     | 2 +-
 lib/compress/zstd_compress_sequences.c                    | 2 +-
 lib/compress/zstd_compress_sequences.h                    | 2 +-
 lib/compress/zstd_compress_superblock.c                   | 2 +-
 lib/compress/zstd_compress_superblock.h                   | 2 +-
 lib/compress/zstd_cwksp.h                                 | 2 +-
 lib/compress/zstd_double_fast.c                           | 2 +-
 lib/compress/zstd_double_fast.h                           | 2 +-
 lib/compress/zstd_fast.c                                  | 2 +-
 lib/compress/zstd_fast.h                                  | 2 +-
 lib/compress/zstd_lazy.c                                  | 2 +-
 lib/compress/zstd_lazy.h                                  | 2 +-
 lib/compress/zstd_ldm.c                                   | 2 +-
 lib/compress/zstd_ldm.h                                   | 2 +-
 lib/compress/zstd_ldm_geartab.h                           | 2 +-
 lib/compress/zstd_opt.c                                   | 2 +-
 lib/compress/zstd_opt.h                                   | 2 +-
 lib/compress/zstdmt_compress.c                            | 2 +-
 lib/compress/zstdmt_compress.h                            | 2 +-
 lib/decompress/huf_decompress.c                           | 2 +-
 lib/decompress/zstd_ddict.c                               | 2 +-
 lib/decompress/zstd_ddict.h                               | 2 +-
 lib/decompress/zstd_decompress.c                          | 2 +-
 lib/decompress/zstd_decompress_block.c                    | 2 +-
 lib/decompress/zstd_decompress_block.h                    | 2 +-
 lib/decompress/zstd_decompress_internal.h                 | 2 +-
 lib/deprecated/zbuff.h                                    | 2 +-
 lib/deprecated/zbuff_common.c                             | 2 +-
 lib/deprecated/zbuff_compress.c                           | 2 +-
 lib/deprecated/zbuff_decompress.c                         | 2 +-
 lib/dictBuilder/cover.c                                   | 2 +-
 lib/dictBuilder/zdict.c                                   | 2 +-
 lib/dll/example/Makefile                                  | 2 +-
 lib/legacy/zstd_legacy.h                                  | 2 +-
 lib/libzstd.mk                                            | 2 +-
 lib/libzstd.pc.in                                         | 2 +-
 lib/zdict.h                                               | 2 +-
 lib/zstd.h                                                | 2 +-
 lib/zstd_errors.h                                         | 2 +-
 programs/Makefile                                         | 2 +-
 programs/benchfn.c                                        | 2 +-
 programs/benchfn.h                                        | 2 +-
 programs/benchzstd.c                                      | 2 +-
 programs/benchzstd.h                                      | 2 +-
 programs/datagen.c                                        | 2 +-
 programs/datagen.h                                        | 2 +-
 programs/dibio.c                                          | 2 +-
 programs/dibio.h                                          | 2 +-
 programs/fileio.c                                         | 2 +-
 programs/fileio.h                                         | 2 +-
 programs/platform.h                                       | 2 +-
 programs/timefn.c                                         | 2 +-
 programs/timefn.h                                         | 2 +-
 programs/util.c                                           | 2 +-
 programs/util.h                                           | 2 +-
 programs/windres/verrsrc.h                                | 2 +-
 programs/windres/zstd.rc                                  | 4 ++--
 programs/zstdcli.c                                        | 2 +-
 tests/DEPRECATED-test-zstd-speed.py                       | 2 +-
 tests/Makefile                                            | 2 +-
 tests/bigdict.c                                           | 2 +-
 tests/checkTag.c                                          | 2 +-
 tests/datagencli.c                                        | 2 +-
 tests/decodecorpus.c                                      | 2 +-
 tests/fullbench.c                                         | 2 +-
 tests/fuzz/block_decompress.c                             | 2 +-
 tests/fuzzer.c                                            | 2 +-
 tests/invalidDictionaries.c                               | 2 +-
 tests/legacy.c                                            | 2 +-
 tests/longmatch.c                                         | 2 +-
 tests/paramgrill.c                                        | 2 +-
 tests/poolTests.c                                         | 2 +-
 tests/roundTripCrash.c                                    | 2 +-
 tests/test-zstd-versions.py                               | 2 +-
 tests/zstreamtest.c                                       | 2 +-
 zlibWrapper/examples/zwrapbench.c                         | 2 +-
 zlibWrapper/gzcompatibility.h                             | 2 +-
 zlibWrapper/zstd_zlibwrapper.c                            | 2 +-
 zlibWrapper/zstd_zlibwrapper.h                            | 2 +-
 194 files changed, 198 insertions(+), 198 deletions(-)

diff --git a/LICENSE b/LICENSE
index d2bbd709b..75800288c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -2,7 +2,7 @@ BSD License
 
 For Zstandard software
 
-Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates. All rights reserved.
+Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
@@ -14,9 +14,9 @@ are permitted provided that the following conditions are met:
    this list of conditions and the following disclaimer in the documentation
    and/or other materials provided with the distribution.
 
- * Neither the name Facebook nor the names of its contributors may be used to
-   endorse or promote products derived from this software without specific
-   prior written permission.
+ * Neither the name Facebook, nor Meta, nor the names of its contributors may
+   be used to endorse or promote products derived from this software without
+   specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
diff --git a/Makefile b/Makefile
index 641feb869..10fbe47f7 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.rc b/build/VS2010/libzstd-dll/libzstd-dll.rc
index 80d3a306c..13e8746ff 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.rc
+++ b/build/VS2010/libzstd-dll/libzstd-dll.rc
@@ -36,7 +36,7 @@ BEGIN
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "libzstd.dll"
-            VALUE "LegalCopyright", "Copyright (c) 2013-2022, Meta Platforms, Inc."
+            VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "libzstd.dll"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/build/VS2010/zstd/zstd.rc b/build/VS2010/zstd/zstd.rc
index 57b150d92..a2118c2df 100644
--- a/build/VS2010/zstd/zstd.rc
+++ b/build/VS2010/zstd/zstd.rc
@@ -36,7 +36,7 @@ BEGIN
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-2022, Meta Platforms, Inc."
+            VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index c41861ad0..c7453e811 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt
index c4684007f..8df2a17b3 100644
--- a/build/cmake/contrib/CMakeLists.txt
+++ b/build/cmake/contrib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt
index 1f945a477..d1ff6c64b 100644
--- a/build/cmake/contrib/gen_html/CMakeLists.txt
+++ b/build/cmake/contrib/gen_html/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt
index fa419a6cd..f7098fa0f 100644
--- a/build/cmake/contrib/pzstd/CMakeLists.txt
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 13f21dafd..b804b20d2 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index b94dad3e7..58d998e42 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 7f6477abc..53e0e7b17 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -1,6 +1,6 @@
 # ################################################################
 # zstd - Makefile
-# Copyright (C) Yann Collet 2014-2022
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # BSD license
diff --git a/build/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c
index c22b8cd25..e6fca9e4a 100644
--- a/build/single_file_libs/zstd-in.c
+++ b/build/single_file_libs/zstd-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/build/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c
index 218e5a698..8d9c1f54b 100644
--- a/build/single_file_libs/zstddeclib-in.c
+++ b/build/single_file_libs/zstddeclib-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/Makefile b/contrib/diagnose_corruption/Makefile
index 7fee63284..ecc9e6395 100644
--- a/contrib/diagnose_corruption/Makefile
+++ b/contrib/diagnose_corruption/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/check_flipped_bits.c b/contrib/diagnose_corruption/check_flipped_bits.c
index e76e99d72..09ddd4674 100644
--- a/contrib/diagnose_corruption/check_flipped_bits.c
+++ b/contrib/diagnose_corruption/check_flipped_bits.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index 6a26d1d14..df6983245 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) 2021-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/Makefile b/contrib/gen_html/Makefile
index 64e1e2cac..26e870399 100644
--- a/contrib/gen_html/Makefile
+++ b/contrib/gen_html/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp
index 83c5994e6..1da5879ad 100644
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Przemyslaw Skibinski, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile
index 2f514e406..40734e62e 100644
--- a/contrib/largeNbDicts/Makefile
+++ b/contrib/largeNbDicts/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index 13216035a..519468506 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/linux_zstd.h b/contrib/linux-kernel/linux_zstd.h
index 716079adf..f109d49f4 100644
--- a/contrib/linux-kernel/linux_zstd.h
+++ b/contrib/linux-kernel/linux_zstd.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/mem.h b/contrib/linux-kernel/mem.h
index 8dfd20684..a7231822b 100644
--- a/contrib/linux-kernel/mem.h
+++ b/contrib/linux-kernel/mem.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h
index 859d5d860..988ce4a20 100644
--- a/contrib/linux-kernel/test/include/linux/compiler.h
+++ b/contrib/linux-kernel/test/include/linux/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h
index c55da480c..b4bdcba0e 100644
--- a/contrib/linux-kernel/test/include/linux/errno.h
+++ b/contrib/linux-kernel/test/include/linux/errno.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h
index f3c42d440..a4d791cd1 100644
--- a/contrib/linux-kernel/test/include/linux/kernel.h
+++ b/contrib/linux-kernel/test/include/linux/kernel.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h
index da9f5d528..574aa7b34 100644
--- a/contrib/linux-kernel/test/include/linux/limits.h
+++ b/contrib/linux-kernel/test/include/linux/limits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h
index f834c0c41..7f6713e73 100644
--- a/contrib/linux-kernel/test/include/linux/math64.h
+++ b/contrib/linux-kernel/test/include/linux/math64.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h
index 77285dae6..06ef56f9e 100644
--- a/contrib/linux-kernel/test/include/linux/module.h
+++ b/contrib/linux-kernel/test/include/linux/module.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h
index ca84b2868..92a25278e 100644
--- a/contrib/linux-kernel/test/include/linux/printk.h
+++ b/contrib/linux-kernel/test/include/linux/printk.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h
index 8760df1c8..15c7408fc 100644
--- a/contrib/linux-kernel/test/include/linux/stddef.h
+++ b/contrib/linux-kernel/test/include/linux/stddef.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h
index d6f48db66..2b48b434c 100644
--- a/contrib/linux-kernel/test/include/linux/swab.h
+++ b/contrib/linux-kernel/test/include/linux/swab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h
index 8610dcaed..b413db6f9 100644
--- a/contrib/linux-kernel/test/include/linux/types.h
+++ b/contrib/linux-kernel/test/include/linux/types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.c b/contrib/match_finders/zstd_edist.c
index ab89145ad..d685cdd9e 100644
--- a/contrib/match_finders/zstd_edist.c
+++ b/contrib/match_finders/zstd_edist.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.h b/contrib/match_finders/zstd_edist.h
index 328a02ccc..c739e2abc 100644
--- a/contrib/match_finders/zstd_edist.h
+++ b/contrib/match_finders/zstd_edist.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/ErrorHolder.h b/contrib/pzstd/ErrorHolder.h
index 0b0456b32..2c2797ede 100644
--- a/contrib/pzstd/ErrorHolder.h
+++ b/contrib/pzstd/ErrorHolder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h
index 2c797b927..84a08d200 100644
--- a/contrib/pzstd/Logging.h
+++ b/contrib/pzstd/Logging.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index d6bd19050..830053cd7 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index 2462736ae..a77824edb 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h
index 1220984dc..92c18a350 100644
--- a/contrib/pzstd/Options.h
+++ b/contrib/pzstd/Options.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 5f90e5a8c..67b941991 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h
index 5c47f4998..3645e5942 100644
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.cpp b/contrib/pzstd/SkippableFrame.cpp
index c866da4ce..3bea4eb65 100644
--- a/contrib/pzstd/SkippableFrame.cpp
+++ b/contrib/pzstd/SkippableFrame.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.h b/contrib/pzstd/SkippableFrame.h
index 78b859173..817415e92 100644
--- a/contrib/pzstd/SkippableFrame.h
+++ b/contrib/pzstd/SkippableFrame.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp
index bc8f8caf6..422b4a56a 100644
--- a/contrib/pzstd/main.cpp
+++ b/contrib/pzstd/main.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp
index 1ac50b803..91e39750d 100644
--- a/contrib/pzstd/test/OptionsTest.cpp
+++ b/contrib/pzstd/test/OptionsTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp
index 5aa329aec..75453f5f6 100644
--- a/contrib/pzstd/test/PzstdTest.cpp
+++ b/contrib/pzstd/test/PzstdTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h
index 2834e9a1e..f777622a3 100644
--- a/contrib/pzstd/test/RoundTrip.h
+++ b/contrib/pzstd/test/RoundTrip.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp
index 22c59f4fb..c37646d1c 100644
--- a/contrib/pzstd/test/RoundTripTest.cpp
+++ b/contrib/pzstd/test/RoundTripTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Buffer.h b/contrib/pzstd/utils/Buffer.h
index 3b1bbd6ec..a85f770ba 100644
--- a/contrib/pzstd/utils/Buffer.h
+++ b/contrib/pzstd/utils/Buffer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h
index a33813f91..8d57d05f0 100644
--- a/contrib/pzstd/utils/FileSystem.h
+++ b/contrib/pzstd/utils/FileSystem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Likely.h b/contrib/pzstd/utils/Likely.h
index cc5a363ae..52243a64e 100644
--- a/contrib/pzstd/utils/Likely.h
+++ b/contrib/pzstd/utils/Likely.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Portability.h b/contrib/pzstd/utils/Portability.h
index 55db32383..ef1f86e51 100644
--- a/contrib/pzstd/utils/Portability.h
+++ b/contrib/pzstd/utils/Portability.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h
index 33b6f4bce..0fd8f9f86 100644
--- a/contrib/pzstd/utils/Range.h
+++ b/contrib/pzstd/utils/Range.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h
index b0dce8eed..7c4bb6235 100644
--- a/contrib/pzstd/utils/ResourcePool.h
+++ b/contrib/pzstd/utils/ResourcePool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ScopeGuard.h b/contrib/pzstd/utils/ScopeGuard.h
index 83f1f9d6a..911fd9842 100644
--- a/contrib/pzstd/utils/ScopeGuard.h
+++ b/contrib/pzstd/utils/ScopeGuard.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h
index 658508473..a087d7c1c 100644
--- a/contrib/pzstd/utils/ThreadPool.h
+++ b/contrib/pzstd/utils/ThreadPool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h
index d2818f83b..d7947b814 100644
--- a/contrib/pzstd/utils/WorkQueue.h
+++ b/contrib/pzstd/utils/WorkQueue.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/BufferTest.cpp b/contrib/pzstd/utils/test/BufferTest.cpp
index 029e5b1f7..58bf08dcd 100644
--- a/contrib/pzstd/utils/test/BufferTest.cpp
+++ b/contrib/pzstd/utils/test/BufferTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/RangeTest.cpp b/contrib/pzstd/utils/test/RangeTest.cpp
index bd8c2f31d..8b7dee271 100644
--- a/contrib/pzstd/utils/test/RangeTest.cpp
+++ b/contrib/pzstd/utils/test/RangeTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
index b4f9329f2..750ee084b 100644
--- a/contrib/pzstd/utils/test/ResourcePoolTest.cpp
+++ b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ScopeGuardTest.cpp b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
index c7b2017d8..0f77cdf38 100644
--- a/contrib/pzstd/utils/test/ScopeGuardTest.cpp
+++ b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
index df64bb56f..a01052e60 100644
--- a/contrib/pzstd/utils/test/ThreadPoolTest.cpp
+++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp
index 4973729b6..16600bb60 100644
--- a/contrib/pzstd/utils/test/WorkQueueTest.cpp
+++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/Makefile b/contrib/recovery/Makefile
index aa46e85c3..be6ea4b0e 100644
--- a/contrib/recovery/Makefile
+++ b/contrib/recovery/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/recover_directory.c b/contrib/recovery/recover_directory.c
index 3d1104c29..b9bd7ab49 100644
--- a/contrib/recovery/recover_directory.c
+++ b/contrib/recovery/recover_directory.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile
index 3177b3a02..fcc04587c 100644
--- a/contrib/seekable_format/examples/Makefile
+++ b/contrib/seekable_format/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_compression.c b/contrib/seekable_format/examples/parallel_compression.c
index 50358ecca..0ec9fbd20 100644
--- a/contrib/seekable_format/examples/parallel_compression.c
+++ b/contrib/seekable_format/examples/parallel_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_processing.c b/contrib/seekable_format/examples/parallel_processing.c
index e6e9d9688..b1709db77 100644
--- a/contrib/seekable_format/examples/parallel_processing.c
+++ b/contrib/seekable_format/examples/parallel_processing.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c
index 9b1205e3b..182b46f64 100644
--- a/contrib/seekable_format/examples/seekable_compression.c
+++ b/contrib/seekable_format/examples/seekable_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression.c b/contrib/seekable_format/examples/seekable_decompression.c
index 154c3bd51..2c4f3ba0c 100644
--- a/contrib/seekable_format/examples/seekable_decompression.c
+++ b/contrib/seekable_format/examples/seekable_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression_mem.c b/contrib/seekable_format/examples/seekable_decompression_mem.c
index 1ecaa766c..44a06fbbf 100644
--- a/contrib/seekable_format/examples/seekable_decompression_mem.c
+++ b/contrib/seekable_format/examples/seekable_decompression_mem.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile
index 64394519b..a81f2229f 100644
--- a/contrib/seekable_format/tests/Makefile
+++ b/contrib/seekable_format/tests/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstd_seekable_compression_format.md b/contrib/seekable_format/zstd_seekable_compression_format.md
index f86735c15..7bd0790e8 100644
--- a/contrib/seekable_format/zstd_seekable_compression_format.md
+++ b/contrib/seekable_format/zstd_seekable_compression_format.md
@@ -2,7 +2,7 @@
 
 ### Notices
 
-Copyright (c) 2017-2022 Meta Platforms, Inc. and affiliates.
+Copyright (c) Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c
index f5e7d4854..113f6f99c 100644
--- a/contrib/seekable_format/zstdseek_compress.c
+++ b/contrib/seekable_format/zstdseek_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index 43d59a781..7128b9d22 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seqBench/Makefile b/contrib/seqBench/Makefile
index 6e5f735f3..e7f08a42c 100644
--- a/contrib/seqBench/Makefile
+++ b/contrib/seqBench/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-2022, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index 7560e36fc..f6deeb13d 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 1eff9f684..042dbe89a 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -3,7 +3,7 @@ Zstandard Compression Format
 
 ### Notices
 
-Copyright (c) 2016-2022 Yann Collet, Meta Platforms, Inc. and affiliates.
+Copyright (c) Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/examples/Makefile b/examples/Makefile
index 5fb4c7138..31f52d357 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/common.h b/examples/common.h
index 621332e11..4873e877a 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c
index a878498a5..83edc1cad 100644
--- a/examples/dictionary_compression.c
+++ b/examples/dictionary_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c
index 77c406645..e6c999964 100644
--- a/examples/dictionary_decompression.c
+++ b/examples/dictionary_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index cf9113838..bf77ca133 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index ca9ef5a5d..b12ad03dc 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_compression.c b/examples/simple_compression.c
index 5b1c3b301..7c880725f 100644
--- a/examples/simple_compression.c
+++ b/examples/simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index 1b0aca6db..f499156f6 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index 4fb7f282b..ed0a3a69c 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index ad630e92c..95fa11227 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c
index aac62668c..957acb61a 100644
--- a/examples/streaming_memory_usage.c
+++ b/examples/streaming_memory_usage.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/Makefile b/lib/Makefile
index f5b2d652a..a4cf61ab1 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index c27eb09ab..db1b4cf13 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * bitstream
  * Part of FSE library
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index bbc26a98d..e423f2467 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/debug.c b/lib/common/debug.c
index b98168d12..ebf7bfccf 100644
--- a/lib/common/debug.c
+++ b/lib/common/debug.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/debug.h b/lib/common/debug.h
index 5805a9280..0e9817ea6 100644
--- a/lib/common/debug.h
+++ b/lib/common/debug.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index b3d9d0b1b..4bf8242ff 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Common functions of New Generation Entropy library
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index bbd380188..baf9b0b0c 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index f82775add..325daad40 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/fse.h b/lib/common/fse.h
index 212f4697d..2ef8df459 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
index fb641e71d..ca894eb14 100644
--- a/lib/common/fse_decompress.c
+++ b/lib/common/fse_decompress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 2905fe8a4..679493bfa 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/mem.h b/lib/common/mem.h
index 7a55f6fc1..98dd47a04 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/pool.c b/lib/common/pool.c
index c37d7077a..3466b85b8 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/pool.h b/lib/common/pool.h
index b5078f3cb..eb22ff509 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index efd010803..fd237c906 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -1,6 +1,6 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ *  Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - xxHash homepage: https://cyan4973.github.io/xxHash/
diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index 6889cad14..b8b73290b 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -1,6 +1,6 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ *  Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - xxHash homepage: https://cyan4973.github.io/xxHash/
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index fce41243d..320855247 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 8463334ce..c9c97b7e4 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/clevels.h b/lib/compress/clevels.h
index 99172c3c7..c18da465f 100644
--- a/lib/compress/clevels.h
+++ b/lib/compress/clevels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 065620bc8..6a9aa11e5 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.c b/lib/compress/hist.c
index 9338276f0..e2fb431f0 100644
--- a/lib/compress/hist.c
+++ b/lib/compress/hist.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.h b/lib/compress/hist.h
index 8b6a1b696..887896b81 100644
--- a/lib/compress/hist.h
+++ b/lib/compress/hist.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index a0fe97c67..bbe6421fb 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 2686711a6..f4a3c8264 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 4d138993b..e4bb2f537 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index 939bcaa96..e97c485ab 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h
index b6befb24c..e5da204c8 100644
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c
index cfd7e5f32..8872d4d35 100644
--- a/lib/compress/zstd_compress_sequences.c
+++ b/lib/compress/zstd_compress_sequences.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.h b/lib/compress/zstd_compress_sequences.h
index 5f4421adf..4a3a05da9 100644
--- a/lib/compress/zstd_compress_sequences.h
+++ b/lib/compress/zstd_compress_sequences.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index fa3bf02de..1971ec646 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.h b/lib/compress/zstd_compress_superblock.h
index 6d16e2b33..8e494f0d5 100644
--- a/lib/compress/zstd_compress_superblock.h
+++ b/lib/compress/zstd_compress_superblock.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index cab928bc6..70a359a27 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index 82f41af69..0ad88ffc7 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h
index 66c8badf6..6f0047c4b 100644
--- a/lib/compress/zstd_double_fast.h
+++ b/lib/compress/zstd_double_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index bddd15997..5f2c6a2ed 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h
index 2032081bb..9e4236b47 100644
--- a/lib/compress/zstd_fast.h
+++ b/lib/compress/zstd_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 8605cf02c..810bf011c 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index e2f9d6c99..c24f1c794 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 698f166d6..3d74ff19e 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index a2d2deef7..f147021d2 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm_geartab.h b/lib/compress/zstd_ldm_geartab.h
index 8087c54cd..ef34bc5c9 100644
--- a/lib/compress/zstd_ldm_geartab.h
+++ b/lib/compress/zstd_ldm_geartab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 668212df9..6e08dc8c7 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index e690e5d2d..342e5a311 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index e3fc54685..7a2c71720 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index ff1fa0ef4..ed4dc0e99 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 3ee35c470..b3ba5926b 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index ec451d37b..3647ad0a7 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h
index b3f394b9c..c4ca8877a 100644
--- a/lib/decompress/zstd_ddict.h
+++ b/lib/decompress/zstd_ddict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 373733373..f00ef3a67 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index d8b2814fa..94728a165 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_block.h b/lib/decompress/zstd_decompress_block.h
index e59f9d3e2..67791dbc3 100644
--- a/lib/decompress/zstd_decompress_block.h
+++ b/lib/decompress/zstd_decompress_block.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 2252c7d5d..32685f2d3 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index 69933ebda..a968245b3 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_common.c b/lib/deprecated/zbuff_common.c
index 1d4310511..5a2f2db35 100644
--- a/lib/deprecated/zbuff_common.c
+++ b/lib/deprecated/zbuff_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_compress.c b/lib/deprecated/zbuff_compress.c
index bf1051ff9..1d8682150 100644
--- a/lib/deprecated/zbuff_compress.c
+++ b/lib/deprecated/zbuff_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
index 7795fc369..51159ef53 100644
--- a/lib/deprecated/zbuff_decompress.c
+++ b/lib/deprecated/zbuff_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 92852c1f0..1ade83987 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 375a17649..f9a5d678c 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dll/example/Makefile b/lib/dll/example/Makefile
index be996fad6..86cf6906e 100644
--- a/lib/dll/example/Makefile
+++ b/lib/dll/example/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index cc2165b31..9f53d4cbd 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/libzstd.mk b/lib/libzstd.mk
index 152e538db..5e11d5d29 100644
--- a/lib/libzstd.mk
+++ b/lib/libzstd.mk
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in
index 33184297c..d5cc0270c 100644
--- a/lib/libzstd.pc.in
+++ b/lib/libzstd.pc.in
@@ -1,5 +1,5 @@
 #   ZSTD - standard compression algorithm
-#   Copyright (C) 2014-2016, Yann Collet, Facebook
+#   Copyright (c) Meta Platforms, Inc. and affiliates.
 #   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
 prefix=@PREFIX@
diff --git a/lib/zdict.h b/lib/zdict.h
index 8f52e2a86..8bed583f1 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd.h b/lib/zstd.h
index af9e2e2d7..d23fa0208 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 2c97a48d7..57a50d024 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/Makefile b/programs/Makefile
index c626aae5d..b79b60b4f 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.c b/programs/benchfn.c
index d94377734..1b5c383ea 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.h b/programs/benchfn.h
index 65409b4f5..1bd93d135 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index bf5279013..285e401ef 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index c19d03ce9..aa683dfc2 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.c b/programs/datagen.c
index 8baf40a5e..ddc690bb1 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.h b/programs/datagen.h
index c088e71d9..ca7270006 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.c b/programs/dibio.c
index a45c65dad..b21338cd8 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.h b/programs/dibio.h
index b8e859f5a..a96104c36 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.c b/programs/fileio.c
index 3d532ae33..418de6fd7 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.h b/programs/fileio.h
index c3264534d..e37398ded 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/platform.h b/programs/platform.h
index f7d558595..18a3587bf 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.c b/programs/timefn.c
index d6f4749b2..7005b2696 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.h b/programs/timefn.h
index b92b5605e..438a8de07 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/util.c b/programs/util.c
index 6f73348b3..de8a27739 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/util.h b/programs/util.h
index 90b7afe91..d022c0014 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/windres/verrsrc.h b/programs/windres/verrsrc.h
index 9d06960b1..61b1f3ddc 100644
--- a/programs/windres/verrsrc.h
+++ b/programs/windres/verrsrc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc
index 56e92e8d8..a2118c2df 100644
--- a/programs/windres/zstd.rc
+++ b/programs/windres/zstd.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet, Meta Platforms, Inc. and affiliates."
+            VALUE "CompanyName", "Meta Platforms, Inc."
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-2022, Yann Collet, Meta Platforms, Inc. and affiliates."
+            VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index f177ee23b..b3d06e809 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/DEPRECATED-test-zstd-speed.py b/tests/DEPRECATED-test-zstd-speed.py
index 568b059b7..71d75b8cc 100755
--- a/tests/DEPRECATED-test-zstd-speed.py
+++ b/tests/DEPRECATED-test-zstd-speed.py
@@ -2,7 +2,7 @@
 # THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
 
 # ################################################################
-# Copyright (c) Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/Makefile b/tests/Makefile
index a8f2401ee..baf616acb 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,6 @@
 
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/bigdict.c b/tests/bigdict.c
index b8fe92162..ff2bb2d70 100644
--- a/tests/bigdict.c
+++ b/tests/bigdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/checkTag.c b/tests/checkTag.c
index 38e09ddc9..26871ed0f 100644
--- a/tests/checkTag.c
+++ b/tests/checkTag.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/datagencli.c b/tests/datagencli.c
index 0d291952f..09ec5e9ae 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index c53e8a3fd..df52ae6a7 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 097c97f13..25c6d2b40 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c
index 8307e6f3c..e4767b3f5 100644
--- a/tests/fuzz/block_decompress.c
+++ b/tests/fuzz/block_decompress.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index edbb613ec..ae460fefc 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c
index 0841cca59..66caa9e99 100644
--- a/tests/invalidDictionaries.c
+++ b/tests/invalidDictionaries.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/legacy.c b/tests/legacy.c
index e0a360101..3be386495 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/longmatch.c b/tests/longmatch.c
index 499cb5541..36b10df76 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 224643c84..5dfd895e9 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 3d2d4ce03..9e62722bf 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c
index bfcd8cfe2..77411cd49 100644
--- a/tests/roundTripCrash.c
+++ b/tests/roundTripCrash.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index 4995e32d4..7117c1952 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -2,7 +2,7 @@
 """Test zstd interoperability between versions"""
 
 # ################################################################
-# Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 2830f25f7..30c5215ae 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 7512f8e97..dd8472d39 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/gzcompatibility.h b/zlibWrapper/gzcompatibility.h
index fffa4bbc9..9d11b984f 100644
--- a/zlibWrapper/gzcompatibility.h
+++ b/zlibWrapper/gzcompatibility.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index fdddbb774..f31cb59db 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index 3ec483ba9..230bf8411 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2022, Przemyslaw Skibinski, Yann Collet, Meta Platforms, Inc. and affiliates.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

From e4018c4e7fa8dd90d65262a78ed9b54469e2a73b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 19 Dec 2022 18:08:35 -0800
Subject: [PATCH 062/197] [docs] Clarify dictionary loading documentation

Reinforce that loading a new dictionary clears the current dictionary.
Except for the multiple-ddict mode.
---
 lib/zstd.h | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index d23fa0208..8bf032623 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1022,8 +1022,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
  *           meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
- *           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+ *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ *           until parameters are reset, a new dictionary is loaded, or the dictionary
+ *           is explicitly invalidated by loading a NULL dictionary.
  *  Note 2 : Loading a dictionary involves building tables.
  *           It's also a CPU consuming operation, with non-negligible impact on latency.
  *           Tables are dependent on compression parameters, and for this reason,
@@ -1036,7 +1037,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
 /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
- *  Reference a prepared dictionary, to be used for all next compressed frames.
+ *  Reference a prepared dictionary, to be used for all future compressed frames.
  *  Note that compression parameters are enforced from within CDict,
  *  and supersede any compression parameter previously set within CCtx.
  *  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@@ -1071,9 +1072,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                  const void* prefix, size_t prefixSize);
 
 /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
- *  Create an internal DDict from dict buffer,
- *  to be used to decompress next frames.
- *  The dictionary remains valid for all future frames, until explicitly invalidated.
+ *  Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ *  The dictionary remains valid for all future frames, until explicitly invalidated, or
+ *  a new dictionary is loaded.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
  *            meaning "return to no-dictionary mode".
@@ -1097,9 +1098,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
  *  The memory for the table is allocated on the first call to refDDict, and can be
  *  freed with ZSTD_freeDCtx().
  *
+ *  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ *  will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Note 1 : Currently, only one dictionary can be managed.
- *           Referencing a new dictionary effectively "discards" any previous one.
  *  Special: referencing a NULL DDict means "return to no-dictionary mode".
  *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
  */

From e3f2c8b11c09c6dfd0b86c03bb36906750235ce1 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 20 Dec 2022 14:17:59 -0500
Subject: [PATCH 063/197] Pin actions/checkout Dependency to Specific Commit
 Hash

It's a bit silly, because if we can't trust GitHub, what are we doing here?
But OSSF complains about it, so let's fix it.
---
 .github/workflows/dev-long-tests.yml          | 44 ++++++-------
 .github/workflows/dev-short-tests.yml         | 62 +++++++++----------
 .../workflows/publish-release-artifacts.yml   |  2 +-
 .github/workflows/scorecards.yml              |  2 +-
 4 files changed, 55 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index dc1c5b355..9cc58b872 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -13,7 +13,7 @@ jobs:
   make-all:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make all
       run: make all
 
@@ -24,7 +24,7 @@ jobs:
       DEVNULLRIGHTS: 1
       READFROMBLOCKDEVICE: 1
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make test
       run: make test
 
@@ -32,28 +32,28 @@ jobs:
   make-test-osx:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: OS-X test
       run: make test # make -c lib all doesn't work because of the fact that it's not a tty
 
   no-intrinsics-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: no intrinsics fuzztest
       run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
 
   tsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
   ubsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: undefined behavior sanitizer zstreamtest
       run: CC=clang make uasan-test-zstream
 
@@ -61,7 +61,7 @@ jobs:
   tsan-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: thread sanitizer fuzztest
       run: CC=clang make tsan-fuzztest
 
@@ -69,7 +69,7 @@ jobs:
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: gcc-8 + ASan + UBSan + Test Zstd
       # See https://askubuntu.com/a/1428822
       run: |
@@ -81,14 +81,14 @@ jobs:
   clang-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Test Zstd
       run: CC=clang make -j uasan-test-zstd </dev/null V=1
 
   gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -102,7 +102,7 @@ jobs:
   gcc-8-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: gcc-8 + ASan + UBSan + Fuzz Test
       # See https://askubuntu.com/a/1428822
       run: |
@@ -114,14 +114,14 @@ jobs:
   clang-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Fuzz Test
       run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
@@ -131,7 +131,7 @@ jobs:
   clang-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
@@ -141,28 +141,28 @@ jobs:
   asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ASan + UBSan + Regression Test
       run: make -j uasanregressiontest
 
   clang-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Regression Test
       run: CC=clang make -j uasanregressiontest
 
   msan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: MSan + Regression Test
       run: make -j msanregressiontest
 
   clang-msan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + MSan + Fuzz Test
       run: |
         sudo apt-get -qqq update
@@ -173,7 +173,7 @@ jobs:
   clang-msan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + MSan + Test Zstd
       run: |
         sudo apt-get update
@@ -183,7 +183,7 @@ jobs:
   armfuzz:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: Qemu ARM emulation + Fuzz Test
         run: |
           sudo apt-get -qqq update
@@ -193,7 +193,7 @@ jobs:
   valgrind-fuzz-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: valgrind + fuzz test stack mode    # ~ 7mn
       shell: 'script -q -e -c "bash {0}"'
       run: |
@@ -213,7 +213,7 @@ jobs:
           { compiler: gcc, platform: x64, action: test, script: ""},
         ]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Mingw long test
       run: |
         $env:PATH_ORIGINAL = $env:PATH
diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index d0d0940fa..6b159d997 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -14,21 +14,21 @@ jobs:
   linux-kernel:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: linux kernel, library + build + test
       run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
 
   benchmarking:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make benchmarking
       run: make benchmarking
 
   check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make check on 32-bit
       run: |
         sudo apt update
@@ -38,7 +38,7 @@ jobs:
   check-x32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
       env:
         CHECK_CONSTRAINED_MEM: true
@@ -50,7 +50,7 @@ jobs:
   gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: gcc-7 + libzstdmt compilation
       # See https://askubuntu.com/a/1428822
       run: |
@@ -67,7 +67,7 @@ jobs:
   cmake-build-and-test-check:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: cmake build and test check
       run: |
         FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
@@ -78,7 +78,7 @@ jobs:
   cpp-gnu90-c99-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: C++, gnu90 and c99 compatibility
       run: |
         make cxxtest
@@ -92,7 +92,7 @@ jobs:
   mingw-cross-compilation:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: mingw cross-compilation
       run: |
         # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@@ -103,7 +103,7 @@ jobs:
   armbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ARM Build Test
       run: |
         sudo apt-get -qqq update
@@ -113,7 +113,7 @@ jobs:
   bourne-shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Bourne shell compatibility (shellcheck)
       run: |
         wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
@@ -123,7 +123,7 @@ jobs:
   zlib-wrapper:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: zlib wrapper test
       run: |
         sudo apt-get -qqq update
@@ -134,7 +134,7 @@ jobs:
   lz4-threadpool-libs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: LZ4, thread pool, and libs build testslib wrapper test
       run: |
         make lz4install
@@ -148,7 +148,7 @@ jobs:
   gcc-make-tests-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Make all, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -158,7 +158,7 @@ jobs:
   gcc-8-make:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: gcc-8 build
         # See https://askubuntu.com/a/1428822
         run: |
@@ -170,7 +170,7 @@ jobs:
   implicit-fall-through:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: -Wimplicit-fallthrough build
         run: |
           make clean
@@ -181,7 +181,7 @@ jobs:
   meson-linux:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: Install packages
         run: |
           sudo apt-get update
@@ -205,7 +205,7 @@ jobs:
   meson-windows:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: Install packages
         run: pip install --pre meson
       - name: Initialize the MSVC dev command prompt
@@ -234,7 +234,7 @@ jobs:
             flags: "-A Win32"
           - generator: "MinGW Makefiles"
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
       uses: microsoft/setup-msbuild@v1.1.3
     - name: Build
@@ -253,7 +253,7 @@ jobs:
         platform: [x64, Win32]
         configuration: [Debug, Release]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
       uses: microsoft/setup-msbuild@v1.1.3
     - name: Build
@@ -272,7 +272,7 @@ jobs:
 #        platform: [x64, Win32]
 #        configuration: [Debug, Release]
 #    steps:
-#    - uses: actions/checkout@v3
+#    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
 #    - name: Add MSBuild to PATH
 #      uses: microsoft/setup-msbuild@v1.1.3
 #    - name: Build
@@ -287,7 +287,7 @@ jobs:
   libzstd-size:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: libzstd size test
       run: |
         make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
@@ -298,7 +298,7 @@ jobs:
   minimal-decompressor-macros:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: minimal decompressor macros
       run: |
         make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -313,7 +313,7 @@ jobs:
   dynamic-bmi2:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: dynamic bmi2 tests
       run: |
         make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@@ -325,7 +325,7 @@ jobs:
   test-variants:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make all variants & validate
       run: |
         make -j -C programs allVariants MOREFLAGS=-O0
@@ -351,7 +351,7 @@ jobs:
       XCC: ${{ matrix.xcc }}
       XEMU: ${{ matrix.xemu }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: apt update & install
       run: |
         sudo apt-get update
@@ -404,7 +404,7 @@ jobs:
           { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
         ]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Mingw short test
       run: |
         ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
@@ -437,7 +437,7 @@ jobs:
         platform: [x64, Win32]
         configuration: [Release]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
       uses: microsoft/setup-msbuild@v1.1.3
     - name: Build and run tests
@@ -457,7 +457,7 @@ jobs:
   intel-cet-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Build Zstd
       run: |
         make -j zstd V=1
@@ -478,7 +478,7 @@ jobs:
     container:
       image: debian:testing
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: Install dependencies
         run: |
           apt -y update
@@ -493,7 +493,7 @@ jobs:
   versions-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Versions Compatibility Test
       run: |
         make -C tests versionsTest
@@ -517,7 +517,7 @@ jobs:
 #        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
 #        sudo apt-get update
 #        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@v3
+#    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
 #    - name: make check
 #      run: |
 #        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml
index 4a70fb8c6..0903f0d3c 100644
--- a/.github/workflows/publish-release-artifacts.yml
+++ b/.github/workflows/publish-release-artifacts.yml
@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
 
       - name: Archive
         env:
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 2a6a42ab5..b45fc0851 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -27,7 +27,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.0.0
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
         with:
           persist-credentials: false
 

From 482689b995bd6afb4a7b335bb80b9e60f501e3c2 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 20 Dec 2022 12:27:38 -0800
Subject: [PATCH 064/197] huf log speed optimization: unidirectional scan of
 logs + break when regressing

---
 .github/dependabot.yml                        |  6 ++++
 .github/workflows/dev-long-tests.yml          |  4 +++
 .github/workflows/dev-short-tests.yml         | 14 +++++---
 .../workflows/publish-release-artifacts.yml   |  2 +-
 .github/workflows/scorecards.yml              |  2 +-
 build/meson/meson.build                       | 14 ++------
 contrib/linux-kernel/test/test.c              |  6 +++-
 lib/common/huf.h                              |  2 +-
 lib/compress/huf_compress.c                   | 32 +++++--------------
 lib/compress/zstd_compress.c                  | 15 +++++----
 lib/compress/zstd_lazy.c                      |  3 +-
 programs/fileio.c                             | 16 ++++++++--
 programs/util.c                               |  9 ++++--
 programs/zstdcli.c                            |  6 ++--
 tests/cli-tests/compression/window-resize.sh  |  9 ++++++
 .../window-resize.sh.stderr.ignore            |  0
 .../compression/window-resize.sh.stdout.glob  |  3 ++
 tests/fuzz/zstd_helpers.c                     |  2 +-
 tests/fuzzer.c                                | 22 ++++++-------
 tests/regression/config.c                     |  4 +--
 tests/zstreamtest.c                           |  2 +-
 21 files changed, 96 insertions(+), 77 deletions(-)
 create mode 100644 .github/dependabot.yml
 create mode 100755 tests/cli-tests/compression/window-resize.sh
 create mode 100644 tests/cli-tests/compression/window-resize.sh.stderr.ignore
 create mode 100644 tests/cli-tests/compression/window-resize.sh.stdout.glob

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..5ace4600a
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index aadc0ab54..ac34c7165 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -71,7 +71,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: gcc-8 + ASan + UBSan + Test Zstd
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -qqq update
         make gcc8install
         CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
@@ -102,7 +104,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: gcc-8 + ASan + UBSan + Fuzz Test
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -qqq update
         make gcc8install
         CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 37186b5c9..08df1c07c 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -52,7 +52,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: gcc-7 + libzstdmt compilation
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -qqq update
         make gcc7install
         CC=gcc-7 CFLAGS=-Werror make -j all
@@ -158,7 +160,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: gcc-8 build
+        # See https://askubuntu.com/a/1428822
         run: |
+          echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
           sudo apt-get -qqq update
           make gcc8install
           CC=gcc-8 CFLAGS="-Werror" make -j all
@@ -187,7 +191,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
+      uses: microsoft/setup-msbuild@v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
@@ -206,7 +210,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
+      uses: microsoft/setup-msbuild@v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@@ -225,7 +229,7 @@ jobs:
 #    steps:
 #    - uses: actions/checkout@v3
 #    - name: Add MSBuild to PATH
-#      uses: microsoft/setup-msbuild@v1.0.2
+#      uses: microsoft/setup-msbuild@v1.1.3
 #    - name: Build
 #      working-directory: ${{env.GITHUB_WORKSPACE}}
 #      run: >
@@ -271,7 +275,7 @@ jobs:
 
   qemu-consistency:
     name: QEMU ${{ matrix.name }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
       matrix:
@@ -376,7 +380,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
+      uses: microsoft/setup-msbuild@v1.1.3
     - name: Build and run tests
       working-directory: ${{env.GITHUB_WORKSPACE}}
       env:
diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml
index b5a3ac689..4a70fb8c6 100644
--- a/.github/workflows/publish-release-artifacts.yml
+++ b/.github/workflows/publish-release-artifacts.yml
@@ -68,7 +68,7 @@ jobs:
           fi
 
       - name: Publish
-        uses: skx/github-action-publish-binaries@release-1.3
+        uses: skx/github-action-publish-binaries@release-2.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 2eae47494..1c19b48b2 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -32,7 +32,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3
+        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6
         with:
           results_file: results.sarif
           results_format: sarif
diff --git a/build/meson/meson.build b/build/meson/meson.build
index f264760a3..98d556aa3 100644
--- a/build/meson/meson.build
+++ b/build/meson/meson.build
@@ -23,7 +23,9 @@ project('zstd',
     # so this isn't safe
     #'werror=true'
   ],
-  version: 'DUMMY',
+  version: run_command(
+    find_program('GetZstdLibraryVersion.py'), '../../lib/zstd.h',
+    check: true).stdout().strip(),
   meson_version: '>=0.48.0')
 
 cc = meson.get_compiler('c')
@@ -45,16 +47,6 @@ compiler_msvc = 'msvc'
 
 zstd_version = meson.project_version()
 
-zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
-GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true)
-r = run_command(GetZstdLibraryVersion_py, zstd_h_file)
-if r.returncode() == 0
-  zstd_version = r.stdout().strip()
-  message('Project version is now: @0@'.format(zstd_version))
-else
-  error('Cannot find project version in @0@'.format(zstd_h_file))
-endif
-
 zstd_libversion = zstd_version
 
 # =============================================================================
diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c
index 6cd1730bb..67d248e0c 100644
--- a/contrib/linux-kernel/test/test.c
+++ b/contrib/linux-kernel/test/test.c
@@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) {
   asm volatile("" : "+r"(x));
 }
 
+static void __attribute__((noinline)) fill_stack(void) {
+  memset(g_stack, 0x33, 8192);
+}
+
 static void __attribute__((noinline)) set_stack(void) {
 
   char stack[8192];
   g_stack = stack;
-  memset(g_stack, 0x33, 8192);
   use(g_stack);
 }
 
@@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) {
 
 static void test_stack_usage(test_data_t const *data) {
   set_stack();
+  fill_stack();
   test_f2fs();
   test_btrfs(data);
   test_decompress_unzstd(data);
diff --git a/lib/common/huf.h b/lib/common/huf.h
index dee99da77..595b2f6db 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -173,7 +173,7 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
-#define HUF_OPTIMAL_DEPTH_THRESHOLD 3
+#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
 typedef enum {
    HUF_depth_fast, /** Use heuristic to find the table depth**/
    HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index a6a5a257c..eb2cb7b94 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1253,14 +1253,6 @@ unsigned HUF_minTableLog(unsigned symbolCardinality)
     return minBitsSymbols;
 }
 
-#define ESTIMATE_TOTAL_SIZE(huffLog) {\
-            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, huffLog, workSpace, wkspSize);\
-            if (ERR_isError(maxBits)) continue;\
-            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);\
-            if (ERR_isError(hSize)) continue;\
-            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;\
-            }\
-
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
 {
     unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
@@ -1273,28 +1265,20 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
         const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
         size_t optSize = ((size_t) ~0);
-        unsigned optLogGuess = optLog;
 
         if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/
 
-        /* Search below estimate log until size increases */
-        for (; optLogGuess >= minTableLog; optLogGuess--) {
-            ESTIMATE_TOTAL_SIZE(optLogGuess);
+        /* Search until size increases */
+        for (unsigned optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
 
-            if (newSize > optSize) {
-                break;
-            }
-            optSize = newSize;
-            optLog = optLogGuess;
-        }
+            if (ERR_isError(maxBits)) continue;
 
-        if (optSize < ((size_t) ~0)) {
-            return optLog;
-        }
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
 
-        /* Search above estimate log until size increases */
-        for (; optLogGuess <= maxTableLog; optLogGuess++) {
-            ESTIMATE_TOTAL_SIZE(optLogGuess);
+            if (ERR_isError(hSize)) continue;
+
+            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
 
             if (newSize > optSize) {
                 break;
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index adf1f6e7a..0069a7b1b 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -265,9 +265,9 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
     return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
 }
 
-/* Returns 1 if compression parameters are such that we should
+/* Returns ZSTD_ps_enable if compression parameters are such that we should
  * enable long distance matching (wlog >= 27, strategy >= btopt).
- * Returns 0 otherwise.
+ * Returns ZSTD_ps_disable otherwise.
  */
 static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
                                  const ZSTD_compressionParameters* const cParams) {
@@ -482,8 +482,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         return bounds;
 
     case ZSTD_c_enableLongDistanceMatching:
-        bounds.lowerBound = 0;
-        bounds.upperBound = 1;
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
     case ZSTD_c_ldmHashLog:
@@ -854,6 +854,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         return (size_t)CCtxParams->enableDedicatedDictSearch;
 
     case ZSTD_c_enableLongDistanceMatching :
+        BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
         CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
         return CCtxParams->ldmParams.enableLdm;
 
@@ -1096,7 +1097,7 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
 
 size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
+    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
     RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                     "Can't set pledgedSrcSize when not in init stage.");
     cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
@@ -1369,8 +1370,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     }
 
     /* resize windowLog if input is small enough, to use less memory */
-    if ( (srcSize < maxWindowResize)
-      && (dictSize < maxWindowResize) )  {
+    if ( (srcSize <= maxWindowResize)
+      && (dictSize <= maxWindowResize) )  {
         U32 const tSize = (U32)(srcSize + dictSize);
         static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
         U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 2ee9cf630..d5a7b5cbf 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1078,7 +1078,7 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
     }
 # endif /* ZSTD_ARCH_ARM_NEON */
     /* SWAR */
-    {   const size_t chunkSize = sizeof(size_t);
+    {   const int chunkSize = sizeof(size_t);
         const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
         const size_t xFF = ~((size_t)0);
         const size_t x01 = xFF / 0xFF;
@@ -2123,7 +2123,6 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
 size_t ZSTD_compressBlock_lazy2_extDict_row(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize)
-
 {
     return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
 }
diff --git a/programs/fileio.c b/programs/fileio.c
index 96cf602a3..e80d37011 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -2749,6 +2749,7 @@ typedef struct {
     int numSkippableFrames;
     int decompUnavailable;
     int usesCheck;
+    BYTE checksum[4];
     U32 nbFiles;
     unsigned dictID;
 } fileInfo_t;
@@ -2843,8 +2844,8 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
                     int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
                     if (contentChecksumFlag) {
                         info->usesCheck = 1;
-                        ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0,
-                                info_frame_error, "Error: could not skip past checksum");
+                        ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4,
+                                info_frame_error, "Error: could not read checksum");
                 }   }
                 info->numActualFrames++;
             }
@@ -2936,7 +2937,16 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel)
                     (unsigned long long)info->decompressedSize);
             DISPLAYOUT("Ratio: %.4f\n", ratio);
         }
-        DISPLAYOUT("Check: %s\n", checkString);
+
+        if (info->usesCheck && info->numActualFrames == 1) {
+            DISPLAYOUT("Check: %s %02x%02x%02x%02x\n", checkString,
+                info->checksum[3], info->checksum[2],
+                info->checksum[1], info->checksum[0]
+            );
+        } else {
+            DISPLAYOUT("Check: %s\n", checkString);
+        }
+
         DISPLAYOUT("\n");
     }
 }
diff --git a/programs/util.c b/programs/util.c
index a3af26211..63b3ae176 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -569,7 +569,7 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
         for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
             size_t const curLen = strlen(table2->fileNames[idx2]);
             memcpy(buf+pos, table2->fileNames[idx2], curLen);
-            assert(newTableIdx <= newTable->tableSize);
+            assert(newTableIdx < newTable->tableSize);
             newTable->fileNames[newTableIdx] = buf+pos;
             pos += curLen+1;
     }   }
@@ -693,8 +693,11 @@ static int UTIL_prepareFileList(const char *dirName,
                 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
                 assert(newListSize >= 0);
                 *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
-                *bufEnd = *bufStart + newListSize;
-                if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+                if (*bufStart != NULL) {
+                    *bufEnd = *bufStart + newListSize;
+                } else {
+                    free(path); closedir(dir); return 0;
+                }
             }
             if (*bufStart + *pos + pathLength < *bufEnd) {
                 memcpy(*bufStart + *pos, path, pathLength + 1);  /* with final \0 */
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 583c8a591..362f320a9 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -216,7 +216,6 @@ static void usage_advanced(const char* programName)
     DISPLAYOUT("\n");
     DISPLAYOUT("Advanced compression options :\n");
     DISPLAYOUT("     --ultra           enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
-    DISPLAYOUT("     --long[=#]        enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
     DISPLAYOUT("     --fast[=#]        switch to very fast compression levels (default: %u)\n", 1);
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
@@ -224,9 +223,9 @@ static void usage_advanced(const char* programName)
         DISPLAYOUT("     --no-name         do not store original filename when compressing\n");
     }
 #endif
-    DISPLAYOUT("     --adapt           dynamically adapt compression level to I/O conditions\n");
-    DISPLAYOUT("     --[no-]row-match-finder :  force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
+    DISPLAYOUT("     --long[=#]        enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
     DISPLAYOUT("     --patch-from=FILE :  specify the file to be used as a reference point for zstd's diff engine. \n");
+    DISPLAYOUT("     --adapt           dynamically adapt compression level to I/O conditions\n");
 # ifdef ZSTD_MULTITHREAD
     DISPLAYOUT("  -T#                  spawn # compression threads (default: 1, 0==# cores) \n");
     DISPLAYOUT("  -B#                  select size of each job (default: 0==automatic) \n");
@@ -240,6 +239,7 @@ static void usage_advanced(const char* programName)
     DISPLAYOUT("     --target-compressed-block-size=# :  generate compressed block of approximately targeted size \n");
     DISPLAYOUT("     --no-dictID       don't write dictID into header (dictionary compression only)\n");
     DISPLAYOUT("     --[no-]compress-literals :  force (un)compressed literals\n");
+    DISPLAYOUT("     --[no-]row-match-finder :  force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
 
     DISPLAYOUT("     --format=zstd     compress files to the .zst format (default)\n");
 #ifdef ZSTD_GZCOMPRESS
diff --git a/tests/cli-tests/compression/window-resize.sh b/tests/cli-tests/compression/window-resize.sh
new file mode 100755
index 000000000..3b5e6fe24
--- /dev/null
+++ b/tests/cli-tests/compression/window-resize.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+datagen -g1G > file
+zstd --long=31 -1 --single-thread --no-content-size -f file
+zstd -l -v file.zst
+
+# We want to ignore stderr (its outputting "*** zstd command line interface
+# 64-bits v1.5.3, by Yann Collet ***")
+
+rm file file.zst
diff --git a/tests/cli-tests/compression/window-resize.sh.stderr.ignore b/tests/cli-tests/compression/window-resize.sh.stderr.ignore
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/cli-tests/compression/window-resize.sh.stdout.glob b/tests/cli-tests/compression/window-resize.sh.stdout.glob
new file mode 100644
index 000000000..313d216e1
--- /dev/null
+++ b/tests/cli-tests/compression/window-resize.sh.stdout.glob
@@ -0,0 +1,3 @@
+...
+Window Size: 1.000 GiB (1073741824 B)
+...
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index b4a6509db..08ce70dd7 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -80,7 +80,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer);
     setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer);
     /* Select long distance matching parameters */
-    setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer);
+    setRand(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_auto, ZSTD_ps_disable, producer);
     setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer);
     setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
             ZSTD_LDM_MINMATCH_MAX, producer);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 81c2d9dba..879e537bc 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -851,7 +851,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         RDG_genBuffer(dict, size, 0.5, 0.5, seed);
         RDG_genBuffer(src, size, 0.5, 0.5, seed);
 
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
 
         ZSTD_freeCCtx(cctx);
@@ -875,7 +875,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
-            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
             CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
             cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
             CHECK_Z(cSize);
@@ -900,7 +900,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
         memcpy(dict + testSize, CNBuffer, testSize);
         for (level = 1; level <= 5; ++level) {
-            for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) {
+            for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
                 size_t cSize0;
                 XXH64_hash_t compressedChecksum0;
 
@@ -956,7 +956,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* Enable MT, LDM, and opt parser */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
 
@@ -995,7 +995,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         /* Disable content size to skip single-pass decompression. */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
@@ -1092,7 +1092,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* compress on level 1 using refPrefix and ldm */
         ZSTD_CCtx_refPrefix(cctx, dict, size);;
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1))
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
         refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
         assert(!ZSTD_isError(refPrefixLdmCompressedSize));
 
@@ -2820,7 +2820,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
             assert(cctx != NULL);
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
+            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
             {   size_t const compressedSize = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
@@ -2836,7 +2836,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
             DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
+            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
             {   size_t const result = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
@@ -3492,7 +3492,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* Enable MT, LDM, and use refPrefix() for a small dict */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
 
         CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
@@ -3686,7 +3686,7 @@ static int longUnitTests(U32 const seed, double compressibility)
 
         /* Enable checksum to verify round trip. */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
 
         /* Round trip once with ldm. */
@@ -3696,7 +3696,7 @@ static int longUnitTests(U32 const seed, double compressibility)
 
         ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
 
         /* Round trip once without ldm. */
diff --git a/tests/regression/config.c b/tests/regression/config.c
index 57cd110c6..30d0ca5e2 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -183,7 +183,7 @@ static config_t no_pledged_src_size_with_dict = {
 };
 
 static param_value_t const ldm_param_values[] = {
-    {.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
+    {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
 };
 
 static config_t ldm = {
@@ -204,7 +204,7 @@ static config_t mt = {
 
 static param_value_t const mt_ldm_param_values[] = {
     {.param = ZSTD_c_nbWorkers, .value = 2},
-    {.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
+    {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
 };
 
 static config_t mt_ldm = {
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index ce9020f12..348f72ed4 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2281,7 +2281,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
 
                 /* mess with long distance matching parameters */
                 if (bigTests) {
-                    if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
+                    if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );

From c26f348dc810ae536a63f088b380b5e6b341b45a Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 20 Dec 2022 12:43:46 -0800
Subject: [PATCH 065/197] fix CI errors

---
 lib/compress/huf_compress.c  |  3 +-
 tests/regression/results.csv | 60 ++++++++++++++++++------------------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index eb2cb7b94..d7755e0f2 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1265,11 +1265,12 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
         const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
         size_t optSize = ((size_t) ~0);
+        unsigned optLogGuess;
 
         if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/
 
         /* Search until size increases */
-        for (unsigned optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+        for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
             maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
 
             if (ERR_isError(maxBits)) continue;
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index dee7d5721..ffa9b22e0 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -12,9 +12,9 @@ silesia.tar,                        level 7,                            compress
 silesia.tar,                        level 9,                            compress simple,                    4552899
 silesia.tar,                        level 13,                           compress simple,                    4502956
 silesia.tar,                        level 16,                           compress simple,                    4360527
-silesia.tar,                        level 19,                           compress simple,                    4266970
+silesia.tar,                        level 19,                           compress simple,                    4267014
 silesia.tar,                        uncompressed literals,              compress simple,                    4854086
-silesia.tar,                        uncompressed literals optimal,      compress simple,                    4266970
+silesia.tar,                        uncompressed literals optimal,      compress simple,                    4267014
 silesia.tar,                        huffman literals,                   compress simple,                    6179047
 github.tar,                         level -5,                           compress simple,                    52115
 github.tar,                         level -3,                           compress simple,                    45678
@@ -29,9 +29,9 @@ github.tar,                         level 7,                            compress
 github.tar,                         level 9,                            compress simple,                    36760
 github.tar,                         level 13,                           compress simple,                    35501
 github.tar,                         level 16,                           compress simple,                    40471
-github.tar,                         level 19,                           compress simple,                    32149
+github.tar,                         level 19,                           compress simple,                    32127
 github.tar,                         uncompressed literals,              compress simple,                    38831
-github.tar,                         uncompressed literals optimal,      compress simple,                    32149
+github.tar,                         uncompressed literals optimal,      compress simple,                    32127
 github.tar,                         huffman literals,                   compress simple,                    42560
 silesia,                            level -5,                           compress cctx,                      6857372
 silesia,                            level -3,                           compress cctx,                      6503412
@@ -46,7 +46,7 @@ silesia,                            level 7,                            compress
 silesia,                            level 9,                            compress cctx,                      4543018
 silesia,                            level 13,                           compress cctx,                      4493990
 silesia,                            level 16,                           compress cctx,                      4359864
-silesia,                            level 19,                           compress cctx,                      4296438
+silesia,                            level 19,                           compress cctx,                      4296439
 silesia,                            long distance mode,                 compress cctx,                      4842075
 silesia,                            multithreaded,                      compress cctx,                      4842075
 silesia,                            multithreaded long distance mode,   compress cctx,                      4842075
@@ -55,7 +55,7 @@ silesia,                            small hash log,                     compress
 silesia,                            small chain log,                    compress cctx,                      4912197
 silesia,                            explicit params,                    compress cctx,                      4794052
 silesia,                            uncompressed literals,              compress cctx,                      4842075
-silesia,                            uncompressed literals optimal,      compress cctx,                      4296438
+silesia,                            uncompressed literals optimal,      compress cctx,                      4296439
 silesia,                            huffman literals,                   compress cctx,                      6172202
 silesia,                            multithreaded with advanced params, compress cctx,                      4842075
 github,                             level -5,                           compress cctx,                      204407
@@ -110,7 +110,7 @@ silesia,                            level 7,                            zstdcli,
 silesia,                            level 9,                            zstdcli,                            4543066
 silesia,                            level 13,                           zstdcli,                            4494038
 silesia,                            level 16,                           zstdcli,                            4359912
-silesia,                            level 19,                           zstdcli,                            4296486
+silesia,                            level 19,                           zstdcli,                            4296487
 silesia,                            long distance mode,                 zstdcli,                            4833785
 silesia,                            multithreaded,                      zstdcli,                            4842123
 silesia,                            multithreaded long distance mode,   zstdcli,                            4833785
@@ -135,7 +135,7 @@ silesia.tar,                        level 7,                            zstdcli,
 silesia.tar,                        level 9,                            zstdcli,                            4552903
 silesia.tar,                        level 13,                           zstdcli,                            4502960
 silesia.tar,                        level 16,                           zstdcli,                            4360531
-silesia.tar,                        level 19,                           zstdcli,                            4266974
+silesia.tar,                        level 19,                           zstdcli,                            4267018
 silesia.tar,                        no source size,                     zstdcli,                            4854160
 silesia.tar,                        long distance mode,                 zstdcli,                            4845745
 silesia.tar,                        multithreaded,                      zstdcli,                            4854164
@@ -213,7 +213,7 @@ github.tar,                         level 13,                           zstdcli,
 github.tar,                         level 13 with dict,                 zstdcli,                            37134
 github.tar,                         level 16,                           zstdcli,                            40475
 github.tar,                         level 16 with dict,                 zstdcli,                            33378
-github.tar,                         level 19,                           zstdcli,                            32153
+github.tar,                         level 19,                           zstdcli,                            32131
 github.tar,                         level 19 with dict,                 zstdcli,                            32716
 github.tar,                         no source size,                     zstdcli,                            38832
 github.tar,                         no source size with dict,           zstdcli,                            38004
@@ -249,7 +249,7 @@ silesia,                            level 12 row 1,                     advanced
 silesia,                            level 12 row 2,                     advanced one pass,                  4503116
 silesia,                            level 13,                           advanced one pass,                  4493990
 silesia,                            level 16,                           advanced one pass,                  4359864
-silesia,                            level 19,                           advanced one pass,                  4296438
+silesia,                            level 19,                           advanced one pass,                  4296439
 silesia,                            no source size,                     advanced one pass,                  4842075
 silesia,                            long distance mode,                 advanced one pass,                  4833710
 silesia,                            multithreaded,                      advanced one pass,                  4842075
@@ -283,7 +283,7 @@ silesia.tar,                        level 12 row 1,                     advanced
 silesia.tar,                        level 12 row 2,                     advanced one pass,                  4513797
 silesia.tar,                        level 13,                           advanced one pass,                  4502956
 silesia.tar,                        level 16,                           advanced one pass,                  4360527
-silesia.tar,                        level 19,                           advanced one pass,                  4266970
+silesia.tar,                        level 19,                           advanced one pass,                  4267014
 silesia.tar,                        no source size,                     advanced one pass,                  4854086
 silesia.tar,                        long distance mode,                 advanced one pass,                  4840452
 silesia.tar,                        multithreaded,                      advanced one pass,                  4854160
@@ -527,7 +527,7 @@ github.tar,                         level 16 with dict dms,             advanced
 github.tar,                         level 16 with dict dds,             advanced one pass,                  33206
 github.tar,                         level 16 with dict copy,            advanced one pass,                  33374
 github.tar,                         level 16 with dict load,            advanced one pass,                  39081
-github.tar,                         level 19,                           advanced one pass,                  32149
+github.tar,                         level 19,                           advanced one pass,                  32127
 github.tar,                         level 19 with dict,                 advanced one pass,                  32712
 github.tar,                         level 19 with dict dms,             advanced one pass,                  32555
 github.tar,                         level 19 with dict dds,             advanced one pass,                  32555
@@ -567,7 +567,7 @@ silesia,                            level 12 row 1,                     advanced
 silesia,                            level 12 row 2,                     advanced one pass small out,        4503116
 silesia,                            level 13,                           advanced one pass small out,        4493990
 silesia,                            level 16,                           advanced one pass small out,        4359864
-silesia,                            level 19,                           advanced one pass small out,        4296438
+silesia,                            level 19,                           advanced one pass small out,        4296439
 silesia,                            no source size,                     advanced one pass small out,        4842075
 silesia,                            long distance mode,                 advanced one pass small out,        4833710
 silesia,                            multithreaded,                      advanced one pass small out,        4842075
@@ -601,7 +601,7 @@ silesia.tar,                        level 12 row 1,                     advanced
 silesia.tar,                        level 12 row 2,                     advanced one pass small out,        4513797
 silesia.tar,                        level 13,                           advanced one pass small out,        4502956
 silesia.tar,                        level 16,                           advanced one pass small out,        4360527
-silesia.tar,                        level 19,                           advanced one pass small out,        4266970
+silesia.tar,                        level 19,                           advanced one pass small out,        4267014
 silesia.tar,                        no source size,                     advanced one pass small out,        4854086
 silesia.tar,                        long distance mode,                 advanced one pass small out,        4840452
 silesia.tar,                        multithreaded,                      advanced one pass small out,        4854160
@@ -845,7 +845,7 @@ github.tar,                         level 16 with dict dms,             advanced
 github.tar,                         level 16 with dict dds,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict copy,            advanced one pass small out,        33374
 github.tar,                         level 16 with dict load,            advanced one pass small out,        39081
-github.tar,                         level 19,                           advanced one pass small out,        32149
+github.tar,                         level 19,                           advanced one pass small out,        32127
 github.tar,                         level 19 with dict,                 advanced one pass small out,        32712
 github.tar,                         level 19 with dict dms,             advanced one pass small out,        32555
 github.tar,                         level 19 with dict dds,             advanced one pass small out,        32555
@@ -885,7 +885,7 @@ silesia,                            level 12 row 1,                     advanced
 silesia,                            level 12 row 2,                     advanced streaming,                 4503116
 silesia,                            level 13,                           advanced streaming,                 4493990
 silesia,                            level 16,                           advanced streaming,                 4359864
-silesia,                            level 19,                           advanced streaming,                 4296438
+silesia,                            level 19,                           advanced streaming,                 4296439
 silesia,                            no source size,                     advanced streaming,                 4842039
 silesia,                            long distance mode,                 advanced streaming,                 4833710
 silesia,                            multithreaded,                      advanced streaming,                 4842075
@@ -919,7 +919,7 @@ silesia.tar,                        level 12 row 1,                     advanced
 silesia.tar,                        level 12 row 2,                     advanced streaming,                 4513797
 silesia.tar,                        level 13,                           advanced streaming,                 4502956
 silesia.tar,                        level 16,                           advanced streaming,                 4360527
-silesia.tar,                        level 19,                           advanced streaming,                 4266970
+silesia.tar,                        level 19,                           advanced streaming,                 4267014
 silesia.tar,                        no source size,                     advanced streaming,                 4859267
 silesia.tar,                        long distance mode,                 advanced streaming,                 4840452
 silesia.tar,                        multithreaded,                      advanced streaming,                 4854160
@@ -1163,7 +1163,7 @@ github.tar,                         level 16 with dict dms,             advanced
 github.tar,                         level 16 with dict dds,             advanced streaming,                 33206
 github.tar,                         level 16 with dict copy,            advanced streaming,                 33374
 github.tar,                         level 16 with dict load,            advanced streaming,                 39081
-github.tar,                         level 19,                           advanced streaming,                 32149
+github.tar,                         level 19,                           advanced streaming,                 32127
 github.tar,                         level 19 with dict,                 advanced streaming,                 32712
 github.tar,                         level 19 with dict dms,             advanced streaming,                 32555
 github.tar,                         level 19 with dict dds,             advanced streaming,                 32555
@@ -1195,10 +1195,10 @@ silesia,                            level 7,                            old stre
 silesia,                            level 9,                            old streaming,                      4543018
 silesia,                            level 13,                           old streaming,                      4493990
 silesia,                            level 16,                           old streaming,                      4359864
-silesia,                            level 19,                           old streaming,                      4296438
+silesia,                            level 19,                           old streaming,                      4296439
 silesia,                            no source size,                     old streaming,                      4842039
 silesia,                            uncompressed literals,              old streaming,                      4842075
-silesia,                            uncompressed literals optimal,      old streaming,                      4296438
+silesia,                            uncompressed literals optimal,      old streaming,                      4296439
 silesia,                            huffman literals,                   old streaming,                      6172207
 silesia.tar,                        level -5,                           old streaming,                      6856523
 silesia.tar,                        level -3,                           old streaming,                      6505954
@@ -1213,10 +1213,10 @@ silesia.tar,                        level 7,                            old stre
 silesia.tar,                        level 9,                            old streaming,                      4552900
 silesia.tar,                        level 13,                           old streaming,                      4502956
 silesia.tar,                        level 16,                           old streaming,                      4360527
-silesia.tar,                        level 19,                           old streaming,                      4266970
+silesia.tar,                        level 19,                           old streaming,                      4267014
 silesia.tar,                        no source size,                     old streaming,                      4859267
 silesia.tar,                        uncompressed literals,              old streaming,                      4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming,                      4266970
+silesia.tar,                        uncompressed literals optimal,      old streaming,                      4267014
 silesia.tar,                        huffman literals,                   old streaming,                      6179056
 github,                             level -5,                           old streaming,                      204407
 github,                             level -5 with dict,                 old streaming,                      46718
@@ -1277,12 +1277,12 @@ github.tar,                         level 13,                           old stre
 github.tar,                         level 13 with dict,                 old streaming,                      37130
 github.tar,                         level 16,                           old streaming,                      40471
 github.tar,                         level 16 with dict,                 old streaming,                      33374
-github.tar,                         level 19,                           old streaming,                      32149
+github.tar,                         level 19,                           old streaming,                      32127
 github.tar,                         level 19 with dict,                 old streaming,                      32712
 github.tar,                         no source size,                     old streaming,                      38828
 github.tar,                         no source size with dict,           old streaming,                      38000
 github.tar,                         uncompressed literals,              old streaming,                      38831
-github.tar,                         uncompressed literals optimal,      old streaming,                      32149
+github.tar,                         uncompressed literals optimal,      old streaming,                      32127
 github.tar,                         huffman literals,                   old streaming,                      42560
 silesia,                            level -5,                           old streaming advanced,             6854744
 silesia,                            level -3,                           old streaming advanced,             6503319
@@ -1297,7 +1297,7 @@ silesia,                            level 7,                            old stre
 silesia,                            level 9,                            old streaming advanced,             4543018
 silesia,                            level 13,                           old streaming advanced,             4493990
 silesia,                            level 16,                           old streaming advanced,             4359864
-silesia,                            level 19,                           old streaming advanced,             4296438
+silesia,                            level 19,                           old streaming advanced,             4296439
 silesia,                            no source size,                     old streaming advanced,             4842039
 silesia,                            long distance mode,                 old streaming advanced,             4842075
 silesia,                            multithreaded,                      old streaming advanced,             4842075
@@ -1307,7 +1307,7 @@ silesia,                            small hash log,                     old stre
 silesia,                            small chain log,                    old streaming advanced,             4912197
 silesia,                            explicit params,                    old streaming advanced,             4795452
 silesia,                            uncompressed literals,              old streaming advanced,             4842075
-silesia,                            uncompressed literals optimal,      old streaming advanced,             4296438
+silesia,                            uncompressed literals optimal,      old streaming advanced,             4296439
 silesia,                            huffman literals,                   old streaming advanced,             6172207
 silesia,                            multithreaded with advanced params, old streaming advanced,             4842075
 silesia.tar,                        level -5,                           old streaming advanced,             6856523
@@ -1323,7 +1323,7 @@ silesia.tar,                        level 7,                            old stre
 silesia.tar,                        level 9,                            old streaming advanced,             4552900
 silesia.tar,                        level 13,                           old streaming advanced,             4502956
 silesia.tar,                        level 16,                           old streaming advanced,             4360527
-silesia.tar,                        level 19,                           old streaming advanced,             4266970
+silesia.tar,                        level 19,                           old streaming advanced,             4267014
 silesia.tar,                        no source size,                     old streaming advanced,             4859267
 silesia.tar,                        long distance mode,                 old streaming advanced,             4859271
 silesia.tar,                        multithreaded,                      old streaming advanced,             4859271
@@ -1333,7 +1333,7 @@ silesia.tar,                        small hash log,                     old stre
 silesia.tar,                        small chain log,                    old streaming advanced,             4917021
 silesia.tar,                        explicit params,                    old streaming advanced,             4806873
 silesia.tar,                        uncompressed literals,              old streaming advanced,             4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4266970
+silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4267014
 silesia.tar,                        huffman literals,                   old streaming advanced,             6179056
 silesia.tar,                        multithreaded with advanced params, old streaming advanced,             4859271
 github,                             level -5,                           old streaming advanced,             213265
@@ -1403,7 +1403,7 @@ github.tar,                         level 13,                           old stre
 github.tar,                         level 13 with dict,                 old streaming advanced,             35807
 github.tar,                         level 16,                           old streaming advanced,             40471
 github.tar,                         level 16 with dict,                 old streaming advanced,             38578
-github.tar,                         level 19,                           old streaming advanced,             32149
+github.tar,                         level 19,                           old streaming advanced,             32127
 github.tar,                         level 19 with dict,                 old streaming advanced,             32704
 github.tar,                         no source size,                     old streaming advanced,             38828
 github.tar,                         no source size with dict,           old streaming advanced,             38015
@@ -1415,7 +1415,7 @@ github.tar,                         small hash log,                     old stre
 github.tar,                         small chain log,                    old streaming advanced,             41669
 github.tar,                         explicit params,                    old streaming advanced,             41385
 github.tar,                         uncompressed literals,              old streaming advanced,             38831
-github.tar,                         uncompressed literals optimal,      old streaming advanced,             32149
+github.tar,                         uncompressed literals optimal,      old streaming advanced,             32127
 github.tar,                         huffman literals,                   old streaming advanced,             42560
 github.tar,                         multithreaded with advanced params, old streaming advanced,             38831
 github,                             level -5 with dict,                 old streaming cdict,                46718

From 651a38106095dd27ec149d64f2f6876802fca06c Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 19 Dec 2022 18:54:22 -0800
Subject: [PATCH 066/197] [build][cmake] Fix cmake with custom assembler

Tell CMake to explicitly compile our assembly as C code, because we
require it is compiled by a C compiler, and it is only enabled for
clang/gcc.

Fixes #3193.
---
 build/cmake/CMakeLists.txt     | 20 +++++++++++---------
 build/cmake/lib/CMakeLists.txt |  6 ++++++
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index c7453e811..0bffc87d9 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -8,18 +8,18 @@
 # ################################################################
 
 cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
-  
-# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. 
-# Set and use the newest cmake policies that are validated to work 
-set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") 
+
+# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
+# Set and use the newest cmake policies that are validated to work
+set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
 set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
 if("${CMAKE_MAJOR_VERSION}" LESS 3)
-  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
-elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND 
+  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
+elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
         "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
-    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
-else() 
-    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") 
+    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
+else()
+    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
 endif()
 cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
 
@@ -40,11 +40,13 @@ if( CMAKE_MAJOR_VERSION LESS 3 )
   set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
   set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
   enable_language(C)   # Main library is in C
+  enable_language(ASM) # And ASM
   enable_language(CXX) # Testing contributed code also utilizes CXX
 else()
   project(zstd
     VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
     LANGUAGES C   # Main library is in C
+              ASM # And ASM
               CXX # Testing contributed code also utilizes CXX
     )
 endif()
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index b804b20d2..30349586b 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -77,6 +77,12 @@ if (MSVC)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
 endif ()
 
+# Explicitly set the language to C for all files, including ASM files.
+# Our assembly expects to be compiled by a C compiler, and is only enabled for
+# __GNUC__ compatible compilers. Otherwise all the ASM code is disabled by
+# macros.
+set_source_files_properties(${Sources} PROPERTIES LANGUAGE C)
+
 # Split project to static and shared libraries build
 set(library_targets)
 if (ZSTD_BUILD_SHARED)

From 150aa23ef47b5b0fcab425ae3d6679f252b14e1c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 20 Dec 2022 17:10:01 -0500
Subject: [PATCH 067/197] Pin Remaining Action Dependencies (Except OSS-Fuzz)

The one that isn't pinned is the OSS-Fuzz builder and runner. They don't
offer tagged releases. I could pin to the current master commit, but I'm not
sure how desirable that is.
---
 .github/workflows/dev-long-tests.yml  | 2 +-
 .github/workflows/dev-short-tests.yml | 8 ++++----
 .github/workflows/scorecards.yml      | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index 9cc58b872..f75f61f8e 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -279,7 +279,7 @@ jobs:
         dry-run: false
         sanitizer: ${{ matrix.sanitizer }}
     - name: Upload Crash
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
       if: failure() && steps.build.outcome == 'success'
       with:
         name: ${{ matrix.sanitizer }}-artifacts
diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 6b159d997..e9aa4046b 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -236,7 +236,7 @@ jobs:
     steps:
     - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
@@ -255,7 +255,7 @@ jobs:
     steps:
     - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@@ -274,7 +274,7 @@ jobs:
 #    steps:
 #    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
 #    - name: Add MSBuild to PATH
-#      uses: microsoft/setup-msbuild@v1.1.3
+#      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
 #    - name: Build
 #      working-directory: ${{env.GITHUB_WORKSPACE}}
 #      run: >
@@ -439,7 +439,7 @@ jobs:
     steps:
     - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build and run tests
       working-directory: ${{env.GITHUB_WORKSPACE}}
       env:
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index b45fc0851..afdf4d63f 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -51,7 +51,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@v3 # tag=v3.0.0
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
         with:
           name: SARIF file
           path: results.sarif

From 678335c4f30a05857a94ac285c1d2eb6ce52cb88 Mon Sep 17 00:00:00 2001
From: Jon Palmisciano <jp@jonpalmisc.com>
Date: Tue, 20 Dec 2022 15:34:02 -0500
Subject: [PATCH 068/197] Rewrite help output to improve readability

---
 programs/zstdcli.c | 191 ++++++++++++++++++++++++---------------------
 1 file changed, 102 insertions(+), 89 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index cc6bbb935..d26df2850 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -143,29 +143,35 @@ static int exeNameMatch(const char* exeName, const char* test)
  */
 static void usage(FILE* f, const char* programName)
 {
-    DISPLAY_F(f, "Usage: %s [OPTION]... [FILE]... [-o file]\n", programName);
-    DISPLAY_F(f, "Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).\n\n");
-    DISPLAY_F(f, "  -o file              result stored into `file` (only 1 output file)\n");
-#ifndef ZSTD_NOCOMPRESS
-    DISPLAY_F(f, "  -1 .. -%d            compression level (faster .. better; default: %d)\n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
-#endif
-#ifndef ZSTD_NODECOMPRESS
-    DISPLAY_F(f, "  -d, --decompress     decompression\n");
-#endif
-    DISPLAY_F(f, "  -f, --force          disable input and output checks. Allows overwriting existing files,\n");
-    DISPLAY_F(f, "                       input from console, output to stdout, operating on links,\n");
-    DISPLAY_F(f, "                       block devices, etc. During decompression and when the output\n");
-    DISPLAY_F(f, "                       destination is stdout, pass-through unrecognized formats as-is.\n");
-    DISPLAY_F(f, "      --rm             remove source file(s) after successful de/compression\n");
-    DISPLAY_F(f, "  -k, --keep           preserve source file(s) (default) \n");
+    DISPLAY_F(f, "Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.\n\n");
+    DISPLAY_F(f, "Usage: %s [OPTIONS...] [INPUT... | -] [-o OUTPUT]\n\n", programName);
+    DISPLAY_F(f, "Options:\n");
+    DISPLAY_F(f, "  -o OUTPUT                     Write output to a single file, OUTPUT.\n");
+    DISPLAY_F(f, "  -k, --keep                    Preserve INPUT file(s). [Default] \n");
+    DISPLAY_F(f, "  --rm                          Remove INPUT file(s) after successful (de)compression.\n");
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
-        DISPLAY_F(f, "  -n, --no-name        do not store original filename when compressing\n");
+        DISPLAY_F(f, "  -n, --no-name                 Do not store original filename when compressing.\n\n");
     }
 #endif
-    DISPLAY_F(f, "  -D DICT              use DICT as Dictionary for compression or decompression\n");
-    DISPLAY_F(f, "  -h                   display usage and exit\n");
-    DISPLAY_F(f, "  -H,--help            display long help and exit\n");
+    DISPLAY_F(f, "\n");
+#ifndef ZSTD_NOCOMPRESS
+    DISPLAY_F(f, "  -#                            Desired compression level, where `#` is a number between 1 and %d;\n", ZSTDCLI_CLEVEL_MAX);
+    DISPLAY_F(f, "                                lower numbers provide faster compression, higher numbers yield\n");
+    DISPLAY_F(f, "                                better compression ratios. [Default: %d]\n\n", ZSTDCLI_CLEVEL_DEFAULT);
+#endif
+#ifndef ZSTD_NODECOMPRESS
+    DISPLAY_F(f, "  -d, --decompress              Perform decompression.\n");
+#endif
+    DISPLAY_F(f, "  -D DICT                       Use DICT as the dictionary for compression or decompression.\n\n");
+    DISPLAY_F(f, "  -f, --force                   Disable input and output checks. Allows overwriting existing files,\n");
+    DISPLAY_F(f, "                                receiving input from the console, printing ouput to STDOUT, and\n");
+    DISPLAY_F(f, "                                operating on links, block devices, etc. Unrecognized formats will be\n");
+    DISPLAY_F(f, "                                passed-through through as-is.\n\n");
+
+    DISPLAY_F(f, "  -h                            Display short usage and exit.\n");
+    DISPLAY_F(f, "  -H, --help                    Display full help and exit.\n");
+    DISPLAY_F(f, "  -V, --version                 Display the program version and exit.\n");
     DISPLAY_F(f, "\n");
 }
 
@@ -173,130 +179,137 @@ static void usage_advanced(const char* programName)
 {
     DISPLAYOUT(WELCOME_MESSAGE);
     usage(stdout, programName);
-    DISPLAYOUT("Advanced options :\n");
-    DISPLAYOUT("  -V, --version        display Version number and exit\n");
+    DISPLAYOUT("Advanced options:\n");
+    DISPLAYOUT("  -c, --stdout                  Write to STDOUT (even if it is a console) and keep the INPUT file(s).\n\n");
 
-    DISPLAYOUT("  -c, --stdout         write to standard output (even if it is the console), keep original file\n");
-
-    DISPLAYOUT("  -v, --verbose        verbose mode; specify multiple times to increase verbosity\n");
-    DISPLAYOUT("  -q, --quiet          suppress warnings; specify twice to suppress errors too\n");
-    DISPLAYOUT("      --[no-]progress  forcibly display, or never display the progress counter\n");
-    DISPLAYOUT("                       note: any (de)compressed output to terminal will mix with progress counter text\n");
+    DISPLAYOUT("  -v, --verbose                 Enable verbose output; pass multiple times to increase verbosity.\n");
+    DISPLAYOUT("  -q, --quiet                   Suppress warnings; pass twice to suppress errors.\n");
+#ifndef ZSTD_NOTRACE
+    DISPLAYOUT("  --trace LOG                   Log tracing information to LOG.\n");
+#endif
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --[no-]progress               Forcibly show/hide the progress counter. NOTE: Any (de)compressed\n");
+    DISPLAYOUT("                                output to terminal will mix with progress counter text.\n\n");
 
 #ifdef UTIL_HAS_CREATEFILELIST
-    DISPLAYOUT("  -r                   operate recursively on directories\n");
-    DISPLAYOUT("     --filelist FILE   read list of files to operate upon from FILE\n");
-    DISPLAYOUT("     --output-dir-flat DIR :  processed files are stored into DIR\n");
+    DISPLAYOUT("  -r                            Operate recursively on directories.\n");
+    DISPLAYOUT("  --filelist LIST               Read a list of files to operate on from LIST.\n");
+    DISPLAYOUT("  --output-dir-flat DIR         Store processed files in DIR.\n");
 #endif
 
 #ifdef UTIL_HAS_MIRRORFILELIST
-    DISPLAYOUT("     --output-dir-mirror DIR :  processed files are stored into DIR respecting original directory structure\n");
+    DISPLAYOUT("  --output-dir-mirror DIR       Store processed files in DIR, respecting original directory structure.\n");
 #endif
     if (AIO_supported())
-        DISPLAYOUT("     --[no-]asyncio    use asynchronous IO (default: enabled)\n");
+        DISPLAYOUT("  --[no-]asyncio                Use asynchronous IO. [Default: Enabled]\n");
 
+    DISPLAYOUT("\n");
 #ifndef ZSTD_NOCOMPRESS
-    DISPLAYOUT("     --[no-]check      during compression, add XXH64 integrity checksum to frame (default: enabled)\n");
+    DISPLAYOUT("  --[no-]check                  Add XXH64 integrity checksums during compression. [Default: Add, Validate]\n");
 #ifndef ZSTD_NODECOMPRESS
-    DISPLAYOUT("                       if specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate)\n");
+    DISPLAYOUT("                                If `-d` is present, ignore/validate checksums during decompression.\n");
 #endif
 #else
 #ifdef ZSTD_NOCOMPRESS
-    DISPLAYOUT("      --[no-]check     during decompression, ignore/validate checksums in compressed frame (default: validate)");
+    DISPLAYOUT("  --[no-]check                  Ignore/validate checksums during decompression. [Default: Validate]");
 #endif
-    DISPLAYOUT("\n");
 #endif /* ZSTD_NOCOMPRESS */
 
-#ifndef ZSTD_NOTRACE
-    DISPLAYOUT("     --trace FILE      log tracing information to FILE\n");
-#endif
-    DISPLAYOUT("     --                all arguments after \"--\" are treated as files\n");
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --                            Treat remaining arguments after `--` as files.\n");
 
 #ifndef ZSTD_NOCOMPRESS
     DISPLAYOUT("\n");
-    DISPLAYOUT("Advanced compression options :\n");
-    DISPLAYOUT("     --ultra           enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
-    DISPLAYOUT("     --fast[=#]        switch to very fast compression levels (default: %u)\n", 1);
+    DISPLAYOUT("Advanced compression options:\n");
+    DISPLAYOUT("  --ultra                       Enable levels beyond %i, up to %i; requires more memory.\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
+    DISPLAYOUT("  --fast[=#]                    Use to very fast compression levels. [Default: %u]\n", 1);
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
-        DISPLAYOUT("     --best            compatibility alias for -9 \n");
-        DISPLAYOUT("     --no-name         do not store original filename when compressing\n");
+        DISPLAYOUT("  --best                        Compatibility alias for `-9`.\n");
     }
 #endif
-    DISPLAYOUT("     --long[=#]        enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
-    DISPLAYOUT("     --patch-from=FILE :  specify the file to be used as a reference point for zstd's diff engine. \n");
-    DISPLAYOUT("     --adapt           dynamically adapt compression level to I/O conditions\n");
+    DISPLAYOUT("  --adapt                       Dynamically adapt compression level to I/O conditions.\n");
+    DISPLAYOUT("  --long[=#]                    Enable long distance matching with window log #. [Default: %u]\n", g_defaultMaxWindowLog);
+    DISPLAYOUT("  --patch-from=REF              Use REF as the reference point for Zstandard's diff engine. \n\n");
 # ifdef ZSTD_MULTITHREAD
-    DISPLAYOUT("  -T#                  spawn # compression threads (default: 1, 0==# cores) \n");
-    DISPLAYOUT("  -B#                  select size of each job (default: 0==automatic) \n");
-    DISPLAYOUT("     --single-thread   use a single thread for both I/O and compression (result slightly different than -T1) \n");
-    DISPLAYOUT("     --auto-threads={physical,logical} :  use either physical cores or logical cores as default when specifying -T0 (default: physical)\n");
-    DISPLAYOUT("     --rsyncable       compress using a rsync-friendly method (-B sets block size) \n");
+    DISPLAYOUT("  -T#                           Spawn # compression threads. [Default: 1; pass 0 for core count.]\n");
+    DISPLAYOUT("  --single-thread               Share a single thread for I/O and compression (slightly different than `-T1`).\n");
+    DISPLAYOUT("  --auto-threads={physical|logical}\n");
+    DISPLAYOUT("                                Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
+    DISPLAYOUT("  -B#                           Set job size to #. [Default: 0 (automatic)]\n");
+    DISPLAYOUT("  --rsyncable                   Compress using a rsync-friendly method (`-B` sets block size). \n");
+    DISPLAYOUT("\n");
 # endif
-    DISPLAYOUT("     --exclude-compressed :  only compress files that are not already compressed \n");
-    DISPLAYOUT("     --stream-size=#   specify size of streaming input from `stdin` \n");
-    DISPLAYOUT("     --size-hint=#     optimize compression parameters for streaming input of approximately this size \n");
-    DISPLAYOUT("     --target-compressed-block-size=# :  generate compressed block of approximately targeted size \n");
-    DISPLAYOUT("     --no-dictID       don't write dictID into header (dictionary compression only)\n");
-    DISPLAYOUT("     --[no-]compress-literals :  force (un)compressed literals\n");
-    DISPLAYOUT("     --[no-]row-match-finder :  force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
+    DISPLAYOUT("  --exclude-compressed          Only compress files that are not already compressed.\n\n");
 
-    DISPLAYOUT("     --format=zstd     compress files to the .zst format (default)\n");
+    DISPLAYOUT("  --stream-size=#               Specify size of streaming input from STDIN.\n");
+    DISPLAYOUT("  --size-hint=#                 Optimize compression parameters for streaming input of approximately size #.\n");
+    DISPLAYOUT("  --target-compressed-block-size=#\n");
+    DISPLAYOUT("                                Generate compressed blocks of approximately # size.\n\n");
+    DISPLAYOUT("  --no-dictID                   Don't write `dictID` into the header (dictionary compression only).\n");
+    DISPLAYOUT("  --[no-]compress-literals      Force (un)compressed literals.\n");
+    DISPLAYOUT("  --[no-]row-match-finder       Explicitly enable/disable the fast, row-based matchfinder for\n");
+    DISPLAYOUT("                                the 'greedy', 'lazy', and 'lazy2' strategies.\n");
+
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --format=zstd                 Compress files to the `.zst` format. [Default]\n");
 #ifdef ZSTD_GZCOMPRESS
-    DISPLAYOUT("     --format=gzip     compress files to the .gz format\n");
+    DISPLAYOUT("  --format=gzip                 Compress files to the `.gz` format.\n");
 #endif
 #ifdef ZSTD_LZMACOMPRESS
-    DISPLAYOUT("     --format=xz       compress files to the .xz format\n");
-    DISPLAYOUT("     --format=lzma     compress files to the .lzma format\n");
+    DISPLAYOUT("  --format=xz                   Compress files to the `.xz` format.\n");
+    DISPLAYOUT("  --format=lzma                 Compress files to the `.lzma` format.\n");
 #endif
 #ifdef ZSTD_LZ4COMPRESS
-    DISPLAYOUT( "     --format=lz4     compress files to the .lz4 format\n");
+    DISPLAYOUT( "  --format=lz4                 Compress files to the `.lz4` format.\n");
 #endif
 #endif  /* !ZSTD_NOCOMPRESS */
 
 #ifndef ZSTD_NODECOMPRESS
     DISPLAYOUT("\n");
-    DISPLAYOUT("Advanced decompression options :\n");
-    DISPLAYOUT("  -l                   print information about zstd compressed files\n");
-    DISPLAYOUT("     --test            test compressed file integrity\n");
-    DISPLAYOUT("  -M#                  Set a memory usage limit for decompression\n");
+    DISPLAYOUT("Advanced decompression options:\n");
+    DISPLAYOUT("  -l                            Print information about Zstandard-compressed files.\n");
+    DISPLAYOUT("  --test                        Test compressed file integrity.\n");
+    DISPLAYOUT("  -M#                           Set the memory usage limit to # megabytes.\n");
 # if ZSTD_SPARSE_DEFAULT
-    DISPLAYOUT("     --[no-]sparse     sparse mode (default: enabled on file, disabled on stdout)\n");
+    DISPLAYOUT("  --[no-]sparse                 Enable sparse mode. [Default: Enabled for files, disabled for STDOUT.]\n");
 # else
-    DISPLAYOUT("     --[no-]sparse     sparse mode (default: disabled)\n");
+    DISPLAYOUT("  --[no-]sparse                 Enable sparse mode. [Default: Disabled]\n");
 # endif
     {
-        char const* passThroughDefault = "disabled";
+        char const* passThroughDefault = "Disabled";
         if (exeNameMatch(programName, ZSTD_CAT) ||
             exeNameMatch(programName, ZSTD_ZCAT) ||
             exeNameMatch(programName, ZSTD_GZCAT)) {
-            passThroughDefault = "enabled";
+            passThroughDefault = "Enabled";
         }
-        DISPLAYOUT("     --[no-]pass-through : passes through uncompressed files as-is (default: %s)\n", passThroughDefault);
+        DISPLAYOUT("  --[no-]pass-through           Pass through uncompressed files as-is. [Default: %s]\n", passThroughDefault);
     }
 #endif  /* ZSTD_NODECOMPRESS */
 
 #ifndef ZSTD_NODICT
     DISPLAYOUT("\n");
-    DISPLAYOUT("Dictionary builder :\n");
-    DISPLAYOUT("     --train ##        create a dictionary from a training set of files\n");
-    DISPLAYOUT("     --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] :  use the cover algorithm with optional args\n");
-    DISPLAYOUT("     --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] :  use the fast cover algorithm with optional args\n");
-    DISPLAYOUT("     --train-legacy[=s=#] :  use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
-    DISPLAYOUT("  -o DICT              DICT is dictionary name (default: %s)\n", g_defaultDictName);
-    DISPLAYOUT("     --maxdict=#       limit dictionary to specified size (default: %u)\n", g_defaultMaxDictSize);
-    DISPLAYOUT("     --dictID=#        force dictionary ID to specified value (default: random)\n");
+    DISPLAYOUT("Dictionary builder:\n");
+    DISPLAYOUT("  --train                       Create a dictionary from a training set of files.\n\n");
+    DISPLAYOUT("  --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]\n");
+    DISPLAYOUT("                                Use the cover algorithm (with optional arguments).\n");
+    DISPLAYOUT("  --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]\n");
+    DISPLAYOUT("                                Use the fast cover algorithm (with optional arguments).\n\n");
+    DISPLAYOUT("  --train-legacy[=s=#]          Use the legacy algorithm with selectivity #. [Default: %u]\n", g_defaultSelectivityLevel);
+    DISPLAYOUT("  -o NAME                       Use NAME as dictionary name. [Default: %s]\n", g_defaultDictName);
+    DISPLAYOUT("  --maxdict=#                   Limit dictionary to specified size #. [Default: %u]\n", g_defaultMaxDictSize);
+    DISPLAYOUT("  --dictID=#                    Force dictionary ID to #. [Default: Random]\n");
 #endif
 
 #ifndef ZSTD_NOBENCH
     DISPLAYOUT("\n");
-    DISPLAYOUT("Benchmark options : \n");
-    DISPLAYOUT("  -b#                  benchmark file(s), using # compression level (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT);
-    DISPLAYOUT("  -e#                  test all compression levels successively from -b# to -e# (default: 1)\n");
-    DISPLAYOUT("  -i#                  minimum evaluation time in seconds (default: 3s)\n");
-    DISPLAYOUT("  -B#                  cut file into independent chunks of size # (default: no chunking)\n");
-    DISPLAYOUT("  -S                   output one benchmark result per input file (default: consolidated result)\n");
-    DISPLAYOUT("     --priority=rt     set process priority to real-time\n");
+    DISPLAYOUT("Benchmark options:\n");
+    DISPLAYOUT("  -b#                           Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
+    DISPLAYOUT("  -e#                           Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
+    DISPLAYOUT("  -i#                           Set the minimum evaluation to time # seconds. [Default: 3]\n");
+    DISPLAYOUT("  -B#                           Cut file into independent chunks of size #. [Default: No chunking]\n");
+    DISPLAYOUT("  -S                            Output one benchmark result per input file. [Default: Consolidated result]\n");
+    DISPLAYOUT("  --priority=rt                 Set process priority to real-time.\n");
 #endif
 
 }

From 9c93dd71cdb301595a8a9e364348967a173c010c Mon Sep 17 00:00:00 2001
From: Jon Palmisciano <jp@jonpalmisc.com>
Date: Tue, 20 Dec 2022 20:11:23 -0500
Subject: [PATCH 069/197] Clean up welcome message

---
 programs/zstdcli.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index d26df2850..84396e507 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -52,12 +52,12 @@
 /*-************************************
 *  Constants
 **************************************/
-#define COMPRESSOR_NAME "zstd command line interface"
+#define COMPRESSOR_NAME "Zstandard CLI"
 #ifndef ZSTD_VERSION
 #  define ZSTD_VERSION "v" ZSTD_VERSION_STRING
 #endif
 #define AUTHOR "Yann Collet"
-#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
+#define WELCOME_MESSAGE "*** %s (%i-bit) %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
 
 #define ZSTD_ZSTDMT "zstdmt"
 #define ZSTD_UNZSTD "unzstd"
@@ -178,6 +178,7 @@ static void usage(FILE* f, const char* programName)
 static void usage_advanced(const char* programName)
 {
     DISPLAYOUT(WELCOME_MESSAGE);
+    DISPLAYOUT("\n");
     usage(stdout, programName);
     DISPLAYOUT("Advanced options:\n");
     DISPLAYOUT("  -c, --stdout                  Write to STDOUT (even if it is a console) and keep the INPUT file(s).\n\n");

From 125bcde17cc49e2838da1da64efcc69cc478c379 Mon Sep 17 00:00:00 2001
From: Jon Palmisciano <jp@jonpalmisc.com>
Date: Tue, 20 Dec 2022 20:11:37 -0500
Subject: [PATCH 070/197] Update tests to expect new CLI help output

---
 tests/cli-tests/basic/help.sh.stdout.glob     | 41 +++++++++++--------
 tests/cli-tests/basic/version.sh.stdout.glob  |  4 +-
 .../progress/no-progress.sh.stderr.glob       | 20 ++++-----
 3 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/tests/cli-tests/basic/help.sh.stdout.glob b/tests/cli-tests/basic/help.sh.stdout.glob
index bfb0f7753..5580dc669 100644
--- a/tests/cli-tests/basic/help.sh.stdout.glob
+++ b/tests/cli-tests/basic/help.sh.stdout.glob
@@ -1,25 +1,34 @@
 + zstd -h
-Usage: zstd *OPTION*... *FILE*... *-o file*
-Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).
+Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
 
-  -o file              result stored into `file` (only 1 output file)
-  -1 .. -19            compression level (faster .. better; default: 3)
-  -d, --decompress     decompression
-  -f, --force          disable input and output checks. Allows overwriting existing files,
-                       input from console, output to stdout, operating on links,
-                       block devices, etc. During decompression and when the output
-                       destination is stdout, pass-through unrecognized formats as-is.
-      --rm             remove source file(s) after successful de/compression
-  -k, --keep           preserve source file(s) (default)
-  -D DICT              use DICT as Dictionary for compression or decompression
-  -h                   display usage and exit
-  -H,--help            display long help and exit
+Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
+
+Options:
+  -o OUTPUT                     Write output to a single file, OUTPUT.
+  -k, --keep                    Preserve INPUT file(s). *Default*
+  --rm                          Remove INPUT file(s) after successful (de)compression.
+
+  -#                            Desired compression level, where `#` is a number between 1 and 19;
+                                lower numbers provide faster compression, higher numbers yield
+                                better compression ratios. *Default: 3*
+
+  -d, --decompress              Perform decompression.
+  -D DICT                       Use DICT as the dictionary for compression or decompression.
+
+  -f, --force                   Disable input and output checks. Allows overwriting existing files,
+                                receiving input from the console, printing ouput to STDOUT, and
+                                operating on links, block devices, etc. Unrecognized formats will be
+                                passed-through through as-is.
+
+  -h                            Display short usage and exit.
+  -H, --help                    Display full help and exit.
+  -V, --version                 Display the program version and exit.
 
 + zstd -H
 ...
-Advanced options :
+Advanced options:
 ...
 + zstd --help
 ...
-Advanced options :
+Advanced options:
 ...
diff --git a/tests/cli-tests/basic/version.sh.stdout.glob b/tests/cli-tests/basic/version.sh.stdout.glob
index 54968fa41..4cc9fb9b2 100644
--- a/tests/cli-tests/basic/version.sh.stdout.glob
+++ b/tests/cli-tests/basic/version.sh.stdout.glob
@@ -1,2 +1,2 @@
-*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
-*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
diff --git a/tests/cli-tests/progress/no-progress.sh.stderr.glob b/tests/cli-tests/progress/no-progress.sh.stderr.glob
index f07ad3803..b4262517a 100644
--- a/tests/cli-tests/progress/no-progress.sh.stderr.glob
+++ b/tests/cli-tests/progress/no-progress.sh.stderr.glob
@@ -55,36 +55,36 @@ decompress 2 files
 
 args = --no-progress --fake-stderr-is-console -v
 compress file to file
-*zstd*
+*Zstandard CLI*
 hello*hello.zst*
 compress pipe to pipe
-*zstd*
+*Zstandard CLI*
 *stdin*stdout*
 compress pipe to file
-*zstd*
+*Zstandard CLI*
 *stdin*hello.zst*
 compress file to pipe
-*zstd*
+*Zstandard CLI*
 *hello*stdout*
 compress 2 files
-*zstd*
+*Zstandard CLI*
 *hello*hello.zst*
 *world*world.zst*
 2 files compressed*
 decompress file to file
-*zstd*
+*Zstandard CLI*
 hello.zst*
 decompress pipe to pipe
-*zstd*
+*Zstandard CLI*
 *stdin*
 decompress pipe to file
-*zstd*
+*Zstandard CLI*
 *stdin*
 decompress file to pipe
-*zstd*
+*Zstandard CLI*
 hello.zst*
 decompress 2 files
-*zstd*
+*Zstandard CLI*
 hello.zst*
 world.zst*
 2 files decompressed*

From b6e8112261a0d60476d52a10be8fd3be0287996d Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Dec 2022 17:00:44 -0800
Subject: [PATCH 071/197] [util] Add traces enabled by --trace-file-stat

Print traces to stderr when --trace-file-stat is passed.
We trace all functions that read and write file metadata.
---
 programs/util.c    | 165 +++++++++++++++++++++++++++++++++++----------
 programs/util.h    |   5 ++
 programs/zstdcli.c |   1 +
 3 files changed, 137 insertions(+), 34 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index de8a27739..21d7d3b46 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -66,6 +66,27 @@ extern "C" {
 #define UTIL_DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
 
+static int g_traceDepth = 0;
+int g_traceFileStat = 0;
+
+#define UTIL_TRACE_CALL(...)                                         \
+    {                                                                \
+        if (g_traceFileStat) {                                       \
+            UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \
+            UTIL_DISPLAY(__VA_ARGS__);                               \
+            UTIL_DISPLAY("\n");                                      \
+            ++g_traceDepth;                                          \
+        }                                                            \
+    }
+
+#define UTIL_TRACE_RET(ret)                                                     \
+    {                                                                           \
+        if (g_traceFileStat) {                                                  \
+            --g_traceDepth;                                                     \
+            UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \
+        }                                                                      \
+    }
+
 /* A modified version of realloc().
  * If UTIL_realloc() fails the original block is freed.
  */
@@ -121,21 +142,34 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
 *  Functions
 ***************************************/
 
+void UTIL_traceFileStat(void)
+{
+    g_traceFileStat = 1;
+}
+
 int UTIL_stat(const char* filename, stat_t* statbuf)
 {
+    int ret;
+    UTIL_TRACE_CALL("UTIL_stat(%s)", filename);
 #if defined(_MSC_VER)
-    return !_stat64(filename, statbuf);
+    ret = !_stat64(filename, statbuf);
 #elif defined(__MINGW32__) && defined (__MSVCRT__)
-    return !_stati64(filename, statbuf);
+    ret = !_stati64(filename, statbuf);
 #else
-    return !stat(filename, statbuf);
+    ret = !stat(filename, statbuf);
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isRegularFile(const char* infilename)
 {
     stat_t statbuf;
-    return UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename);
+    ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isRegularFileStat(const stat_t* statbuf)
@@ -151,44 +185,66 @@ int UTIL_isRegularFileStat(const stat_t* statbuf)
 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
 {
     stat_t localStatBuf;
+    UTIL_TRACE_CALL("UTIL_chmod(%s, %u)", filename, (unsigned)permissions);
     if (statbuf == NULL) {
-        if (!UTIL_stat(filename, &localStatBuf)) return 0;
+        if (!UTIL_stat(filename, &localStatBuf)) {
+            UTIL_TRACE_RET(0);
+            return 0;
+        }
         statbuf = &localStatBuf;
     }
-    if (!UTIL_isRegularFileStat(statbuf)) return 0; /* pretend success, but don't change anything */
-    return chmod(filename, permissions);
+    if (!UTIL_isRegularFileStat(statbuf)) {
+        UTIL_TRACE_RET(0);
+        return 0; /* pretend success, but don't change anything */
+    }
+    UTIL_TRACE_CALL("chmod");
+    {
+        int const ret = chmod(filename, permissions);
+        UTIL_TRACE_RET(ret);
+        UTIL_TRACE_RET(ret);
+        return ret;
+    }
 }
 
 /* set access and modification times */
 int UTIL_utime(const char* filename, const stat_t *statbuf)
 {
     int ret;
+    UTIL_TRACE_CALL("UTIL_utime(%s)", filename);
     /* We check that st_mtime is a macro here in order to give us confidence
      * that struct stat has a struct timespec st_mtim member. We need this
      * check because there are some platforms that claim to be POSIX 2008
      * compliant but which do not have st_mtim... */
 #if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime)
-    /* (atime, mtime) */
-    struct timespec timebuf[2] = { {0, UTIME_NOW} };
-    timebuf[1] = statbuf->st_mtim;
-    ret = utimensat(AT_FDCWD, filename, timebuf, 0);
+    {
+        /* (atime, mtime) */
+        struct timespec timebuf[2] = { {0, UTIME_NOW} };
+        timebuf[1] = statbuf->st_mtim;
+        ret = utimensat(AT_FDCWD, filename, timebuf, 0);
+    }
 #else
-    struct utimbuf timebuf;
-    timebuf.actime = time(NULL);
-    timebuf.modtime = statbuf->st_mtime;
-    ret = utime(filename, &timebuf);
+    {
+        struct utimbuf timebuf;
+        timebuf.actime = time(NULL);
+        timebuf.modtime = statbuf->st_mtime;
+        ret = utime(filename, &timebuf);
+    }
 #endif
     errno = 0;
+    UTIL_TRACE_RET(ret);
     return ret;
 }
 
 int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
 {
     int res = 0;
-
     stat_t curStatBuf;
-    if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf))
+    UTIL_TRACE_CALL("UTIL_setFileStat(%s)", filename);
+
+    if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
+        UTIL_TRACE_RET(-1);
         return -1;
+    }
 
     /* set access and modification times */
     res += UTIL_utime(filename, statbuf);
@@ -200,13 +256,18 @@ int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
     res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 07777);  /* Copy file permissions */
 
     errno = 0;
+    UTIL_TRACE_RET(-res);
     return -res; /* number of errors is returned */
 }
 
 int UTIL_isDirectory(const char* infilename)
 {
     stat_t statbuf;
-    return UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename);
+    ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isDirectoryStat(const stat_t* statbuf)
@@ -224,33 +285,44 @@ int UTIL_compareStr(const void *p1, const void *p2) {
 
 int UTIL_isSameFile(const char* fName1, const char* fName2)
 {
+    int ret;
     assert(fName1 != NULL); assert(fName2 != NULL);
+    UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2);
 #if defined(_MSC_VER) || defined(_WIN32)
     /* note : Visual does not support file identification by inode.
      *        inode does not work on Windows, even with a posix layer, like msys2.
      *        The following work-around is limited to detecting exact name repetition only,
      *        aka `filename` is considered different from `subdir/../filename` */
-    return !strcmp(fName1, fName2);
+    ret = !strcmp(fName1, fName2);
 #else
     {   stat_t file1Stat;
         stat_t file2Stat;
-        return UTIL_stat(fName1, &file1Stat)
+        ret =  UTIL_stat(fName1, &file1Stat)
             && UTIL_stat(fName2, &file2Stat)
             && (file1Stat.st_dev == file2Stat.st_dev)
             && (file1Stat.st_ino == file2Stat.st_ino);
     }
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 /* UTIL_isFIFO : distinguish named pipes */
 int UTIL_isFIFO(const char* infilename)
 {
+    UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename);
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #if PLATFORM_POSIX_VERSION >= 200112L
-    stat_t statbuf;
-    if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) return 1;
+    {
+        stat_t statbuf;
+        if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) {
+            UTIL_TRACE_RET(1);
+            return 1;
+        }
+    }
 #endif
     (void)infilename;
+    UTIL_TRACE_RET(0);
     return 0;
 }
 
@@ -278,13 +350,20 @@ int UTIL_isBlockDevStat(const stat_t* statbuf)
 
 int UTIL_isLink(const char* infilename)
 {
+    UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename);
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #if PLATFORM_POSIX_VERSION >= 200112L
-    stat_t statbuf;
-    int const r = lstat(infilename, &statbuf);
-    if (!r && S_ISLNK(statbuf.st_mode)) return 1;
+    {
+        stat_t statbuf;
+        int const r = lstat(infilename, &statbuf);
+        if (!r && S_ISLNK(statbuf.st_mode)) {
+            UTIL_TRACE_RET(1);
+            return 1;
+        }
+    }
 #endif
     (void)infilename;
+    UTIL_TRACE_RET(0);
     return 0;
 }
 
@@ -294,13 +373,18 @@ static int g_fakeStdoutIsConsole = 0;
 
 int UTIL_isConsole(FILE* file)
 {
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file));
     if (file == stdin && g_fakeStdinIsConsole)
-        return 1;
-    if (file == stderr && g_fakeStderrIsConsole)
-        return 1;
-    if (file == stdout && g_fakeStdoutIsConsole)
-        return 1;
-    return IS_CONSOLE(file);
+        ret = 1;
+    else if (file == stderr && g_fakeStderrIsConsole)
+        ret = 1;
+    else if (file == stdout && g_fakeStdoutIsConsole)
+        ret = 1;
+    else
+        ret = IS_CONSOLE(file);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 void UTIL_fakeStdinIsConsole(void)
@@ -319,8 +403,16 @@ void UTIL_fakeStderrIsConsole(void)
 U64 UTIL_getFileSize(const char* infilename)
 {
     stat_t statbuf;
-    if (!UTIL_stat(infilename, &statbuf)) return UTIL_FILESIZE_UNKNOWN;
-    return UTIL_getFileSizeStat(&statbuf);
+    UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename);
+    if (!UTIL_stat(infilename, &statbuf)) {
+        UTIL_TRACE_RET(-1);
+        return UTIL_FILESIZE_UNKNOWN;
+    }
+    {
+        U64 const size = UTIL_getFileSizeStat(&statbuf);
+        UTIL_TRACE_RET((int)size);
+        return size;
+    }
 }
 
 U64 UTIL_getFileSizeStat(const stat_t* statbuf)
@@ -397,11 +489,16 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
 {
     U64 total = 0;
     unsigned n;
+    UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles);
     for (n=0; n<nbFiles; n++) {
         U64 const size = UTIL_getFileSize(fileNamesTable[n]);
-        if (size == UTIL_FILESIZE_UNKNOWN) return UTIL_FILESIZE_UNKNOWN;
+        if (size == UTIL_FILESIZE_UNKNOWN) {
+            UTIL_TRACE_RET(-1);
+            return UTIL_FILESIZE_UNKNOWN;
+        }
         total += size;
     }
+    UTIL_TRACE_RET((int)total);
     return total;
 }
 
diff --git a/programs/util.h b/programs/util.h
index d022c0014..137be31fb 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -189,6 +189,11 @@ void UTIL_fakeStdinIsConsole(void);
 void UTIL_fakeStdoutIsConsole(void);
 void UTIL_fakeStderrIsConsole(void);
 
+/**
+ * Emit traces for functions that read, or modify file metadata.
+ */
+void UTIL_traceFileStat(void);
+
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
 U64 UTIL_getFileSize(const char* infilename);
 U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 7ff18dcb0..9937a9c97 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1004,6 +1004,7 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--fake-stdin-is-console")) { UTIL_fakeStdinIsConsole(); continue; }
                 if (!strcmp(argument, "--fake-stdout-is-console")) { UTIL_fakeStdoutIsConsole(); continue; }
                 if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }
+                if (!strcmp(argument, "--trace-file-stat")) { UTIL_traceFileStat(); continue; }
 
                 /* long commands with arguments */
 #ifndef ZSTD_NODICT

From 7df6e25b8530392d121e73ba5cfaf71b3c0ec3ef Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Dec 2022 17:21:09 -0800
Subject: [PATCH 072/197] [cli-tests] Add --set-exact-output to update the
 expected output

`./run.py --set-exact-output` will update `stdout.expect` and
`stderr.expect` to match the expected output. This doesn't apply to
outputs which use `.glob` or `.ignore`.
---
 tests/cli-tests/README.md | 10 ++++++++++
 tests/cli-tests/run.py    | 18 +++++++++++++++---
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/tests/cli-tests/README.md b/tests/cli-tests/README.md
index cdf9b8e71..7ca07c3e9 100644
--- a/tests/cli-tests/README.md
+++ b/tests/cli-tests/README.md
@@ -45,6 +45,16 @@ Examples:
 ./run.py --preserve --verbose basic/help.sh
 ```
 
+### Updating exact output
+
+If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
+
+Example:
+```
+./run.py --set-exact-output
+./run.py basic/help.sh --set-exact-output
+```
+
 ## Writing a test
 
 Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
diff --git a/tests/cli-tests/run.py b/tests/cli-tests/run.py
index 2696d8b64..8b4e21a53 100755
--- a/tests/cli-tests/run.py
+++ b/tests/cli-tests/run.py
@@ -209,6 +209,7 @@ class Options:
         preserve: bool,
         scratch_dir: str,
         test_dir: str,
+        set_exact_output: bool,
     ) -> None:
         self.env = env
         self.timeout = timeout
@@ -216,6 +217,7 @@ class Options:
         self.preserve = preserve
         self.scratch_dir = scratch_dir
         self.test_dir = test_dir
+        self.set_exact_output = set_exact_output
 
 
 class TestCase:
@@ -335,7 +337,7 @@ class TestCase:
             self._test_stdin.close()
             self._test_stdin = None
 
-    def _check_output_exact(self, out_name: str, expected: bytes) -> None:
+    def _check_output_exact(self, out_name: str, expected: bytes, exact_name: str) -> None:
         """
         Check the output named :out_name: for an exact match against the :expected: content.
         Saves the success and message.
@@ -349,6 +351,10 @@ class TestCase:
             self._success[check_name] = False
             self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{diff(expected, actual)}"
 
+            if self._opts.set_exact_output:
+                with open(exact_name, "wb") as f:
+                    f.write(actual)
+
     def _check_output_glob(self, out_name: str, expected: bytes) -> None:
         """
         Check the output named :out_name: for a glob match against the :expected: glob.
@@ -386,7 +392,7 @@ class TestCase:
         ignore_name = f"{self._test_file}.{out_name}.ignore"
 
         if os.path.exists(exact_name):
-            return self._check_output_exact(out_name, read_file(exact_name))
+            return self._check_output_exact(out_name, read_file(exact_name), exact_name)
         elif os.path.exists(glob_name):
             return self._check_output_glob(out_name, read_file(glob_name))
         elif os.path.exists(ignore_name):
@@ -394,7 +400,7 @@ class TestCase:
             self._success[check_name] = True
             self._message[check_name] = f"{out_name} ignored!"
         else:
-            return self._check_output_exact(out_name, bytes())
+            return self._check_output_exact(out_name, bytes(), exact_name)
 
     def _check_stderr(self) -> None:
         """Checks the stderr output against the expectation."""
@@ -678,6 +684,11 @@ if __name__ == "__main__":
             "Scratch directory located in TEST_DIR/scratch/."
         )
     )
+    parser.add_argument(
+        "--set-exact-output",
+        action="store_true",
+        help="Set stderr.exact and stdout.exact for all failing tests, unless .ignore or .glob already exists"
+    )
     parser.add_argument(
         "tests",
         nargs="*",
@@ -714,6 +725,7 @@ if __name__ == "__main__":
         preserve=args.preserve,
         test_dir=args.test_dir,
         scratch_dir=scratch_dir,
+        set_exact_output=args.set_exact_output,
     )
 
     if len(args.tests) == 0:

From 31a703ec13c197d1ec0939855bb31599ae9be1c8 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Dec 2022 15:02:27 -0800
Subject: [PATCH 073/197] [cmake] Add noexecstack to compiler/linker flags

---
 .../AddZstdCompilationFlags.cmake             | 42 ++++++++++++-------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
index 8d04458c3..8e36f61d0 100644
--- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
@@ -1,7 +1,8 @@
 include(CheckCXXCompilerFlag)
 include(CheckCCompilerFlag)
+include(CheckLinkerFlag)
 
-function(EnableCompilerFlag _flag _C _CXX)
+function(EnableCompilerFlag _flag _C _CXX _LD)
     string(REGEX REPLACE "\\+" "PLUS" varname "${_flag}")
     string(REGEX REPLACE "[^A-Za-z0-9]+" "_" varname "${varname}")
     string(REGEX REPLACE "^_+" "" varname "${varname}")
@@ -18,6 +19,13 @@ function(EnableCompilerFlag _flag _C _CXX)
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
+    if (_LD)
+        CHECK_LINKER_FLAG(C ${_flag} LD_FLAG_${varname})
+        if (LD_FLAG_${varname})
+            set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
+            set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
+        endif ()
+    endif ()
 endfunction()
 
 macro(ADD_ZSTD_COMPILATION_FLAGS)
@@ -30,33 +38,39 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
         # EnableCompilerFlag("-std=c99" true false)   # Set C compiation to c99 standard
         if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
             # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" true true)
+            EnableCompilerFlag("/clang:-Wall" true true flase)
         else ()
-            EnableCompilerFlag("-Wall" true true)
+            EnableCompilerFlag("-Wall" true true false)
         endif ()
-        EnableCompilerFlag("-Wextra" true true)
-        EnableCompilerFlag("-Wundef" true true)
-        EnableCompilerFlag("-Wshadow" true true)
-        EnableCompilerFlag("-Wcast-align" true true)
-        EnableCompilerFlag("-Wcast-qual" true true)
-        EnableCompilerFlag("-Wstrict-prototypes" true false)
+        EnableCompilerFlag("-Wextra" true true false)
+        EnableCompilerFlag("-Wundef" true true false)
+        EnableCompilerFlag("-Wshadow" true true false)
+        EnableCompilerFlag("-Wcast-align" true true false)
+        EnableCompilerFlag("-Wcast-qual" true true false)
+        EnableCompilerFlag("-Wstrict-prototypes" true false false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("-DDEBUGLEVEL=1" true true)
+            EnableCompilerFlag("-DDEBUGLEVEL=1" true true false)
         endif ()
+        # Add noexecstack flags
+        # LDFLAGS
+        EnableCompilerFlag("-z noexecstack" false false true)
+        # CFLAGS & CXXFLAGS
+        EnableCompilerFlag("-Qunused-arguments" true true false)
+        EnableCompilerFlag("-Wa,--noexecstack" true true false)
     elseif (MSVC) # Add specific compilation flags for Windows Visual
 
         set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
         if (CMAKE_GENERATOR MATCHES "Visual Studio" AND ACTIVATE_MULTITHREADED_COMPILATION)
-            EnableCompilerFlag("/MP" true true)
+            EnableCompilerFlag("/MP" true true false)
         endif ()
 
         # UNICODE SUPPORT
-        EnableCompilerFlag("/D_UNICODE" true true)
-        EnableCompilerFlag("/DUNICODE" true true)
+        EnableCompilerFlag("/D_UNICODE" true true false)
+        EnableCompilerFlag("/DUNICODE" true true false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("/DDEBUGLEVEL=1" true true)
+            EnableCompilerFlag("/DDEBUGLEVEL=1" true true false)
         endif ()
     endif ()
 

From 40a718813070e23c3c7a437477a9e984d4de3f8b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Dec 2022 16:09:25 -0800
Subject: [PATCH 074/197] Fix `make clangbuild` & add CI

Fix the errors for:
* `-Wdocumentation`
* `-Wconversion` except `-Wsign-conversion`
---
 .github/workflows/dev-short-tests.yml |  8 ++++++
 contrib/largeNbDicts/largeNbDicts.c   |  4 +--
 lib/compress/zstd_compress.c          |  2 +-
 lib/dictBuilder/cover.c               |  4 +--
 lib/dictBuilder/zdict.c               |  4 +--
 lib/zdict.h                           |  6 ++---
 lib/zstd.h                            |  8 +++---
 programs/benchfn.c                    |  6 ++---
 programs/fileio.c                     | 12 ++++-----
 programs/util.h                       |  1 -
 tests/decodecorpus.c                  |  6 ++---
 tests/paramgrill.c                    | 38 +++++++++++++--------------
 zlibWrapper/examples/zwrapbench.c     | 10 +++----
 13 files changed, 58 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index e9aa4046b..b9998e024 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -498,6 +498,14 @@ jobs:
       run: |
         make -C tests versionsTest
 
+  clangbuild:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - name: make clangbuild
+      run: |
+        make clangbuild
+
 
 # For reference : icc tests
 # icc tests are currently failing on Github Actions, likely to issues during installation stage
diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index 519468506..7b2af0bae 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -856,7 +856,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary,
     CONTROL(cTotalSizeNoDict != 0);
     DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f  (%u bytes) \n",
                     clevel,
-                    (double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
+                    (double)totalSrcSlicesSize / (double)cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
 
     size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
     CONTROL(cSizes != NULL);
@@ -865,7 +865,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary,
     CONTROL(cTotalSize != 0);
     DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f  (%u bytes) \n",
                     (unsigned)dictBuffer.size,
-                    (double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
+                    (double)totalSrcSlicesSize / (double)cTotalSize, (unsigned)cTotalSize);
 
     /* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
     shrinkSizes(dstSlices, cSizes);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f4a3c8264..4fb203db4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -376,7 +376,7 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete
 
 /**
  * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone.
- * @param param Validated zstd parameters.
+ * @param params Validated zstd parameters.
  */
 static void ZSTD_CCtxParams_setZstdParams(
         ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 1ade83987..18c8ec538 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -647,7 +647,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
 
 void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
 {
-  const double ratio = (double)nbDmers / maxDictSize;
+  const double ratio = (double)nbDmers / (double)maxDictSize;
   if (ratio >= 10) {
       return;
   }
@@ -1040,7 +1040,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
       return COVER_dictSelectionError(totalCompressedSize);
     }
 
-    if (totalCompressedSize <= largestCompressed * regressionTolerance) {
+    if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) {
       COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize };
       free(largestDictbuffer);
       return selection;
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index f9a5d678c..f41d505f8 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -373,7 +373,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
             elt = table[u];
             /* sort : improve rank */
             while ((u>1) && (table[u-1].savings < elt.savings))
-            table[u] = table[u-1], u--;
+                table[u] = table[u-1], u--;
             table[u] = elt;
             return u;
     }   }
@@ -524,7 +524,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
             if (solution.length==0) { cursor++; continue; }
             ZDICT_insertDictItem(dictList, dictListSize, solution, buffer);
             cursor += solution.length;
-            DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100);
+            DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0);
     }   }
 
 _cleanup:
diff --git a/lib/zdict.h b/lib/zdict.h
index 8bed583f1..7b37e50e8 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -212,9 +212,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
                                     const size_t* samplesSizes, unsigned nbSamples);
 
 typedef struct {
-    int      compressionLevel;   /*< optimize for a specific zstd compression level; 0 means default */
-    unsigned notificationLevel;  /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
-    unsigned dictID;             /*< force dictID value; 0 means auto mode (32-bits random value)
+    int      compressionLevel;   /**< optimize for a specific zstd compression level; 0 means default */
+    unsigned notificationLevel;  /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+    unsigned dictID;             /**< force dictID value; 0 means auto mode (32-bits random value)
                                   *   NOTE: The zstd format reserves some dictionary IDs for future use.
                                   *         You may use them in private settings, but be warned that they
                                   *         may be used by zstd in a public dictionary registry in the future.
diff --git a/lib/zstd.h b/lib/zstd.h
index 8bf032623..dd72e17ed 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2535,8 +2535,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
   Data fragment must be large enough to ensure successful decoding.
  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
-  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
-           >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
   It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@@ -2555,7 +2555,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
 
   The most memory efficient way is to use a round buffer of sufficient size.
   Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
-  which can @return an error code if required value is too large for current system (in 32-bits mode).
+  which can return an error code if required value is too large for current system (in 32-bits mode).
   In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
   up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
   which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@@ -2575,7 +2575,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
 
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
   It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
   It can also be an error code, which can be tested with ZSTD_isError().
 
diff --git a/programs/benchfn.c b/programs/benchfn.c
index 1b5c383ea..8e6726f8d 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -229,9 +229,9 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
             cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
 
             /* estimate nbLoops for next run to last approximately 1 second */
-            if (loopDuration_ns > (runBudget_ns / 50)) {
+            if (loopDuration_ns > ((double)runBudget_ns / 50)) {
                 double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
-                cont->nbLoops = (unsigned)(runBudget_ns / fastestRun_ns) + 1;
+                cont->nbLoops = (unsigned)((double)runBudget_ns / fastestRun_ns) + 1;
             } else {
                 /* previous run was too short : blindly increase workload by x multiplier */
                 const unsigned multiplier = 10;
@@ -239,7 +239,7 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
                 cont->nbLoops *= multiplier;
             }
 
-            if(loopDuration_ns < runTimeMin_ns) {
+            if(loopDuration_ns < (double)runTimeMin_ns) {
                 /* don't report results for which benchmark run time was too small : increased risks of rounding errors */
                 assert(completed == 0);
                 continue;
diff --git a/programs/fileio.c b/programs/fileio.c
index 418de6fd7..6bfa559e9 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1059,12 +1059,12 @@ FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but no
             DISPLAYUPDATE_PROGRESS(
                     "\rRead : %u MB ==> %.2f%% ",
                     (unsigned)(inFileSize>>20),
-                    (double)outFileSize/inFileSize*100)
+                    (double)outFileSize/(double)inFileSize*100)
         } else {
             DISPLAYUPDATE_PROGRESS(
                     "\rRead : %u / %u MB ==> %.2f%% ",
                     (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                    (double)outFileSize/inFileSize*100);
+                    (double)outFileSize/(double)inFileSize*100);
     }   }
 
     while (1) {
@@ -1157,11 +1157,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
         if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
             DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20),
-                            (double)outFileSize/inFileSize*100)
+                            (double)outFileSize/(double)inFileSize*100)
         else
             DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                            (double)outFileSize/inFileSize*100);
+                            (double)outFileSize/(double)inFileSize*100);
         if (ret == LZMA_STREAM_END) break;
     }
 
@@ -1241,11 +1241,11 @@ FIO_compressLz4Frame(cRess_t* ress,
             if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
                 DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20),
-                                (double)outFileSize/inFileSize*100)
+                                (double)outFileSize/(double)inFileSize*100)
             } else {
                 DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                                (double)outFileSize/inFileSize*100);
+                                (double)outFileSize/(double)inFileSize*100);
             }
 
             /* Write Block */
diff --git a/programs/util.h b/programs/util.h
index d022c0014..605d400a2 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -262,7 +262,6 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2);
 /*! UTIL_expandFNT() :
  *  read names from @fnt, and expand those corresponding to directories
  *  update @fnt, now containing only file names,
- * @return : 0 in case of success, 1 if error
  *  note : in case of error, @fnt[0] is NULL
  */
 void UTIL_expandFNT(FileNamesTable** fnt, int followLinks);
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index df52ae6a7..20ad5add0 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -136,7 +136,7 @@ static void RAND_genDist(U32* seed, BYTE* dist, double weight)
     BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */
 
     while (i < DISTSIZE) {
-        size_t states = ((size_t)(weight * statesLeft)) + 1;
+        size_t states = ((size_t)(weight * (double)statesLeft)) + 1;
         size_t j;
         for (j = 0; j < states && i < DISTSIZE; j++, i++) {
             dist[i] = symb;
@@ -662,11 +662,11 @@ generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
          * ensure nice numbers */
         U32 matchLen =
                 MIN_SEQ_LEN +
-                ROUND(RAND_exp(seed, excessMatch / (double)(numSequences - i)));
+                ROUND(RAND_exp(seed, (double)excessMatch / (double)(numSequences - i)));
         U32 literalLen =
                 (RAND(seed) & 7)
                         ? ROUND(RAND_exp(seed,
-                                         literalsSize /
+                                         (double)literalsSize /
                                                  (double)(numSequences - i)))
                         : 0;
         /* actual offset, code to send, and point to copy up to when shifting
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 5dfd895e9..8971c65d6 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -566,10 +566,10 @@ resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_
     double cs = 0., ds = 0., rt, cm = 0.;
     const double r1 = 1, r2 = 0.1, rtr = 0.5;
     double ret;
-    if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; }
-    if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; }
-    if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; }
-    rt = ((double)srcSize / res.cSize);
+    if(target.cSpeed) { cs = (double)res.cSpeed / (double)target.cSpeed; }
+    if(target.dSpeed) { ds = (double)res.dSpeed / (double)target.dSpeed; }
+    if(target.cMem != (U32)-1) { cm = (double)target.cMem / (double)res.cMem; }
+    rt = ((double)srcSize / (double)res.cSize);
 
     ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr +
          (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
@@ -581,8 +581,8 @@ resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_
 static double
 resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes)
 {
-    double normalizedCSpeedGain1 = ((double)result1.cSpeed / lvlRes.cSpeed) - 1;
-    double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
+    double normalizedCSpeedGain1 = ((double)result1.cSpeed / (double)lvlRes.cSpeed) - 1;
+    double normalizedRatioGain1 = ((double)lvlRes.cSize / (double)result1.cSize) - 1;
     if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
@@ -854,7 +854,7 @@ BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize)
     }
 
     {   double const ratio = res.result.cSize ?
-                            (double)srcSize / res.result.cSize : 0;
+                            (double)srcSize / (double)res.result.cSize : 0;
         double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT;
         double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT;
 
@@ -937,7 +937,7 @@ BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, co
         }
         fprintf(f, "================================\n");
         fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
-            (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT);
+            (double)srcSize / (double)g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT);
 
 
         fprintf(f, "Overall Winner: \n");
@@ -977,7 +977,7 @@ BMK_print_cLevelEntry(FILE* f, const int cLevel,
     }
     /* print comment */
     {   double const ratio = result.cSize ?
-                            (double)srcSize / result.cSize : 0;
+                            (double)srcSize / (double)result.cSize : 0;
         double const cSpeedMBps = (double)result.cSpeed / MB_UNIT;
         double const dSpeedMBps = (double)result.dSpeed / MB_UNIT;
 
@@ -1726,19 +1726,19 @@ static int allBench(BMK_benchResult_t* resultPtr,
 
     /* calculate uncertainty in compression / decompression runs */
     if (benchres.cSpeed) {
-        U64 const loopDurationC = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
+        double const loopDurationC = (double)(((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
         uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
     }
 
     if (benchres.dSpeed) {
-        U64 const loopDurationD = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
+        double const loopDurationD = (double)(((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
         uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
     }
 
     /* optimistic assumption of benchres */
     {   BMK_benchResult_t resultMax = benchres;
-        resultMax.cSpeed = (unsigned long long)(resultMax.cSpeed * uncertaintyConstantC * VARIANCE);
-        resultMax.dSpeed = (unsigned long long)(resultMax.dSpeed * uncertaintyConstantD * VARIANCE);
+        resultMax.cSpeed = (unsigned long long)((double)resultMax.cSpeed * uncertaintyConstantC * VARIANCE);
+        resultMax.dSpeed = (unsigned long long)((double)resultMax.dSpeed * uncertaintyConstantD * VARIANCE);
 
         /* disregard infeasible results in feas mode */
         /* disregard if resultMax < winner in infeas mode */
@@ -1850,8 +1850,8 @@ static int BMK_seed(winnerInfo_t* winners,
 
         if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
             /* Validate solution is "good enough" */
-            double W_ratio = (double)buf.srcSize / testResult.cSize;
-            double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
+            double W_ratio = (double)buf.srcSize / (double)testResult.cSize;
+            double O_ratio = (double)buf.srcSize / (double)winners[cLevel].result.cSize;
             double W_ratioNote = log (W_ratio);
             double O_ratioNote = log (O_ratio);
             size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB);
@@ -1864,11 +1864,11 @@ static int BMK_seed(winnerInfo_t* winners,
             double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
             double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
 
-            double W_CSpeed_note = W_ratioNote * (double)( 30 + 10*cLevel) + log(testResult.cSpeed);
-            double O_CSpeed_note = O_ratioNote * (double)( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed);
+            double W_CSpeed_note = W_ratioNote * (double)( 30 + 10*cLevel) + log((double)testResult.cSpeed);
+            double O_CSpeed_note = O_ratioNote * (double)( 30 + 10*cLevel) + log((double)winners[cLevel].result.cSpeed);
 
-            double W_DSpeed_note = W_ratioNote * (double)( 20 + 2*cLevel) + log(testResult.dSpeed);
-            double O_DSpeed_note = O_ratioNote * (double)( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed);
+            double W_DSpeed_note = W_ratioNote * (double)( 20 + 2*cLevel) + log((double)testResult.dSpeed);
+            double O_DSpeed_note = O_ratioNote * (double)( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed);
 
             if (W_DMemUsed_note < O_DMemUsed_note) {
                 /* uses too much Decompression memory for too little benefit */
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index dd8472d39..aef29be0b 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -388,7 +388,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
                     marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
-                    (double)srcSize / fastestC );
+                    (double)srcSize / (double)fastestC );
 
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
 #if 1
@@ -527,8 +527,8 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
                     marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
-                    (double)srcSize / fastestC,
-                    (double)srcSize / fastestD );
+                    (double)srcSize / (double)fastestC,
+                    (double)srcSize / (double)fastestD );
 
             /* CRC Checking */
             {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -558,8 +558,8 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
         }   /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
 
         if (g_displayLevel == 1) {
-            double cSpeed = (double)srcSize / fastestC;
-            double dSpeed = (double)srcSize / fastestD;
+            double cSpeed = (double)srcSize / (double)fastestC;
+            double dSpeed = (double)srcSize / (double)fastestD;
             if (g_additionalParam)
                 DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
             else

From 4b40e405d355f67cac166db8fdd901c34b9a228e Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Dec 2022 17:48:24 -0800
Subject: [PATCH 075/197] [tests] Remove deprecated function from longmatch.c
 test

Thanks to @eli-schwartz for pointing it out!

We should maybe consider adding a helper function for applying
`ZSTD_parameters` and `ZSTD_compressionParameters` to a context.
That would aid the transition to the new API in situations like this.
---
 tests/longmatch.c | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/tests/longmatch.c b/tests/longmatch.c
index 36b10df76..547b26190 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -36,27 +36,27 @@ compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size)
 int main(int argc, const char** argv)
 {
   ZSTD_CStream* ctx;
-  ZSTD_parameters params;
-  size_t rc;
-  unsigned windowLog;
+  unsigned windowLog = 18;
   (void)argc;
   (void)argv;
   /* Create stream */
-  ctx = ZSTD_createCStream();
+  ctx = ZSTD_createCCtx();
   if (!ctx) { return 1; }
   /* Set parameters */
-  memset(&params, 0, sizeof(params));
-  params.cParams.windowLog = 18;
-  params.cParams.chainLog = 13;
-  params.cParams.hashLog = 14;
-  params.cParams.searchLog = 1;
-  params.cParams.minMatch = 7;
-  params.cParams.targetLength = 16;
-  params.cParams.strategy = ZSTD_fast;
-  windowLog = params.cParams.windowLog;
-  /* Initialize stream */
-  rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0);
-  if (ZSTD_isError(rc)) { return 2; }
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, windowLog)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, 13)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, 14)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, 1)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, 7)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, 16)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, ZSTD_fast)))
+    return 2;
   {
     U64 compressed = 0;
     const U64 toCompress = ((U64)1) << 33;
@@ -97,5 +97,6 @@ int main(int argc, const char** argv)
     free(srcBuffer);
     free(dstBuffer);
   }
+  ZSTD_freeCCtx(ctx);
   return 0;
 }

From 7fe7a166c29ecb68b9b19b2d595b138ae2416703 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Dec 2022 17:36:27 -0800
Subject: [PATCH 076/197] [cli-tests] Add tests that use --trace-file-stat

Basic tests for (de)compressing in the following modes:
* file to file
* file to stdout
* stdin to file
* stdin to stdout

These are basic tests, and aren't testing more advanced scenarios, but
it adds the groundwork for more complex tests as needed.

Fixes #3010.
---
 .../file-stat/compress-file-to-file.sh        |  8 +++++
 .../compress-file-to-file.sh.stderr.exact     | 36 +++++++++++++++++++
 .../file-stat/compress-file-to-stdout.sh      |  8 +++++
 .../compress-file-to-stdout.sh.stderr.exact   | 22 ++++++++++++
 .../file-stat/compress-stdin-to-file.sh       |  8 +++++
 .../compress-stdin-to-file.sh.stderr.exact    | 28 +++++++++++++++
 .../file-stat/compress-stdin-to-stdout.sh     |  8 +++++
 .../compress-stdin-to-stdout.sh.stderr.exact  | 20 +++++++++++
 .../file-stat/decompress-file-to-file.sh      |  7 ++++
 .../decompress-file-to-file.sh.stderr.exact   | 30 ++++++++++++++++
 .../file-stat/decompress-file-to-stdout.sh    |  7 ++++
 .../decompress-file-to-stdout.sh.stderr.exact | 14 ++++++++
 .../file-stat/decompress-stdin-to-file.sh     |  7 ++++
 .../decompress-stdin-to-file.sh.stderr.exact  | 20 +++++++++++
 .../file-stat/decompress-stdin-to-stdout.sh   |  7 ++++
 ...decompress-stdin-to-stdout.sh.stderr.exact | 12 +++++++
 16 files changed, 242 insertions(+)
 create mode 100755 tests/cli-tests/file-stat/compress-file-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/compress-file-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/compress-stdin-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-file-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-file-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-stdin-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact

diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh b/tests/cli-tests/file-stat/compress-file-to-file.sh
new file mode 100755
index 000000000..949f34ff1
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
new file mode 100644
index 000000000..34bb704c7
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -0,0 +1,36 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_isDirectory(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file, file.zst)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_utime(file.zst)
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh b/tests/cli-tests/file-stat/compress-file-to-stdout.sh
new file mode 100755
index 000000000..99ebfc46f
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..e01922f84
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,22 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_isDirectory(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-file.sh b/tests/cli-tests/file-stat/compress-stdin-to-file.sh
new file mode 100755
index 000000000..837946198
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 000000000..dce9005f4
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,28 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
new file mode 100755
index 000000000..64f4b0362
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..d47bdcf09
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,20 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh b/tests/cli-tests/file-stat/decompress-file-to-file.sh
new file mode 100755
index 000000000..3e08c247b
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dq --trace-file-stat file.zst
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
new file mode 100644
index 000000000..a2d7ad282
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -0,0 +1,30 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file.zst, file)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_utime(file)
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
new file mode 100755
index 000000000..518c2a950
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat file.zst > file
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..87fc5e97c
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,14 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-file.sh b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
new file mode 100755
index 000000000..135d75524
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst -o file
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 000000000..749fd395a
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,20 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
new file mode 100755
index 000000000..495f07b17
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst > file
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..61487f61e
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,12 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0

From 382026f09646867500819652ff27cfa47e1e0768 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Dec 2022 14:04:36 -0500
Subject: [PATCH 077/197] Man Page Tweaks, Edits, Formatting Fixes

This started as an application of the edits suggested in #3201 and expanded
from there.
---
 programs/zstd.1.md     | 142 +++++++++++++++++++++++------------------
 programs/zstdgrep.1.md |  10 +--
 programs/zstdless.1.md |   6 +-
 3 files changed, 88 insertions(+), 70 deletions(-)

diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 37c2ba187..45a88a347 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -4,7 +4,7 @@ zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
 SYNOPSIS
 --------
 
-`zstd` [*OPTIONS*] [-|_INPUT-FILE_] [-o _OUTPUT-FILE_]
+`zstd` [<OPTIONS>] [-|<INPUT-FILE>] [-o <OUTPUT-FILE>]
 
 `zstdmt` is equivalent to `zstd -T0`
 
@@ -16,7 +16,7 @@ SYNOPSIS
 DESCRIPTION
 -----------
 `zstd` is a fast lossless compression algorithm and data compression tool,
-with command line syntax similar to `gzip (1)` and `xz (1)`.
+with command line syntax similar to `gzip`(1) and `xz`(1).
 It is based on the **LZ77** family, with further FSE & huff0 entropy stages.
 `zstd` offers highly configurable compression speed,
 from fast modes at > 200 MB/s per core,
@@ -24,7 +24,7 @@ to strong modes with excellent compression ratios.
 It also features a very fast decoder, with speeds > 500 MB/s per core.
 
 `zstd` command line syntax is generally similar to gzip,
-but features the following differences :
+but features the following differences:
 
   - Source files are preserved by default.
     It's possible to remove them automatically by using the `--rm` command.
@@ -35,12 +35,13 @@ but features the following differences :
     Use `-q` to turn it off.
   - `zstd` does not accept input from console,
     though it does accept `stdin` when it's not the console.
+  - `zstd` does not store the input's filename or attributes, only its contents.
 
 `zstd` processes each _file_ according to the selected operation mode.
 If no _files_ are given or _file_ is `-`, `zstd` reads from standard input
 and writes the processed data to standard output.
 `zstd` will refuse to write compressed data to standard output
-if it is a terminal : it will display an error message and skip the _file_.
+if it is a terminal: it will display an error message and skip the file.
 Similarly, `zstd` will refuse to read compressed data from standard input
 if it is a terminal.
 
@@ -52,14 +53,15 @@ whose name is derived from the source _file_ name:
 * When decompressing, the `.zst` suffix is removed from the source filename to
   get the target filename
 
-### Concatenation with .zst files
+### Concatenation with .zst Files
 It is possible to concatenate multiple `.zst` files. `zstd` will decompress
 such agglomerated file as if it was a single `.zst` file.
 
 OPTIONS
 -------
 
-### Integer suffixes and special values
+### Integer Suffixes and Special Values
+
 In most places where an integer argument is expected,
 an optional suffix is supported to easily indicate large integers.
 There must be no space between the integer and the suffix.
@@ -71,7 +73,8 @@ There must be no space between the integer and the suffix.
     Multiply the integer by 1,048,576 (2\^20).
     `Mi`, `M`, and `MB` are accepted as synonyms for `MiB`.
 
-### Operation mode
+### Operation Mode
+
 If multiple operation mode options are given,
 the last one takes effect.
 
@@ -88,19 +91,21 @@ the last one takes effect.
     decompressed data is discarded and checksummed for errors.
     No files are created or removed.
 * `-b#`:
-    Benchmark file(s) using compression level #
-* `--train FILEs`:
-    Use FILEs as a training set to create a dictionary.
+    Benchmark file(s) using compression level _#_.
+    See _BENCHMARK_ below for a description of this operation.
+* `--train FILES`:
+    Use _FILES_ as a training set to create a dictionary.
     The training set should contain a lot of small files (> 100).
+    See _DICTIONARY BUILDER_ below for a description of this operation.
 * `-l`, `--list`:
     Display information related to a zstd compressed file, such as size, ratio, and checksum.
     Some of these fields may not be available.
     This command's output can be augmented with the `-v` modifier.
 
-### Operation modifiers
+### Operation Modifiers
 
 * `-#`:
-    `#` compression level \[1-19] (default: 3)
+    selects `#` compression level \[1-19\] (default: 3)
 * `--ultra`:
     unlocks high compression levels 20+ (maximum 22), using a lot more memory.
     Note that decompression will also require more memory when using these levels.
@@ -122,21 +127,24 @@ the last one takes effect.
     As compression is serialized with I/O, this can be slightly slower.
     Single-thread mode features significantly lower memory usage,
     which can be useful for systems with limited amount of memory, such as 32-bit systems.
-    Note 1 : this mode is the only available one when multithread support is disabled.
-    Note 2 : this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
+
+    Note 1: this mode is the only available one when multithread support is disabled.
+
+    Note 2: this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
     Final compressed result is also slightly different from `-T1`.
 * `--auto-threads={physical,logical} (default: physical)`:
     When using a default amount of threads via `-T0`, choose the default based on the number
     of detected physical or logical cores.
-* `--adapt[=min=#,max=#]` :
+* `--adapt[=min=#,max=#]`:
     `zstd` will dynamically adapt compression level to perceived I/O conditions.
     Compression level adaptation can be observed live by using command `-v`.
     Adaptation can be constrained between supplied `min` and `max` levels.
     The feature works when combined with multi-threading and `--long` mode.
     It does not work with `--single-thread`.
-    It sets window size to 8 MB by default (can be changed manually, see `wlog`).
+    It sets window size to 8 MiB by default (can be changed manually, see `wlog`).
     Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
-    _note_ : at the time of this writing, `--adapt` can remain stuck at low speed
+
+    _Note_: at the time of this writing, `--adapt` can remain stuck at low speed
     when combined with multiple worker threads (>=2).
 * `--long[=#]`:
     enables long distance matching with `#` `windowLog`, if `#` is not
@@ -153,18 +161,21 @@ the last one takes effect.
 * `--patch-from FILE`:
     Specify the file to be used as a reference point for zstd's diff engine.
     This is effectively dictionary compression with some convenient parameter
-    selection, namely that windowSize > srcSize.
+    selection, namely that _windowSize_ > _srcSize_.
 
-    Note: cannot use both this and -D together
-    Note: `--long` mode will be automatically activated if chainLog < fileLog
-        (fileLog being the windowLog required to cover the whole file). You
+    Note: cannot use both this and `-D` together.
+
+    Note: `--long` mode will be automatically activated if _chainLog_ < _fileLog_
+        (_fileLog_ being the _windowLog_ required to cover the whole file). You
         can also manually force it.
-    Note: for all levels, you can use --patch-from in --single-thread mode
-        to improve compression ratio at the cost of speed
+
+    Note: for all levels, you can use `--patch-from` in `--single-thread` mode
+        to improve compression ratio at the cost of speed.
+
     Note: for level 19, you can get increased compression ratio at the cost
         of speed by specifying `--zstd=targetLength=` to be something large
-        (i.e. 4096), and by setting a large `--zstd=chainLog=`
-* `--rsyncable` :
+        (i.e. 4096), and by setting a large `--zstd=chainLog=`.
+* `--rsyncable`:
     `zstd` will periodically synchronize the compression state to make the
     compressed file more rsync-friendly. There is a negligible impact to
     compression ratio, and the faster compression levels will see a small
@@ -177,24 +188,24 @@ the last one takes effect.
 * `--[no-]content-size`:
     enable / disable whether or not the original size of the file is placed in
     the header of the compressed file. The default option is
-    --content-size (meaning that the original size will be placed in the header).
+    `--content-size` (meaning that the original size will be placed in the header).
 * `--no-dictID`:
     do not store dictionary ID within frame header (dictionary compression).
     The decoder will have to rely on implicit knowledge about which dictionary to use,
     it won't be able to check if it's correct.
 * `-M#`, `--memory=#`:
-    Set a memory usage limit. By default, `zstd` uses 128 MB for decompression
+    Set a memory usage limit. By default, `zstd` uses 128 MiB for decompression
     as the maximum amount of memory the decompressor is allowed to use, but you can
     override this manually if need be in either direction (i.e. you can increase or
     decrease it).
 
-    This is also used during compression when using with --patch-from=. In this case,
-    this parameter overrides that maximum size allowed for a dictionary. (128 MB).
+    This is also used during compression when using with `--patch-from=`. In this case,
+    this parameter overrides that maximum size allowed for a dictionary. (128 MiB).
 
     Additionally, this can be used to limit memory for dictionary training. This parameter
-    overrides the default limit of 2 GB. zstd will load training samples up to the memory limit
+    overrides the default limit of 2 GiB. zstd will load training samples up to the memory limit
     and ignore the rest.
-* `--stream-size=#` :
+* `--stream-size=#`:
     Sets the pledged source size of input coming from a stream. This value must be exact, as it
     will be included in the produced frame header. Incorrect stream sizes will cause an error.
     This information will be used to better optimize compression parameters, resulting in
@@ -207,7 +218,7 @@ the last one takes effect.
     Exact guesses result in better compression ratios. Overestimates result in slightly
     degraded compression ratios, while underestimates may result in significant degradation.
 * `-o FILE`:
-    save result into `FILE`
+    save result into `FILE`.
 * `-f`, `--force`:
     disable input and output checks. Allows overwriting existing files, input
     from console, output to stdout, operating on links, block devices, etc.
@@ -227,11 +238,11 @@ the last one takes effect.
     enable / disable passing through uncompressed files as-is. During
     decompression when pass-through is enabled, unrecognized formats will be
     copied as-is from the input to the output. By default, pass-through will
-    occur when the output destination is stdout and the force (-f) option is
+    occur when the output destination is stdout and the force (`-f`) option is
     set.
 * `--rm`:
     remove source file(s) after successful compression or decompression. If used in combination with
-    -o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
+    `-o`, will trigger a confirmation prompt (which can be silenced with `-f`), as this is a destructive operation.
 * `-k`, `--keep`:
     keep source file(s) after successful compression or decompression.
     This is the default behavior.
@@ -270,7 +281,7 @@ the last one takes effect.
     display help/long help and exit
 * `-V`, `--version`:
     display version number and exit.
-    Advanced : `-vV` also displays supported formats.
+    Advanced: `-vV` also displays supported formats.
     `-vvV` also displays POSIX support.
     `-q` will only display the version number, suitable for machine reading.
 * `-v`, `--verbose`:
@@ -281,15 +292,13 @@ the last one takes effect.
 * `--no-progress`:
     do not display the progress bar, but keep all other messages.
 * `--show-default-cparams`:
-    Shows the default compression parameters that will be used for a
-    particular src file. If the provided src file is not a regular file
-    (e.g. named pipe), the cli will just output the default parameters.
-    That is, the parameters that are used when the src size is unknown.
+    shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size.
+    If the provided file is not a regular file (e.g. a pipe), this flag will output the parameters used for inputs of unknown size.
 * `--`:
     All arguments after `--` are treated as files
 
 
-### gzip Operation modifiers
+### gzip Operation Modifiers
 When invoked via a `gzip` symlink, `zstd` will support further
 options that intend to mimic the `gzip` behavior:
 
@@ -300,7 +309,7 @@ options that intend to mimic the `gzip` behavior:
     alias to the option `-9`.
 
 
-### Interactions with Environment Variables
+### Environment Variables
 
 Employing environment variables to set parameters has security implications.
 Therefore, this avenue is intentionally limited.
@@ -341,7 +350,7 @@ Compression of small files similar to the sample set will be greatly improved.
     Since dictionary compression is mostly effective for small files,
     the expectation is that the training set will only contain small files.
     In the case where some samples happen to be large,
-    only the first 128 KB of these samples will be used for training.
+    only the first 128 KiB of these samples will be used for training.
 
     `--train` supports multithreading if `zstd` is compiled with threading support (default).
     Additional advanced parameters can be specified with `--train-fastcover`.
@@ -389,11 +398,13 @@ Compression of small files similar to the sample set will be greatly improved.
     It's possible to provide an explicit number ID instead.
     It's up to the dictionary manager to not assign twice the same ID to
     2 different dictionaries.
-    Note that short numbers have an advantage :
+    Note that short numbers have an advantage:
     an ID < 256 will only need 1 byte in the compressed frame header,
     and an ID < 65536 will only need 2 bytes.
     This compares favorably to 4 bytes default.
 
+    Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2\^31, so they should not be used in public.
+
 * `--train-cover[=k#,d=#,steps=#,split=#,shrink[=#]]`:
     Select parameters for the default dictionary builder algorithm named cover.
     If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
@@ -482,7 +493,7 @@ BENCHMARK
 * `--priority=rt`:
     set process priority to real-time
 
-**Output Format:** CompressionLevel#Filename : InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+**Output Format:** CompressionLevel#Filename: InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
 
 **Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
 
@@ -499,9 +510,10 @@ This minimum is either 512 KB, or `overlapSize`, whichever is largest.
 Different job sizes will lead to non-identical compressed frames.
 
 ### --zstd[=options]:
-`zstd` provides 22 predefined compression levels.
-The selected or default predefined compression level can be changed with
-advanced compression options.
+`zstd` provides 22 predefined regular compression levels plus the fast levels.
+This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor.
+(You can see the result of this translation with `--show-default-cparams`.)
+These specific parameters can be overridden with advanced compression options.
 The _options_ are provided as a comma-separated list.
 You may specify only the options you want to change and the rest will be
 taken from the selected or default compression level.
@@ -510,10 +522,10 @@ The list of available _options_:
 - `strategy`=_strat_, `strat`=_strat_:
     Specify a strategy used by a match finder.
 
-    There are 9 strategies numbered from 1 to 9, from faster to stronger:
-    1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
-    4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
-    7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
+    There are 9 strategies numbered from 1 to 9, from fastest to strongest:
+    1=`ZSTD_fast`, 2=`ZSTD_dfast`, 3=`ZSTD_greedy`,
+    4=`ZSTD_lazy`, 5=`ZSTD_lazy2`, 6=`ZSTD_btlazy2`,
+    7=`ZSTD_btopt`, 8=`ZSTD_btultra`, 9=`ZSTD_btultra2`.
 
 - `windowLog`=_wlog_, `wlog`=_wlog_:
     Specify the maximum number of bits for a match distance.
@@ -533,19 +545,20 @@ The list of available _options_:
     Bigger hash tables cause fewer collisions which usually makes compression
     faster, but requires more memory during compression.
 
-    The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
+    The minimum _hlog_ is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB).
 
 - `chainLog`=_clog_, `clog`=_clog_:
-    Specify the maximum number of bits for a hash chain or a binary tree.
+    Specify the maximum number of bits for the secondary search structure,
+    whose form depends on the selected `strategy`.
 
     Higher numbers of bits increases the chance to find a match which usually
     improves compression ratio.
     It also slows down compression speed and increases memory requirements for
     compression.
-    This option is ignored for the ZSTD_fast strategy.
+    This option is ignored for the `ZSTD_fast` `strategy`, which only has the primary hash table.
 
-    The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
-    and 30 (1 Gib) on 64-bit platforms.
+    The minimum _clog_ is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32-bit platforms
+    and 30 (1B entries / 4 GiB) on 64-bit platforms.
 
 - `searchLog`=_slog_, `slog`=_slog_:
     Specify the maximum number of searches in a hash chain or a binary tree
@@ -567,19 +580,19 @@ The list of available _options_:
 - `targetLength`=_tlen_, `tlen`=_tlen_:
     The impact of this field vary depending on selected strategy.
 
-    For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
+    For `ZSTD_btopt`, `ZSTD_btultra` and `ZSTD_btultra2`, it specifies
     the minimum match length that causes match finder to stop searching.
     A larger `targetLength` usually improves compression ratio
     but decreases compression speed.
-t
-    For ZSTD\_fast, it triggers ultra-fast mode when > 0.
+
+    For `ZSTD_fast`, it triggers ultra-fast mode when > 0.
     The value represents the amount of data skipped between match sampling.
-    Impact is reversed : a larger `targetLength` increases compression speed
+    Impact is reversed: a larger `targetLength` increases compression speed
     but decreases compression ratio.
 
     For all other strategies, this field has no impact.
 
-    The minimum _tlen_ is 0 and the maximum is 128 Kib.
+    The minimum _tlen_ is 0 and the maximum is 128 KiB.
 
 - `overlapLog`=_ovlog_,  `ovlog`=_ovlog_:
     Determine `overlapSize`, amount of data reloaded from previous job.
@@ -591,7 +604,7 @@ t
     9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
     Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
     For example, 8 means "windowSize/2", and 6 means "windowSize/8".
-    Value 0 is special and means "default" : _ovlog_ is automatically determined by `zstd`.
+    Value 0 is special and means "default": _ovlog_ is automatically determined by `zstd`.
     In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
 
 - `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
@@ -641,6 +654,11 @@ similar to predefined level 19 for files bigger than 256 KB:
 
 `--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
 
+SEE ALSO
+--------
+`zstdgrep`(1), `zstdless`(1), `gzip`(1), `xz`(1)
+
+The <zstandard> format is specified in Y. Collet, "Zstandard Compression and the 'application/zstd' Media Type", https://www.ietf.org/rfc/rfc8878.txt, Internet RFC 8878 (February 2021).
 
 BUGS
 ----
diff --git a/programs/zstdgrep.1.md b/programs/zstdgrep.1.md
index 35186a4bf..6370a81c7 100644
--- a/programs/zstdgrep.1.md
+++ b/programs/zstdgrep.1.md
@@ -4,16 +4,16 @@ zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files
 SYNOPSIS
 --------
 
-`zstdgrep` [*grep-flags*] [--] _pattern_ [_files_ ...]
+`zstdgrep` [<grep-flags>] [--] <pattern> [<files> ...]
 
 
 DESCRIPTION
 -----------
-`zstdgrep` runs `grep (1)` on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat (1)`.
+`zstdgrep` runs `grep`(1) on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat`(1).
 
-The grep-flags and pattern arguments are passed on to `grep (1)`.  If an `-e` flag is found in the `grep-flags`, `zstdgrep` will not look for a pattern argument.
+The <grep-flags> and <pattern> arguments are passed on to `grep`(1).  If an `-e` flag is found in the <grep-flags>, `zstdgrep` will not look for a <pattern> argument.
 
-Note that modern `grep` alternatives such as `ripgrep` (`rg`) support `zstd`-compressed files out of the box,
+Note that modern `grep` alternatives such as `ripgrep` (`rg`(1)) support `zstd`-compressed files out of the box,
 and can prove better alternatives than `zstdgrep` notably for unsupported complex pattern searches.
 Note though that such alternatives may also feature some minor command line differences.
 
@@ -23,7 +23,7 @@ In case of missing arguments or missing pattern, 1 will be returned, otherwise 0
 
 SEE ALSO
 --------
-`zstd (1)`
+`zstd`(1)
 
 AUTHORS
 -------
diff --git a/programs/zstdless.1.md b/programs/zstdless.1.md
index d91d48abc..67c1c7676 100644
--- a/programs/zstdless.1.md
+++ b/programs/zstdless.1.md
@@ -4,13 +4,13 @@ zstdless(1) -- view zstandard-compressed files
 SYNOPSIS
 --------
 
-`zstdless` [*flags*] [_file_ ...]
+`zstdless` [<flags>] [<file> ...]
 
 
 DESCRIPTION
 -----------
-`zstdless` runs `less (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
+`zstdless` runs `less`(1) on files or stdin, if no <file> argument is given, after decompressing them with `zstdcat`(1).
 
 SEE ALSO
 --------
-`zstd (1)`
+`zstd`(1)

From 11aba9b316506525a3a1d368b3e6edb6dcb044ff Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Dec 2022 14:13:24 -0500
Subject: [PATCH 078/197] `make man`

---
 programs/zstd.1 | 115 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 74 insertions(+), 41 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index 5cf3c6bb1..e81ac9615 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "August 2022" "zstd 1.5.3" "User Commands"
+.TH "ZSTD" "1" "December 2022" "zstd 1.5.3" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -17,10 +17,10 @@
 \fBzstdcat\fR is equivalent to \fBzstd \-dcf\fR
 .
 .SH "DESCRIPTION"
-\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip (1)\fR and \fBxz (1)\fR\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
+\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip\fR(1) and \fBxz\fR(1)\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
 .
 .P
-\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences :
+\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences:
 .
 .IP "\(bu" 4
 Source files are preserved by default\. It\'s possible to remove them automatically by using the \fB\-\-rm\fR command\.
@@ -34,10 +34,13 @@ When compressing a single file, \fBzstd\fR displays progress notifications and r
 .IP "\(bu" 4
 \fBzstd\fR does not accept input from console, though it does accept \fBstdin\fR when it\'s not the console\.
 .
+.IP "\(bu" 4
+\fBzstd\fR does not store the input\'s filename or attributes, only its contents\.
+.
 .IP "" 0
 .
 .P
-\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal : it will display an error message and skip the \fIfile\fR\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
+\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal: it will display an error message and skip the file\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
 .
 .P
 Unless \fB\-\-stdout\fR or \fB\-o\fR is specified, \fIfiles\fR are written to a new file whose name is derived from the source \fIfile\fR name:
@@ -50,12 +53,12 @@ When decompressing, the \fB\.zst\fR suffix is removed from the source filename t
 .
 .IP "" 0
 .
-.SS "Concatenation with \.zst files"
+.SS "Concatenation with \.zst Files"
 It is possible to concatenate multiple \fB\.zst\fR files\. \fBzstd\fR will decompress such agglomerated file as if it was a single \fB\.zst\fR file\.
 .
 .SH "OPTIONS"
 .
-.SS "Integer suffixes and special values"
+.SS "Integer Suffixes and Special Values"
 In most places where an integer argument is expected, an optional suffix is supported to easily indicate large integers\. There must be no space between the integer and the suffix\.
 .
 .TP
@@ -66,7 +69,7 @@ Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accep
 \fBMiB\fR
 Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
 .
-.SS "Operation mode"
+.SS "Operation Mode"
 If multiple operation mode options are given, the last one takes effect\.
 .
 .TP
@@ -83,20 +86,20 @@ Test the integrity of compressed \fIfiles\fR\. This option is equivalent to \fB\
 .
 .TP
 \fB\-b#\fR
-Benchmark file(s) using compression level #
+Benchmark file(s) using compression level \fI#\fR\. See \fIBENCHMARK\fR below for a description of this operation\.
 .
 .TP
-\fB\-\-train FILEs\fR
-Use FILEs as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\.
+\fB\-\-train FILES\fR
+Use \fIFILES\fR as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\. See \fIDICTIONARY BUILDER\fR below for a description of this operation\.
 .
 .TP
 \fB\-l\fR, \fB\-\-list\fR
 Display information related to a zstd compressed file, such as size, ratio, and checksum\. Some of these fields may not be available\. This command\'s output can be augmented with the \fB\-v\fR modifier\.
 .
-.SS "Operation modifiers"
+.SS "Operation Modifiers"
 .
 .IP "\(bu" 4
-\fB\-#\fR: \fB#\fR compression level [1\-19] (default: 3)
+\fB\-#\fR: selects \fB#\fR compression level [1\-19] (default: 3)
 .
 .IP "\(bu" 4
 \fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
@@ -108,13 +111,22 @@ Display information related to a zstd compressed file, such as size, ratio, and
 \fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to \fBZSTDMT_NBWORKERS_MAX\fR, which is either 64 in 32\-bit mode, or 256 for 64\-bit environments\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
 .
 .IP "\(bu" 4
-\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\. Note 1 : this mode is the only available one when multithread support is disabled\. Note 2 : this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
+\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\.
+.
+.IP
+Note 1: this mode is the only available one when multithread support is disabled\.
+.
+.IP
+Note 2: this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
 .
 .IP "\(bu" 4
 \fB\-\-auto\-threads={physical,logical} (default: physical)\fR: When using a default amount of threads via \fB\-T0\fR, choose the default based on the number of detected physical or logical cores\.
 .
 .IP "\(bu" 4
-\fB\-\-adapt[=min=#,max=#]\fR : \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
+\fB\-\-adapt[=min=#,max=#]\fR: \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MiB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\.
+.
+.IP
+\fINote\fR: at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
 .
 .IP "\(bu" 4
 \fB\-\-long[=#]\fR: enables long distance matching with \fB#\fR \fBwindowLog\fR, if \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
@@ -126,40 +138,49 @@ Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \
 \fB\-D DICT\fR: use \fBDICT\fR as Dictionary to compress or decompress FILE(s)
 .
 .IP "\(bu" 4
-\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize\.
+\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that \fIwindowSize\fR > \fIsrcSize\fR\.
 .
 .IP
-Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be automatically activated if chainLog < fileLog (fileLog being the windowLog required to cover the whole file)\. You can also manually force it\. Note: for all levels, you can use \-\-patch\-from in \-\-single\-thread mode to improve compression ratio at the cost of speed Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR
+Note: cannot use both this and \fB\-D\fR together\.
+.
+.IP
+Note: \fB\-\-long\fR mode will be automatically activated if \fIchainLog\fR < \fIfileLog\fR (\fIfileLog\fR being the \fIwindowLog\fR required to cover the whole file)\. You can also manually force it\.
+.
+.IP
+Note: for all levels, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread\fR mode to improve compression ratio at the cost of speed\.
+.
+.IP
+Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR\.
 .
 .IP "\(bu" 4
-\fB\-\-rsyncable\fR : \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
+\fB\-\-rsyncable\fR: \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
 .
 .IP "\(bu" 4
 \fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled)
 .
 .IP "\(bu" 4
-\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \-\-content\-size (meaning that the original size will be placed in the header)\.
+\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \fB\-\-content\-size\fR (meaning that the original size will be placed in the header)\.
 .
 .IP "\(bu" 4
 \fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
 .
 .IP "\(bu" 4
-\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
+\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MiB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
 .
 .IP
-This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\.
+This is also used during compression when using with \fB\-\-patch\-from=\fR\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MiB)\.
 .
 .IP
-Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GB\. zstd will load training samples up to the memory limit and ignore the rest\.
+Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GiB\. zstd will load training samples up to the memory limit and ignore the rest\.
 .
 .IP "\(bu" 4
-\fB\-\-stream\-size=#\fR : Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
+\fB\-\-stream\-size=#\fR: Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
 .
 .IP "\(bu" 4
 \fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
 .
 .IP "\(bu" 4
-\fB\-o FILE\fR: save result into \fBFILE\fR
+\fB\-o FILE\fR: save result into \fBFILE\fR\.
 .
 .IP "\(bu" 4
 \fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
@@ -171,10 +192,10 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
 \fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
 .
 .IP "\(bu" 4
-\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\-f) option is set\.
+\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\fB\-f\fR) option is set\.
 .
 .IP "\(bu" 4
-\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\.
+\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \fB\-o\fR, will trigger a confirmation prompt (which can be silenced with \fB\-f\fR), as this is a destructive operation\.
 .
 .IP "\(bu" 4
 \fB\-k\fR, \fB\-\-keep\fR: keep source file(s) after successful compression or decompression\. This is the default behavior\.
@@ -201,7 +222,7 @@ If input directory contains "\.\.", the files in this directory will be ignored\
 \fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit
 .
 .IP "\(bu" 4
-\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
+\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced: \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
 .
 .IP "\(bu" 4
 \fB\-v\fR, \fB\-\-verbose\fR: verbose mode, display more information
@@ -213,14 +234,14 @@ If input directory contains "\.\.", the files in this directory will be ignored\
 \fB\-\-no\-progress\fR: do not display the progress bar, but keep all other messages\.
 .
 .IP "\(bu" 4
-\fB\-\-show\-default\-cparams\fR: Shows the default compression parameters that will be used for a particular src file\. If the provided src file is not a regular file (e\.g\. named pipe), the cli will just output the default parameters\. That is, the parameters that are used when the src size is unknown\.
+\fB\-\-show\-default\-cparams\fR: shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size\. If the provided file is not a regular file (e\.g\. a pipe), this flag will output the parameters used for inputs of unknown size\.
 .
 .IP "\(bu" 4
 \fB\-\-\fR: All arguments after \fB\-\-\fR are treated as files
 .
 .IP "" 0
 .
-.SS "gzip Operation modifiers"
+.SS "gzip Operation Modifiers"
 When invoked via a \fBgzip\fR symlink, \fBzstd\fR will support further options that intend to mimic the \fBgzip\fR behavior:
 .
 .TP
@@ -231,7 +252,7 @@ do not store the original filename and timestamps when compressing a file\. This
 \fB\-\-best\fR
 alias to the option \fB\-9\fR\.
 .
-.SS "Interactions with Environment Variables"
+.SS "Environment Variables"
 Employing environment variables to set parameters has security implications\. Therefore, this avenue is intentionally limited\. Only \fBZSTD_CLEVEL\fR and \fBZSTD_NBTHREADS\fR are currently supported\. They set the compression level and number of threads to use during compression, respectively\.
 .
 .P
@@ -251,7 +272,7 @@ They can both be overridden by corresponding command line arguments: \fB\-#\fR f
 Use FILEs as training set to create a dictionary\. The training set should ideally contain a lot of samples (> 100), and weight typically 100x the target dictionary size (for example, ~10 MB for a 100 KB dictionary)\. \fB\-\-train\fR can be combined with \fB\-r\fR to indicate a directory rather than listing all the files, which can be useful to circumvent shell expansion limits\.
 .
 .IP
-Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KB of these samples will be used for training\.
+Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KiB of these samples will be used for training\.
 .
 .IP
 \fB\-\-train\fR supports multithreading if \fBzstd\fR is compiled with threading support (default)\. Additional advanced parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The slower cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Default \fB\-\-train\fR is equivalent to \fB\-\-train\-fastcover=d=8,steps=4\fR\.
@@ -281,7 +302,10 @@ In situations where the training set is larger than maximum memory, the CLI will
 .
 .TP
 \fB\-\-dictID=#\fR
-A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
+A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage: an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
+.
+.IP
+Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2^31, so they should not be used in public\.
 .
 .TP
 \fB\-\-train\-cover[=k#,d=#,steps=#,split=#,shrink[=#]]\fR
@@ -366,7 +390,7 @@ cut file(s) into independent chunks of size # (default: no chunking)
 set process priority to real\-time
 .
 .P
-\fBOutput Format:\fR CompressionLevel#Filename : InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+\fBOutput Format:\fR CompressionLevel#Filename: InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
 .
 .P
 \fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
@@ -377,14 +401,14 @@ set process priority to real\-time
 Specify the size of each compression job\. This parameter is only available when multi\-threading is enabled\. Each compression job is run in parallel, so this value indirectly impacts the nb of active threads\. Default job size varies depending on compression level (generally \fB4 * windowSize\fR)\. \fB\-B#\fR makes it possible to manually select a custom size\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 512 KB, or \fBoverlapSize\fR, whichever is largest\. Different job sizes will lead to non\-identical compressed frames\.
 .
 .SS "\-\-zstd[=options]:"
-\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
+\fBzstd\fR provides 22 predefined regular compression levels plus the fast levels\. This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor\. (You can see the result of this translation with \fB\-\-show\-default\-cparams\fR\.) These specific parameters can be overridden with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
 .
 .TP
 \fBstrategy\fR=\fIstrat\fR, \fBstrat\fR=\fIstrat\fR
 Specify a strategy used by a match finder\.
 .
 .IP
-There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
+There are 9 strategies numbered from 1 to 9, from fastest to strongest: 1=\fBZSTD_fast\fR, 2=\fBZSTD_dfast\fR, 3=\fBZSTD_greedy\fR, 4=\fBZSTD_lazy\fR, 5=\fBZSTD_lazy2\fR, 6=\fBZSTD_btlazy2\fR, 7=\fBZSTD_btopt\fR, 8=\fBZSTD_btultra\fR, 9=\fBZSTD_btultra2\fR\.
 .
 .TP
 \fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@@ -404,17 +428,17 @@ Specify the maximum number of bits for a hash table\.
 Bigger hash tables cause fewer collisions which usually makes compression faster, but requires more memory during compression\.
 .
 .IP
-The minimum \fIhlog\fR is 6 (64 B) and the maximum is 30 (1 GiB)\.
+The minimum \fIhlog\fR is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB)\.
 .
 .TP
 \fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR
-Specify the maximum number of bits for a hash chain or a binary tree\.
+Specify the maximum number of bits for the secondary search structure, whose form depends on the selected \fBstrategy\fR\.
 .
 .IP
-Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the ZSTD_fast strategy\.
+Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the \fBZSTD_fast\fR \fBstrategy\fR, which only has the primary hash table\.
 .
 .IP
-The minimum \fIclog\fR is 6 (64 B) and the maximum is 29 (524 Mib) on 32\-bit platforms and 30 (1 Gib) on 64\-bit platforms\.
+The minimum \fIclog\fR is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32\-bit platforms and 30 (1B entries / 4 GiB) on 64\-bit platforms\.
 .
 .TP
 \fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR
@@ -441,20 +465,23 @@ The minimum \fImml\fR is 3 and the maximum is 7\.
 The impact of this field vary depending on selected strategy\.
 .
 .IP
-For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\. t For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
+For \fBZSTD_btopt\fR, \fBZSTD_btultra\fR and \fBZSTD_btultra2\fR, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\.
+.
+.IP
+For \fBZSTD_fast\fR, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed: a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
 .
 .IP
 For all other strategies, this field has no impact\.
 .
 .IP
-The minimum \fItlen\fR is 0 and the maximum is 128 Kib\.
+The minimum \fItlen\fR is 0 and the maximum is 128 KiB\.
 .
 .TP
 \fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR
 Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
 .
 .IP
-The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default" : \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
+The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default": \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
 .
 .TP
 \fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
@@ -514,6 +541,12 @@ The following parameters sets advanced compression options to something similar
 .P
 \fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
 .
+.SH "SEE ALSO"
+\fBzstdgrep\fR(1), \fBzstdless\fR(1), \fBgzip\fR(1), \fBxz\fR(1)
+.
+.P
+The \fIzstandard\fR format is specified in Y\. Collet, "Zstandard Compression and the \'application/zstd\' Media Type", https://www\.ietf\.org/rfc/rfc8878\.txt, Internet RFC 8878 (February 2021)\.
+.
 .SH "BUGS"
 Report bugs at: https://github\.com/facebook/zstd/issues
 .

From ea2895cef46d5850b00a9fde57e4e250df680bd3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 22 Dec 2022 12:40:27 -0800
Subject: [PATCH 079/197] Support decompression of compressed blocks of size
 ZSTD_BLOCKSIZE_MAX exactly

---
 lib/compress/zstd_compress.c           |  4 ++++
 lib/decompress/zstd_decompress_block.c | 12 ++++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4fb203db4..4cf2c0945 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2770,6 +2770,10 @@ ZSTD_entropyCompressSeqStore(
         if (cSize >= maxCSize) return 0;  /* block not compressed */
     }
     DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
+    /* libzstd decoder before  > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
+     * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
+     */
+    assert(cSize < ZSTD_BLOCKSIZE_MAX);
     return cSize;
 }
 
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 94728a165..494a70a5d 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -2010,12 +2010,20 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
      * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
      * We don't expect that to be the case in 64-bit mode.
      * In block mode, window size is not known, so we have to be conservative.
-     * (note: but it could be evaluated from current-lowLimit)
+     * (note: it could possibly be evaluated from current-lowLimit)
      */
     ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
     DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
 
-    RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
+    /* Note : the wording of the specification
+     * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX.
+     * This generally does not happen, as it makes little sense,
+     * since an uncompressed block would feature same size and have no decompression cost.
+     * Also, note that decoder from reference libzstd before < v1.5.4
+     * would consider this edge case as an error.
+     * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX
+     * for broader compatibility with the deployed ecosystem of zstd decoders */
+    RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
 
     /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);

From 6a9c525903ca65aad61b50381b5781c1187143d1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 22 Dec 2022 11:30:15 -0800
Subject: [PATCH 080/197] spec update : require minimum nb of literals for
 4-streams mode

Reported by @shulib :
the specification for 4-streams mode
doesn't work when the amount of literals to compress is 5 bytes.
Extending it, it also doesn't work for sizes 1 or 2.

This patch updates the specification and the implementation
to require a minimum of 6 literals to trigger or accept the 4-streams mode.

The impact is expected to be a no-op :
the 4-streams mode is never triggered for such small quantity of literals anyway,
since it would be wasteful (it costs ~7.3 bytes more than single-stream mode).
An informal lower limit is set at ~256 bytes,
so the technical minimum is very far from this limit.

This is just meant for completeness of the specification.
---
 doc/zstd_compression_format.md         | 24 ++++++++++++----
 lib/common/error_private.c             |  1 +
 lib/common/huf.h                       |  6 ++--
 lib/common/zstd_internal.h             |  1 +
 lib/compress/zstd_compress_literals.c  |  3 ++
 lib/decompress/huf_decompress.c        | 40 +++++++++++++++-----------
 lib/decompress/zstd_decompress_block.c |  8 +++++-
 lib/zstd_errors.h                      |  1 +
 8 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 042dbe89a..e40677a1d 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -435,7 +435,7 @@ They can be decoded first, and then copied during [Sequence Execution],
 or they can be decoded on the flow during [Sequence Execution].
 
 Literals can be stored uncompressed or compressed using Huffman prefix codes.
-When compressed, an optional tree description can be present,
+When compressed, a tree description may optionally be present,
 followed by 1 or 4 streams.
 
 | `Literals_Section_Header` | [`Huffman_Tree_Description`] | [jumpTable] | Stream1 | [Stream2] | [Stream3] | [Stream4] |
@@ -510,7 +510,7 @@ Its value is : `Size_Format = (Literals_Section_Header[0]>>2) & 3`
                `Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)`
 
 Only Stream1 is present for these cases.
-Note : it's allowed to represent a short value (for example `13`)
+Note : it's allowed to represent a short value (for example `27`)
 using a long format, even if it's less efficient.
 
 __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__ :
@@ -521,19 +521,33 @@ __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__
                Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023).
                `Literals_Section_Header` uses 3 bytes.
 - `Size_Format` == 01 : 4 streams.
-               Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023).
+               Both `Regenerated_Size` and `Compressed_Size` use 10 bits (6-1023).
                `Literals_Section_Header` uses 3 bytes.
 - `Size_Format` == 10 : 4 streams.
-               Both `Regenerated_Size` and `Compressed_Size` use 14 bits (0-16383).
+               Both `Regenerated_Size` and `Compressed_Size` use 14 bits (6-16383).
                `Literals_Section_Header` uses 4 bytes.
 - `Size_Format` == 11 : 4 streams.
-               Both `Regenerated_Size` and `Compressed_Size` use 18 bits (0-262143).
+               Both `Regenerated_Size` and `Compressed_Size` use 18 bits (6-262143).
                `Literals_Section_Header` uses 5 bytes.
 
 Both `Compressed_Size` and `Regenerated_Size` fields follow __little-endian__ convention.
 Note: `Compressed_Size` __includes__ the size of the Huffman Tree description
 _when_ it is present.
 
+4 streams is superior to 1 stream in decompression speed,
+by exploiting instruction level parallelism.
+But it's also more expensive,
+costing on average ~7.3 bytes more than the 1 stream mode, mostly from the jump table.
+
+In general, use the 4 streams mode when there are more literals to decode,
+to favor higher decompression speeds.
+Beyond 1KB, the 4 streams mode is compulsory anyway.
+
+Note that a minimum of 6 bytes is required for the 4 streams mode.
+That's a technical minimum, but it's not recommended to employ the 4 streams mode
+for such a small quantity, that would be wasteful.
+A more practical lower bound would be around ~256 bytes.
+
 #### Raw Literals Block
 The data in Stream1 is `Regenerated_Size` bytes long,
 it contains the raw literals data to be used during [Sequence Execution].
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index baf9b0b0c..fe73c5edd 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -29,6 +29,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
     case PREFIX(corruption_detected): return "Data corruption detected";
     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
     case PREFIX(parameter_unsupported): return "Unsupported parameter";
     case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 679493bfa..f7316f9b2 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -88,8 +88,10 @@ HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
                                unsigned maxSymbolValue, unsigned tableLog);
 
 /** HUF_compress4X_wksp() :
- *  Same as HUF_compress2(), but uses externally allocated `workSpace`.
- * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
+ *  Same as HUF_compress2(), but uses externally allocated @workSpace.
+ * @workSpace's size, aka @wkspSize, must be >= HUF_WORKSPACE_SIZE
+ * @srcSize must be >= 6
+ */
 #define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
 #define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
 HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index c9c97b7e4..48558873d 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -94,6 +94,7 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 
 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
 #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */)   /* for a non-null block */
+#define MIN_LITERALS_FOR_4_STREAMS 6
 
 typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
 
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index e97c485ab..972d40e71 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -164,16 +164,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     switch(lhSize)
     {
     case 3: /* 2 - 2 - 10 - 10 */
+        if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
             MEM_writeLE24(ostart, lhc);
             break;
         }
     case 4: /* 2 - 2 - 14 - 14 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
             MEM_writeLE32(ostart, lhc);
             break;
         }
     case 5: /* 2 - 2 - 18 - 18 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
             MEM_writeLE32(ostart, lhc);
             ostart[4] = (BYTE)(cLitSize >> 10);
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index b3ba5926b..e5c35dae9 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -289,10 +289,11 @@ typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1;   /* single-symbol decodi
 static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
     U64 D4;
     if (MEM_isLittleEndian()) {
-        D4 = (symbol << 8) + nbBits;
+        D4 = (U64)((symbol << 8) + nbBits);
     } else {
-        D4 = symbol + (nbBits << 8);
+        D4 = (U64)(symbol + (nbBits << 8));
     }
+    assert(D4 < (1U << 16));
     D4 *= 0x0001000100010001ULL;
     return D4;
 }
@@ -383,9 +384,8 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * rankStart[0] is not filled because there are no entries in the table for
      * weight 0.
      */
-    {
-        int n;
-        int nextRankStart = 0;
+    {   int n;
+        U32 nextRankStart = 0;
         int const unroll = 4;
         int const nLimit = (int)nbSymbols - unroll + 1;
         for (n=0; n<(int)tableLog+1; n++) {
@@ -412,10 +412,9 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * We can switch based on the length to a different inner loop which is
      * optimized for that particular case.
      */
-    {
-        U32 w;
-        int symbol=wksp->rankVal[0];
-        int rankStart=0;
+    {   U32 w;
+        int symbol = wksp->rankVal[0];
+        int rankStart = 0;
         for (w=1; w<tableLog+1; ++w) {
             int const symbolCount = wksp->rankVal[w];
             int const length = (1 << w) >> 1;
@@ -525,7 +524,7 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons
     while (p < pEnd)
         HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
 
-    return pEnd-pStart;
+    return (size_t)(pEnd-pStart);
 }
 
 FORCE_INLINE_TEMPLATE size_t
@@ -551,6 +550,10 @@ HUF_decompress1X1_usingDTable_internal_body(
     return dstSize;
 }
 
+/* HUF_decompress4X1_usingDTable_internal_body():
+ * Conditions :
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X1_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -594,6 +597,7 @@ HUF_decompress4X1_usingDTable_internal_body(
 
         if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
         if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
@@ -679,8 +683,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
     HUF_DecompressAsmArgs args;
-    {
-        size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+    {   size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
         FORWARD_IF_ERROR(ret, "Failed to init asm args");
         if (ret != 0)
             return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
@@ -700,8 +703,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     (void)iend;
 
     /* finish bit streams one by one. */
-    {
-        size_t const segmentSize = (dstSize+3) / 4;
+    {   size_t const segmentSize = (dstSize+3) / 4;
         BYTE* segmentEnd = (BYTE*)dst;
         int i;
         for (i = 0; i < 4; ++i) {
@@ -1246,6 +1248,11 @@ HUF_decompress1X2_usingDTable_internal_body(
     /* decoded size */
     return dstSize;
 }
+
+/* HUF_decompress4X2_usingDTable_internal_body():
+ * Conditions:
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X2_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -1286,8 +1293,9 @@ HUF_decompress4X2_usingDTable_internal_body(
         DTableDesc const dtd = HUF_getDTableDesc(DTable);
         U32 const dtLog = dtd.tableLog;
 
-        if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
-        if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (length4 > cSrcSize) return ERROR(corruption_detected);  /* overflow */
+        if (opStart4 > oend) return ERROR(corruption_detected);     /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 94728a165..b2351e8f1 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -166,6 +166,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 }
                 RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
                 RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+                if (!singleStream)
+                    RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
+                        "Not enough literals (%zu) for the 4-streams mode (min %u)",
+                        litSize, MIN_LITERALS_FOR_4_STREAMS);
                 RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
                 RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
                 ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
@@ -181,6 +185,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
                             dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
                     } else {
+                        assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
                         hufSuccess = HUF_decompress4X_usingDTable_bmi2(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
                             dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
@@ -509,7 +514,8 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
                 for (i = 8; i < n; i += 8) {
                     MEM_write64(spread + pos + i, sv);
                 }
-                pos += n;
+                assert(n>=0);
+                pos += (size_t)n;
             }
         }
         /* Now we spread those positions across the table.
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 57a50d024..880e8e4f6 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -70,6 +70,7 @@ typedef enum {
   ZSTD_error_frameParameter_windowTooLarge = 16,
   ZSTD_error_corruption_detected = 20,
   ZSTD_error_checksum_wrong      = 22,
+  ZSTD_error_literals_headerWrong = 24,
   ZSTD_error_dictionary_corrupted      = 30,
   ZSTD_error_dictionary_wrong          = 32,
   ZSTD_error_dictionaryCreation_failed = 34,

From 48f4aa7307003bbe496fad4cd4090b7fa93df6f0 Mon Sep 17 00:00:00 2001
From: Daniel Kutenin <kutdanila@yandex.ru>
Date: Fri, 23 Dec 2022 12:07:31 +0000
Subject: [PATCH 081/197] Move deprecated annotation before static to allow C++
 compilation for clang

This fixes last 2 instances of https://github.com/facebook/zstd/issues/3250
---
 lib/zdict.h | 2 +-
 lib/zstd.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/zdict.h b/lib/zdict.h
index 7b37e50e8..2268f948a 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -461,8 +461,8 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
 #  endif
 #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
 
-ZDICTLIB_STATIC_API
 ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
+ZDICTLIB_STATIC_API
 size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
                                   const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
diff --git a/lib/zstd.h b/lib/zstd.h
index dd72e17ed..a80017ba8 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2510,8 +2510,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLev
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
 
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 
 ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);

From 6640377783e73211d1afd440550f8587dd9de75c Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Fri, 23 Dec 2022 14:18:11 -0800
Subject: [PATCH 082/197] cmake build: fix nit

reported by @jaimeMF in https://github.com/facebook/zstd/pull/3392#discussion_r1056643794
---
 build/cmake/CMakeModules/AddZstdCompilationFlags.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
index 8e36f61d0..0265349fb 100644
--- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
@@ -38,7 +38,7 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
         # EnableCompilerFlag("-std=c99" true false)   # Set C compiation to c99 standard
         if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
             # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" true true flase)
+            EnableCompilerFlag("/clang:-Wall" true true false)
         else ()
             EnableCompilerFlag("-Wall" true true false)
         endif ()

From 1f72dca0ff07e00c1805b9bd58b1ae460f29b677 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 26 Dec 2022 05:08:53 +0000
Subject: [PATCH 083/197] Bump ossf/scorecard-action from 2.1.0 to 2.1.2

Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.1.0 to 2.1.2.
- [Release notes](https://github.com/ossf/scorecard-action/releases)
- [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md)
- [Commits](https://github.com/ossf/scorecard-action/compare/937ffa90d79c7d720498178154ad4c7ba1e4ad8c...e38b1902ae4f44df626f11ba0734b14fb91f8f86)

---
updated-dependencies:
- dependency-name: ossf/scorecard-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/scorecards.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index afdf4d63f..ca6f68269 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -32,7 +32,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@937ffa90d79c7d720498178154ad4c7ba1e4ad8c # tag=v2.1.0
+        uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # tag=v2.1.2
         with:
           results_file: results.sarif
           results_format: sarif

From 89342d1e076e86e59d143ce2fa3b8e8af6f65b03 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 27 Dec 2022 23:40:34 -0800
Subject: [PATCH 084/197] New xp library symbol : ZSTD_CCtx_setCParams()

Inspired by #3395,
offer a new capability to set all parameters defined in a ZSTD_compressionParameters structure
with a single symbol invocation
to improve user code brevity.
---
 lib/compress/zstd_compress.c | 19 +++++++++++++++----
 lib/zstd.h                   | 13 +++++++++----
 tests/fuzzer.c               | 30 ++++++++++++++++++++++++++----
 3 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4cf2c0945..f35f7a7a6 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -346,10 +346,13 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
 #define ZSTD_NO_CLEVEL 0
 
 /**
- * Initializes the cctxParams from params and compressionLevel.
+ * Initializes `cctxParams` from `params` and `compressionLevel`.
  * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL.
  */
-static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel)
+static void
+ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
+                        const ZSTD_parameters* params,
+                              int compressionLevel)
 {
     assert(!ZSTD_checkCParams(params->cParams));
     ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
@@ -796,14 +799,14 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
 
     case ZSTD_c_forceAttachDict : {
         const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
-        BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
+        BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref);
         CCtxParams->attachDictPref = pref;
         return CCtxParams->attachDictPref;
     }
 
     case ZSTD_c_literalCompressionMode : {
         const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
-        BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
+        BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm);
         CCtxParams->literalCompressionMode = lcm;
         return CCtxParams->literalCompressionMode;
     }
@@ -1098,6 +1101,14 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
     return 0;
 }
 
+size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
+{
+    FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
+    assert(cctx != NULL);
+    cctx->requestedParams.cParams = cparams;
+    return 0;
+}
+
 size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
diff --git a/lib/zstd.h b/lib/zstd.h
index dd72e17ed..cd44999b6 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1740,6 +1740,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
  *  This function never fails (wide contract) */
 ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
+/*! ZSTD_CCtx_setCParams() :
+ *  Set all parameters provided within @cparams into the working @cctx
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
 /*! ZSTD_compress_advanced() :
  *  Note : this function is now DEPRECATED.
  *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
@@ -1747,10 +1752,10 @@ ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressio
 ZSTD_DEPRECATED("use ZSTD_compress2")
 ZSTDLIB_STATIC_API
 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
-                                          void* dst, size_t dstCapacity,
-                                    const void* src, size_t srcSize,
-                                    const void* dict,size_t dictSize,
-                                          ZSTD_parameters params);
+                              void* dst, size_t dstCapacity,
+                        const void* src, size_t srcSize,
+                        const void* dict,size_t dictSize,
+                              ZSTD_parameters params);
 
 /*! ZSTD_compress_usingCDict_advanced() :
  *  Note : this function is now DEPRECATED.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ae460fefc..3ad8ced5e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -463,9 +463,9 @@ _output_error:
 *   Unit tests
 =============================================*/
 
-static void test_compressBound(int tnb)
+static void test_compressBound(unsigned tnb)
 {
-    DISPLAYLEVEL(3, "test%3i : compressBound : ", tnb);
+    DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
 
     /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
      * for a large range of known valid values */
@@ -485,9 +485,9 @@ static void test_compressBound(int tnb)
     DISPLAYLEVEL(3, "OK \n");
 }
 
-static void test_decompressBound(int tnb)
+static void test_decompressBound(unsigned tnb)
 {
-    DISPLAYLEVEL(3, "test%3i : decompressBound : ", tnb);
+    DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
 
     // Simple compression, with size : should provide size;
     {   const char example[] = "abcd";
@@ -538,6 +538,26 @@ static void test_decompressBound(int tnb)
     DISPLAYLEVEL(3, "OK \n");
 }
 
+static void test_setCParams(unsigned tnb)
+{
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    ZSTD_compressionParameters cparams;
+    assert(cctx);
+
+    DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
+
+    /* valid cparams */
+    cparams = ZSTD_getCParams(1, 0, 0);
+    CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
+
+    /* invalid cparams (must fail) */
+    cparams.windowLog = 99;
+    CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
+
+    free(cctx);
+    DISPLAYLEVEL(3, "OK \n");
+}
+
 static int basicUnitTests(U32 const seed, double compressibility)
 {
     size_t const CNBuffSize = 5 MB;
@@ -588,6 +608,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     test_decompressBound(testNb++);
 
+    test_setCParams(testNb++);
+
     DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
     {
         ZSTD_compressionParameters params;

From b17743e41b4aae01991285d4958b1b727a516008 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 28 Dec 2022 13:14:58 -0800
Subject: [PATCH 085/197] Signal parameter change during MT compression

---
 lib/compress/zstd_compress.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f35f7a7a6..fee502984 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -664,7 +664,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
         if (ZSTD_isUpdateAuthorized(param)) {
             cctx->cParamsChanged = 1;
         } else {
-            RETURN_ERROR(stage_wrong, "can only set params in ctx init stage");
+            RETURN_ERROR(stage_wrong, "can only set params in cctx init stage");
     }   }
 
     switch(param)
@@ -1103,8 +1103,15 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
 
 size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
 {
-    FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
+    DEBUGLOG(4, "ZSTD_CCtx_setCParams");
     assert(cctx != NULL);
+    if (cctx->streamStage != zcss_init) {
+        /* All parameters in @cparams are allowed to be updated during MT compression.
+         * This must be signaled, so that MT compression picks up the changes */
+        cctx->cParamsChanged = 1;
+    }
+    /* only update if parameters are valid */
+    FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
     cctx->requestedParams.cParams = cparams;
     return 0;
 }

From 2a402626dd046bf17e6172fe8d829ed5a443fbe3 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Wed, 28 Dec 2022 16:45:14 -0500
Subject: [PATCH 086/197] External matchfinder API (#3333)

* First building commit with sample matchfinder

* Set up ZSTD_externalMatchCtx struct

* move seqBuffer to ZSTD_Sequence*

* support non-contiguous dictionary

* clean up parens

* add clearExternalMatchfinder, handle allocation errors

* Add useExternalMatchfinder cParam

* validate useExternalMatchfinder cParam

* Disable LDM + external matchfinder

* Check for static CCtx

* Validate mState and mStateDestructor

* Improve LDM check to cover both branches

* Error API with optional fallback

* handle RLE properly for external matchfinder

* nit

* Move to a CDict-like model for resource ownership

* Add hidden useExternalMatchfinder bool to CCtx_params_s

* Eliminate malloc, move to cwksp allocation

* Handle CCtx reset properly

* Ensure seqStore has enough space for external sequences

* fix capitalization

* Add DEBUGLOG statements

* Add compressionLevel param to matchfinder API

* fix c99 issues and add a param combination error code

* nits

* Test external matchfinder API

* C90 compat for simpleExternalMatchFinder

* Fix some @nocommits and an ASAN bug

* nit

* nit

* nits

* forward declare copySequencesToSeqStore functions in zstd_compress_internal.h

* nit

* nit

* nits

* Update copyright headers

* Fix CMake zstreamtest build

* Fix copyright headers (again)

* typo

* Add externalMatchfinder demo program to make contrib

* Reduce memory consumption for small blockSize

* ZSTD_postProcessExternalMatchFinderResult nits

* test sum(matchlen) + sum(litlen) == srcSize in debug builds

* refExternalMatchFinder -> registerExternalMatchFinder

* C90 nit

* zstreamtest nits

* contrib nits

* contrib nits

* allow block splitter + external matchfinder, refactor

* add windowSize param

* add contrib/externalMatchfinder/README.md

* docs

* go back to old RLE heuristic because of the first block issue

* fix initializer element is not a constant expression

* ref contrib from zstd.h

* extremely pedantic compiler warning fix, meson fix, typo fix

* Additional docs on API limitations

* minor nits

* Refactor maxNbSeq calculation into a helper function

* Fix copyright
---
 Makefile                                  |   2 +
 build/cmake/tests/CMakeLists.txt          |   2 +-
 build/meson/tests/meson.build             |   6 +-
 contrib/externalMatchfinder/.gitignore    |   2 +
 contrib/externalMatchfinder/Makefile      |  40 ++++
 contrib/externalMatchfinder/README.md     |  14 ++
 contrib/externalMatchfinder/main.c        | 107 ++++++++++
 contrib/externalMatchfinder/matchfinder.c |  80 ++++++++
 contrib/externalMatchfinder/matchfinder.h |  26 +++
 lib/common/error_private.c                |   2 +
 lib/compress/zstd_compress.c              | 230 ++++++++++++++++++----
 lib/compress/zstd_compress_internal.h     |  53 +++++
 lib/zstd.h                                | 156 ++++++++++++++-
 lib/zstd_errors.h                         |   2 +
 tests/Makefile                            |   2 +-
 tests/external_matchfinder.c              | 117 +++++++++++
 tests/external_matchfinder.h              |  35 ++++
 tests/zstreamtest.c                       |  93 ++++++++-
 18 files changed, 929 insertions(+), 40 deletions(-)
 create mode 100644 contrib/externalMatchfinder/.gitignore
 create mode 100644 contrib/externalMatchfinder/Makefile
 create mode 100644 contrib/externalMatchfinder/README.md
 create mode 100644 contrib/externalMatchfinder/main.c
 create mode 100644 contrib/externalMatchfinder/matchfinder.c
 create mode 100644 contrib/externalMatchfinder/matchfinder.h
 create mode 100644 tests/external_matchfinder.c
 create mode 100644 tests/external_matchfinder.h

diff --git a/Makefile b/Makefile
index 10fbe47f7..d87fc76eb 100644
--- a/Makefile
+++ b/Makefile
@@ -123,6 +123,7 @@ contrib: lib
 	$(MAKE) -C contrib/seekable_format/examples all
 	$(MAKE) -C contrib/seekable_format/tests test
 	$(MAKE) -C contrib/largeNbDicts all
+	$(MAKE) -C contrib/externalMatchfinder all
 	cd build/single_file_libs/ ; ./build_decoder_test.sh
 	cd build/single_file_libs/ ; ./build_library_test.sh
 
@@ -142,6 +143,7 @@ clean:
 	$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
 	$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
 	$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/externalMatchfinder $@ > $(VOID)
 	$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	$(Q)$(RM) -r lz4
 	@echo Cleaning completed
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 53e0e7b17..250f0508f 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -81,7 +81,7 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
 #
 # zstreamtest
 #
-add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
+add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c ${TESTS_DIR}/external_matchfinder.c)
 if (NOT MSVC)
     target_compile_options(zstreamtest PRIVATE "-Wno-deprecated-declarations")
 endif()
diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index f7ba53101..e70b73432 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -65,8 +65,10 @@ fuzzer = executable('fuzzer',
   dependencies: [ testcommon_dep, thread_dep ],
   install: false)
 
-zstreamtest_sources = [join_paths(zstd_rootdir, 'tests/seqgen.c'),
-  join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
+zstreamtest_sources = [
+  join_paths(zstd_rootdir, 'tests/seqgen.c'),
+  join_paths(zstd_rootdir, 'tests/zstreamtest.c'),
+  join_paths(zstd_rootdir, 'tests/external_matchfinder.c')]
 zstreamtest = executable('zstreamtest',
   zstreamtest_sources,
   include_directories: test_includes,
diff --git a/contrib/externalMatchfinder/.gitignore b/contrib/externalMatchfinder/.gitignore
new file mode 100644
index 000000000..46357ef58
--- /dev/null
+++ b/contrib/externalMatchfinder/.gitignore
@@ -0,0 +1,2 @@
+# build artifacts
+externalMatchfinder
diff --git a/contrib/externalMatchfinder/Makefile b/contrib/externalMatchfinder/Makefile
new file mode 100644
index 000000000..2baa558cb
--- /dev/null
+++ b/contrib/externalMatchfinder/Makefile
@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) Yann Collet, Meta Platforms, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common
+
+CFLAGS  ?= -O3
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+default: externalMatchfinder
+
+all: externalMatchfinder
+
+externalMatchfinder: matchfinder.c main.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) externalMatchfinder
diff --git a/contrib/externalMatchfinder/README.md b/contrib/externalMatchfinder/README.md
new file mode 100644
index 000000000..cb7d49d97
--- /dev/null
+++ b/contrib/externalMatchfinder/README.md
@@ -0,0 +1,14 @@
+externalMatchfinder
+=====================
+
+`externalMatchfinder` is a test tool for the external matchfinder API.
+It demonstrates how to use the API to perform a simple round-trip test.
+
+A sample matchfinder is provided in matchfinder.c, but the user can swap
+this out with a different one if desired. The sample matchfinder implements
+LZ compression with a 1KB hashtable. Dictionary compression is not currently supported.
+
+Command line :
+```
+externalMatchfinder filename
+```
diff --git a/contrib/externalMatchfinder/main.c b/contrib/externalMatchfinder/main.c
new file mode 100644
index 000000000..6971a46c7
--- /dev/null
+++ b/contrib/externalMatchfinder/main.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#include "zstd_errors.h"
+#include "matchfinder.h" // simpleExternalMatchFinder
+
+#define CHECK(res)                                      \
+do {                                                    \
+    if (ZSTD_isError(res)) {                            \
+        printf("ERROR: %s\n", ZSTD_getErrorName(res));  \
+        return 1;                                       \
+    }                                                   \
+} while (0)                                             \
+
+int main(int argc, char *argv[]) {
+    if (argc != 2) {
+        printf("Usage: exampleMatchfinder <file>\n");
+        return 1;
+    }
+
+    ZSTD_CCtx* const zc = ZSTD_createCCtx();
+
+    int simpleExternalMatchState = 0xdeadbeef;
+
+    // Here is the crucial bit of code!
+    ZSTD_registerExternalMatchFinder(
+        zc,
+        &simpleExternalMatchState,
+        simpleExternalMatchFinder
+    );
+
+    {
+        size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 1);
+        CHECK(res);
+    }
+
+    FILE *f = fopen(argv[1], "rb");
+    assert(f);
+    {
+        int const ret = fseek(f, 0, SEEK_END);
+        assert(ret == 0);
+    }
+    size_t const srcSize = ftell(f);
+    {
+        int const ret = fseek(f, 0, SEEK_SET);
+        assert(ret == 0);
+    }
+
+    char* const src = malloc(srcSize + 1);
+    assert(src);
+    {
+        size_t const ret = fread(src, srcSize, 1, f);
+        assert(ret == 1);
+        int const ret2 = fclose(f);
+        assert(ret2 == 0);
+    }
+
+    size_t const dstSize = ZSTD_compressBound(srcSize);
+    char* const dst = malloc(dstSize);
+    assert(dst);
+
+    size_t const cSize = ZSTD_compress2(zc, dst, dstSize, src, srcSize);
+    CHECK(cSize);
+
+    char* const val = malloc(srcSize);
+    assert(val);
+
+    {
+        size_t const res = ZSTD_decompress(val, srcSize, dst, cSize);
+        CHECK(res);
+    }
+
+    if (memcmp(src, val, srcSize) == 0) {
+        printf("Compression and decompression were successful!\n");
+        printf("Original size: %lu\n", srcSize);
+        printf("Compressed size: %lu\n", cSize);
+    } else {
+        printf("ERROR: input and validation buffers don't match!\n");
+        for (size_t i = 0; i < srcSize; i++) {
+            if (src[i] != val[i]) {
+                printf("First bad index: %zu\n", i);
+                break;
+            }
+        }
+        return 1;
+    }
+
+    ZSTD_freeCCtx(zc);
+    free(src);
+    free(dst);
+    free(val);
+    return 0;
+}
diff --git a/contrib/externalMatchfinder/matchfinder.c b/contrib/externalMatchfinder/matchfinder.c
new file mode 100644
index 000000000..f119193ef
--- /dev/null
+++ b/contrib/externalMatchfinder/matchfinder.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "matchfinder.h"
+
+#define HSIZE 1024
+static U32 const HLOG = 10;
+static U32 const MLS = 4;
+static U32 const BADIDX = 0xffffffff;
+
+size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    size_t seqCount = 0;
+    U32 hashTable[HSIZE];
+
+    (void)externalMatchState;
+    (void)dict;
+    (void)dictSize;
+    (void)outSeqsCapacity;
+    (void)compressionLevel;
+
+    {   int i;
+        for (i=0; i < HSIZE; i++) {
+            hashTable[i] = BADIDX;
+    }   }
+
+    while (ip + MLS < iend) {
+        size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS);
+        U32 const matchIndex = hashTable[hash];
+        hashTable[hash] = (U32)(ip - istart);
+
+        if (matchIndex != BADIDX) {
+            const BYTE* const match = istart + matchIndex;
+            U32 const matchLen = (U32)ZSTD_count(ip, match, iend);
+            if (matchLen >= ZSTD_MINMATCH_MIN) {
+                U32 const litLen = (U32)(ip - anchor);
+                U32 const offset = (U32)(ip - match);
+                ZSTD_Sequence const seq = {
+                    offset, litLen, matchLen, 0
+                };
+
+                /* Note: it's crucial to stay within the window size! */
+                if (offset <= windowSize) {
+                    outSeqs[seqCount++] = seq;
+                    ip += matchLen;
+                    anchor = ip;
+                    continue;
+                }
+            }
+        }
+
+        ip++;
+    }
+
+    {   ZSTD_Sequence const finalSeq = {
+            0, (U32)(iend - anchor), 0, 0
+        };
+        outSeqs[seqCount++] = finalSeq;
+    }
+
+    return seqCount;
+}
diff --git a/contrib/externalMatchfinder/matchfinder.h b/contrib/externalMatchfinder/matchfinder.h
new file mode 100644
index 000000000..f8ba1c965
--- /dev/null
+++ b/contrib/externalMatchfinder/matchfinder.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MATCHFINDER_H
+#define MATCHFINDER_H
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+#endif
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index fe73c5edd..fb4d70596 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -31,6 +31,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
     case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
     case PREFIX(parameter_unsupported): return "Unsupported parameter";
+    case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
     case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
@@ -51,6 +52,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
     case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
     case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
+    case PREFIX(externalMatchFinder_failed): return "External matchfinder returned an error code";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4cf2c0945..9ded25129 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -278,6 +278,16 @@ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
     return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
 }
 
+/* Enables validation for external sequences in debug builds. */
+static int ZSTD_resolveExternalSequenceValidation(int mode) {
+#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2)
+    (void)mode;
+    return 1;
+#else
+    return mode;
+#endif
+}
+
 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
  * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
@@ -301,6 +311,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     }
     cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+    cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -362,6 +373,7 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par
     cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
     cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
+    cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -584,6 +596,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
+    case ZSTD_c_enableMatchFinderFallback:
+        bounds.lowerBound = 0;
+        bounds.upperBound = 1;
+        return bounds;
+
     default:
         bounds.error = ERROR(parameter_unsupported);
         return bounds;
@@ -649,6 +666,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableMatchFinderFallback:
     default:
         return 0;
     }
@@ -705,6 +723,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableMatchFinderFallback:
         break;
 
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -937,6 +956,11 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
         return CCtxParams->prefetchCDictTables;
 
+    case ZSTD_c_enableMatchFinderFallback:
+        BOUNDCHECK(ZSTD_c_enableMatchFinderFallback, value);
+        CCtxParams->enableMatchFinderFallback = value;
+        return CCtxParams->enableMatchFinderFallback;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -1072,6 +1096,9 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_prefetchCDictTables:
         *value = (int)CCtxParams->prefetchCDictTables;
         break;
+    case ZSTD_c_enableMatchFinderFallback:
+        *value = CCtxParams->enableMatchFinderFallback;
+        break;
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
@@ -1243,6 +1270,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
         RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                         "Can't reset parameters only when not in init stage.");
         ZSTD_clearAllDicts(cctx);
+        ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx));
         return ZSTD_CCtxParams_reset(&cctx->requestedParams);
     }
     return 0;
@@ -1485,6 +1513,13 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
     return tableSpace + optSpace + slackSpace + lazyAdditionalSpace;
 }
 
+/* Helper function for calculating memory requirements.
+ * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */
+static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useExternalMatchFinder) {
+    U32 const divider = (minMatch==3 || useExternalMatchFinder) ? 3 : 4;
+    return blockSize / divider;
+}
+
 static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_compressionParameters* cParams,
         const ldmParams_t* ldmParams,
@@ -1492,12 +1527,12 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_paramSwitch_e useRowMatchFinder,
         const size_t buffInSize,
         const size_t buffOutSize,
-        const U64 pledgedSrcSize)
+        const U64 pledgedSrcSize,
+        int useExternalMatchFinder)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
     size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-    U32    const divider = (cParams->minMatch==3) ? 3 : 4;
-    size_t const maxNbSeq = blockSize / divider;
+    size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
                             + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
@@ -1516,6 +1551,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 
     size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
 
+    size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+    size_t const externalSeqSpace = useExternalMatchFinder
+        ? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+        : 0;
+
     size_t const neededSpace =
         cctxSpace +
         entropySpace +
@@ -1524,7 +1564,8 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         ldmSeqSpace +
         matchStateSize +
         tokenSpace +
-        bufferSpace;
+        bufferSpace +
+        externalSeqSpace;
 
     DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
     return neededSpace;
@@ -1542,7 +1583,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
      * be needed. However, we still allocate two 0-sized buffers, which can
      * take space under ASAN. */
     return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
-        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN);
+        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
 }
 
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1603,7 +1644,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
-            ZSTD_CONTENTSIZE_UNKNOWN);
+            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
     }
 }
 
@@ -1886,8 +1927,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
         size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-        U32    const divider = (params->cParams.minMatch==3) ? 3 : 4;
-        size_t const maxNbSeq = blockSize / divider;
+        size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
         size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
                 : 0;
@@ -1904,7 +1944,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         size_t const neededSpace =
             ZSTD_estimateCCtxSize_usingCCtxParams_internal(
                 &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
-                buffInSize, buffOutSize, pledgedSrcSize);
+                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder);
         int resizeWorkspace;
 
         FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
@@ -2017,6 +2057,14 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
             zc->ldmState.loadedDictEnd = 0;
         }
 
+        /* reserve space for block-level external sequences */
+        if (params->useExternalMatchFinder) {
+            size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+            zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
+            zc->externalMatchCtx.seqBuffer =
+                (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
+        }
+
         DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
         assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
 
@@ -2868,6 +2916,55 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
     ssPtr->longLengthType = ZSTD_llt_none;
 }
 
+/* ZSTD_postProcessExternalMatchFinderResult() :
+ * Validates and post-processes sequences obtained through the external matchfinder API:
+ *   - Checks whether nbExternalSeqs represents an error condition.
+ *   - Appends a block delimiter to outSeqs if one is not already present.
+ *     See zstd.h for context regarding block delimiters.
+ * Returns the number of sequences after post-processing, or an error code. */
+static size_t ZSTD_postProcessExternalMatchFinderResult(
+    ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize
+) {
+    RETURN_ERROR_IF(
+        nbExternalSeqs > outSeqsCapacity,
+        externalMatchFinder_failed,
+        "External matchfinder returned error code %lu",
+        (unsigned long)nbExternalSeqs
+    );
+
+    RETURN_ERROR_IF(
+        nbExternalSeqs == 0 && srcSize > 0,
+        externalMatchFinder_failed,
+        "External matchfinder produced zero sequences for a non-empty src buffer!"
+    );
+
+    if (srcSize == 0) {
+        ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence));
+        return 1;
+    }
+
+    {
+        ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1];
+
+        /* We can return early if lastSeq is already a block delimiter. */
+        if (lastSeq.offset == 0 && lastSeq.matchLength == 0) {
+            return nbExternalSeqs;
+        }
+
+        /* This error condition is only possible if the external matchfinder
+         * produced an invalid parse, by definition of ZSTD_sequenceBound(). */
+        RETURN_ERROR_IF(
+            nbExternalSeqs == outSeqsCapacity,
+            externalMatchFinder_failed,
+            "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!"
+        );
+
+        /* lastSeq is not a block delimiter, so we need to append one. */
+        ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence));
+        return nbExternalSeqs + 1;
+    }
+}
+
 typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
 
 static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
@@ -2915,6 +3012,15 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         }
         if (zc->externSeqStore.pos < zc->externSeqStore.size) {
             assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
+
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useExternalMatchFinder,
+                parameter_combination_unsupported,
+                "Long-distance matching with external matchfinder enabled is not currently supported."
+            );
+
             /* Updates ldmSeqStore.pos */
             lastLLSize =
                 ZSTD_ldm_blockCompress(&zc->externSeqStore,
@@ -2926,6 +3032,14 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
             rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
 
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useExternalMatchFinder,
+                parameter_combination_unsupported,
+                "Long-distance matching with external matchfinder enabled is not currently supported."
+            );
+
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
             /* Updates ldmSeqStore.size */
@@ -2940,10 +3054,64 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                                        zc->appliedParams.useRowMatchFinder,
                                        src, srcSize);
             assert(ldmSeqStore.pos == ldmSeqStore.size);
-        } else {   /* not long range mode */
+        } else if (zc->appliedParams.useExternalMatchFinder) {
+            assert(
+                zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize)
+            );
+            assert(zc->externalMatchCtx.mFinder != NULL);
+
+            {   U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog;
+
+                size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)(
+                    zc->externalMatchCtx.mState,
+                    zc->externalMatchCtx.seqBuffer,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    src, srcSize,
+                    NULL, 0,  /* dict and dictSize, currently not supported */
+                    zc->appliedParams.compressionLevel,
+                    windowSize
+                );
+
+                size_t const nbPostProcessedSeqs = ZSTD_postProcessExternalMatchFinderResult(
+                    zc->externalMatchCtx.seqBuffer,
+                    nbExternalSeqs,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    srcSize
+                );
+
+                /* Return early if there is no error, since we don't need to worry about last literals */
+                if (!ZSTD_isError(nbPostProcessedSeqs)) {
+                    ZSTD_sequencePosition seqPos = {0,0,0};
+                    ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
+                        zc, &seqPos, zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs, src, srcSize
+                    );
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(5, "Copied %lu sequences from external matchfinder to internal seqStore.", (unsigned long)nbExternalSeqs);
+                    return ZSTDbss_compress;
+                }
+
+                /* Propagate the error if fallback is disabled */
+                if (!zc->appliedParams.enableMatchFinderFallback) {
+                    return nbPostProcessedSeqs;
+                }
+
+                /* Fallback to software matchfinder */
+                {   ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+                                                                                            zc->appliedParams.useRowMatchFinder,
+                                                                                            dictMode);
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(
+                        5,
+                        "External matchfinder returned error code %lu. Falling back to internal matchfinder.",
+                        (unsigned long)nbExternalSeqs
+                    );
+                    lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+            }   }
+        } else {   /* not long range mode and no external matchfinder */
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
                                                                                     zc->appliedParams.useRowMatchFinder,
                                                                                     dictMode);
+            assert(zc->externalMatchCtx.mFinder == NULL);
             ms->ldmSeqStore = NULL;
             lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
         }
@@ -5726,6 +5894,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, &params.cParams);
     params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
+    params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
 
 #ifdef ZSTD_MULTITHREAD
     if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
@@ -5927,12 +6096,6 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
     }
 }
 
-typedef struct {
-    U32 idx;            /* Index in array of ZSTD_Sequence */
-    U32 posInSequence;  /* Position within sequence at idx */
-    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
-} ZSTD_sequencePosition;
-
 /* ZSTD_validateSequence() :
  * @offCode : is presumed to follow format required by ZSTD_storeSeq()
  * @returns a ZSTD error code if sequence is not valid
@@ -5970,10 +6133,7 @@ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32
     return offBase;
 }
 
-/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
- * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                               ZSTD_sequencePosition* seqPos,
                                         const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
@@ -6027,19 +6187,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
     return 0;
 }
 
-/* Returns the number of bytes to move the current read position back by.
- * Only non-zero if we ended up splitting a sequence.
- * Otherwise, it may return a ZSTD error if something went wrong.
- *
- * This function will attempt to scan through blockSize bytes
- * represented by the sequences in @inSeqs,
- * storing any (partial) sequences.
- *
- * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
- * avoid splitting a match, or to avoid splitting a match such that it would produce a match
- * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                    const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
                                    const void* src, size_t blockSize)
@@ -6572,3 +6720,19 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
     if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
     return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
 }
+
+void ZSTD_registerExternalMatchFinder(
+    ZSTD_CCtx* zc, void* mState,
+    ZSTD_externalMatchFinder_F* mFinder
+) {
+    ZSTD_externalMatchCtx emctx = {
+        mState,
+        mFinder,
+
+        /* seqBuffer is allocated later (from the cwskp) */
+        NULL, /* seqBuffer */
+        0 /* seqBufferCapacity */
+    };
+    zc->externalMatchCtx = emctx;
+    zc->requestedParams.useExternalMatchFinder = 1;
+}
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index e4bb2f537..f755a1f79 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -150,6 +150,12 @@ typedef struct {
   size_t capacity;      /* The capacity starting from `seq` pointer */
 } rawSeqStore_t;
 
+typedef struct {
+    U32 idx;            /* Index in array of ZSTD_Sequence */
+    U32 posInSequence;  /* Position within sequence at idx */
+    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
+} ZSTD_sequencePosition;
+
 UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
 
 typedef struct {
@@ -340,6 +346,15 @@ struct ZSTD_CCtx_params_s {
 
     /* Controls prefetching in some dictMatchState matchfinders */
     ZSTD_paramSwitch_e prefetchCDictTables;
+
+    /* Controls whether zstd will fall back to an internal matchfinder
+     * if the external matchfinder returns an error code. */
+    int enableMatchFinderFallback;
+
+    /* Indicates whether an external matchfinder has been referenced.
+     * Users can't set this externally.
+     * It is set internally in ZSTD_registerExternalMatchFinder(). */
+    int useExternalMatchFinder;
 };  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
 
 #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
@@ -371,6 +386,14 @@ typedef struct {
     ZSTD_entropyCTablesMetadata_t entropyMetadata;
 } ZSTD_blockSplitCtx;
 
+/* Context for block-level external matchfinder API */
+typedef struct {
+  void* mState;
+  ZSTD_externalMatchFinder_F* mFinder;
+  ZSTD_Sequence* seqBuffer;
+  size_t seqBufferCapacity;
+} ZSTD_externalMatchCtx;
+
 struct ZSTD_CCtx_s {
     ZSTD_compressionStage_e stage;
     int cParamsChanged;                  /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
@@ -440,6 +463,9 @@ struct ZSTD_CCtx_s {
 
     /* Workspace for block splitter */
     ZSTD_blockSplitCtx blockSplitCtx;
+
+    /* Workspace for external matchfinder */
+    ZSTD_externalMatchCtx externalMatchCtx;
 };
 
 typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
@@ -1411,4 +1437,31 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
  */
 void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
 
+/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
+ * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
+ * Note that the block delimiter must include the last literals of the block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+                                              ZSTD_sequencePosition* seqPos,
+                                        const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                        const void* src, size_t blockSize);
+
+/* Returns the number of bytes to move the current read position back by.
+ * Only non-zero if we ended up splitting a sequence.
+ * Otherwise, it may return a ZSTD error if something went wrong.
+ *
+ * This function will attempt to scan through blockSize bytes
+ * represented by the sequences in @inSeqs,
+ * storing any (partial) sequences.
+ *
+ * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
+ * avoid splitting a match, or to avoid splitting a match such that it would produce a match
+ * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+                                   const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                   const void* src, size_t blockSize);
+
 #endif /* ZSTD_COMPRESS_H */
diff --git a/lib/zstd.h b/lib/zstd.h
index dd72e17ed..14a7d2306 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -478,6 +478,7 @@ typedef enum {
      * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableMatchFinderFallback
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -497,7 +498,8 @@ typedef enum {
      ZSTD_c_experimentalParam13=1010,
      ZSTD_c_experimentalParam14=1011,
      ZSTD_c_experimentalParam15=1012,
-     ZSTD_c_experimentalParam16=1013
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014
 } ZSTD_cParameter;
 
 typedef struct {
@@ -560,7 +562,7 @@ typedef enum {
  *                  They will be used to compress next frame.
  *                  Resetting session never fails.
  *  - The parameters : changes all parameters back to "default".
- *                  This removes any reference to any dictionary too.
+ *                  This also removes any reference to any dictionary or external matchfinder.
  *                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
  *                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
  *  - Both : similar to resetting the session, followed by resetting parameters.
@@ -2044,6 +2046,20 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  */
 #define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
 
+/* ZSTD_c_enableMatchFinderFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal matchfinder if an
+ * external matchfinder is registered and returns an error code. This fallback is
+ * block-by-block: the internal matchfinder will only be called for blocks where
+ * the external matchfinder returns an error code. Fallback compression will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full external matchfinder API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
+
 /*! ZSTD_CCtx_getParameter() :
  *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  *  and store it into int* value.
@@ -2676,6 +2692,142 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_
 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
 
 
+/* ********************** EXTERNAL MATCHFINDER API **********************
+ *
+ * *** OVERVIEW ***
+ * This API allows users to replace the zstd internal block-level matchfinder
+ * with an external matchfinder function. Potential applications of the API
+ * include hardware-accelerated matchfinders and matchfinders specialized to
+ * particular types of data.
+ *
+ * See contrib/externalMatchfinder for an example program employing the
+ * external matchfinder API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_externalMatchFinder_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ *   - externalMatchState: a pointer to a user-managed state for the external
+ *     matchfinder.
+ *
+ *   - outSeqs, outSeqsCapacity: an output buffer for sequences produced by the
+ *     external matchfinder. outSeqsCapacity is guaranteed >=
+ *     ZSTD_sequenceBound(srcSize). The memory backing outSeqs is managed by
+ *     the CCtx.
+ *
+ *   - src, srcSize: an input buffer which the external matchfinder must parse
+ *     into sequences. srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ *   - dict, dictSize: a history buffer, which may be empty, which the external
+ *     matchfinder may reference as it produces sequences for the src buffer.
+ *     Currently, zstd will always pass dictSize == 0 into external matchfinders,
+ *     but this will change in the future.
+ *
+ *   - compressionLevel: a signed integer representing the zstd compression level
+ *     set by the user for the current operation. The external matchfinder may
+ *     choose to use this information to change its compression strategy and
+ *     speed/ratio tradeoff. Note: The compression level does not reflect zstd
+ *     parameters set through the advanced API.
+ *
+ *   - windowSize: a size_t representing the maximum allowed offset for external
+ *     sequences. Note that sequence offsets are sometimes allowed to exceed the
+ *     windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ *     for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_EXTERNAL_MATCHFINDER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ *   - The sum of matchLengths and literalLengths is equal to srcSize.
+ *   - All sequences in the parse have matchLength != 0, except for the final
+ *     sequence. matchLength is not constrained for the final sequence.
+ *   - All offsets respect the windowSize parameter as specified in
+ *     doc/zstd_compression_format.md.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal matchfinder or fail the compression operation. The user can
+ * choose between the two behaviors by setting the
+ * ZSTD_c_enableMatchFinderFallback cParam. Fallback compression will follow any
+ * other cParam settings, such as compression level, the same as in a normal
+ * compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_externalMatchFinder_F
+ * function by calling ZSTD_registerExternalMatchFinder(cctx, externalMatchState,
+ * externalMatchFinder). This setting will persist until the next parameter reset
+ * of the CCtx.
+ *
+ * The externalMatchState must be initialized by the user before calling
+ * ZSTD_registerExternalMatchFinder. The user is responsible for destroying the
+ * externalMatchState.
+ *
+ * *** LIMITATIONS ***
+ * External matchfinders are compatible with all zstd compression APIs. There are
+ * only two limitations.
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
+ * external matchfinder.
+ *   - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in
+ *     some cases (see its documentation for details). Users must explicitly set
+ *     ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an
+ *     external matchfinder is registered.
+ *   - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ *     whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ *     check the docs on ZSTD_c_enableLongDistanceMatching whenever the external
+ *     matchfinder API is used in conjunction with advanced settings (like windowLog).
+ *
+ * Second, history buffers are not supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external matchfinder (for now). This has two implications:
+ *   - Dictionaries are not supported. Compression will *not* fail if the user
+ *     references a dictionary, but the dictionary won't have any effect.
+ *   - Stream history is not supported. All compression APIs, including streaming
+ *     APIs, work with the external matchfinder, but the external matchfinder won't
+ *     receive any history from the previous block. Each block is an independent chunk.
+ *
+ * Long-term, we plan to overcome both limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_EXTERNAL_MATCHFINDER_ERROR ((size_t)(-1))
+
+typedef size_t ZSTD_externalMatchFinder_F (
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+/*! ZSTD_registerExternalMatchFinder() :
+ * Instruct zstd to use an external matchfinder function.
+ *
+ * The externalMatchState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * The user is strongly encouraged to read the full API documentation (above)
+ * before calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerExternalMatchFinder(
+  ZSTD_CCtx* cctx,
+  void* externalMatchState,
+  ZSTD_externalMatchFinder_F* externalMatchFinder
+);
+
 #endif   /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
 
 #if defined (__cplusplus)
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 880e8e4f6..bd6dbee5f 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -75,6 +75,7 @@ typedef enum {
   ZSTD_error_dictionary_wrong          = 32,
   ZSTD_error_dictionaryCreation_failed = 34,
   ZSTD_error_parameter_unsupported   = 40,
+  ZSTD_error_parameter_combination_unsupported = 41,
   ZSTD_error_parameter_outOfBound    = 42,
   ZSTD_error_tableLog_tooLarge       = 44,
   ZSTD_error_maxSymbolValue_tooLarge = 46,
@@ -92,6 +93,7 @@ typedef enum {
   ZSTD_error_seekableIO          = 102,
   ZSTD_error_dstBuffer_wrong     = 104,
   ZSTD_error_srcBuffer_wrong     = 105,
+  ZSTD_error_externalMatchFinder_failed = 106,
   ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;
 
diff --git a/tests/Makefile b/tests/Makefile
index baf616acb..8226176cc 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -169,7 +169,7 @@ fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PR
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
 CLEAN += zstreamtest zstreamtest32
-ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
+ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c external_matchfinder.c
 ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
 ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
 zstreamtest32 : CFLAGS += -m32
diff --git a/tests/external_matchfinder.c b/tests/external_matchfinder.c
new file mode 100644
index 000000000..8ae76d519
--- /dev/null
+++ b/tests/external_matchfinder.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "external_matchfinder.h"
+#include <string.h>
+#include "zstd_compress_internal.h"
+
+#define HSIZE 1024
+static U32 const HLOG = 10;
+static U32 const MLS = 4;
+static U32 const BADIDX = 0xffffffff;
+
+static size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    size_t seqCount = 0;
+    U32 hashTable[HSIZE];
+
+    (void)externalMatchState;
+    (void)dict;
+    (void)dictSize;
+    (void)outSeqsCapacity;
+    (void)compressionLevel;
+
+    {   int i;
+        for (i=0; i < HSIZE; i++) {
+            hashTable[i] = BADIDX;
+    }   }
+
+    while (ip + MLS < iend) {
+        size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS);
+        U32 const matchIndex = hashTable[hash];
+        hashTable[hash] = (U32)(ip - istart);
+
+        if (matchIndex != BADIDX) {
+            const BYTE* const match = istart + matchIndex;
+            U32 const matchLen = (U32)ZSTD_count(ip, match, iend);
+            if (matchLen >= ZSTD_MINMATCH_MIN) {
+                U32 const litLen = (U32)(ip - anchor);
+                U32 const offset = (U32)(ip - match);
+                ZSTD_Sequence const seq = {
+                    offset, litLen, matchLen, 0
+                };
+
+                /* Note: it's crucial to stay within the window size! */
+                if (offset <= windowSize) {
+                    outSeqs[seqCount++] = seq;
+                    ip += matchLen;
+                    anchor = ip;
+                    continue;
+                }
+            }
+        }
+
+        ip++;
+    }
+
+    {   ZSTD_Sequence const finalSeq = {
+            0, (U32)(iend - anchor), 0, 0
+        };
+        outSeqs[seqCount++] = finalSeq;
+    }
+
+    return seqCount;
+}
+
+size_t zstreamExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    EMF_testCase const testCase = *((EMF_testCase*)externalMatchState);
+    memset(outSeqs, 0, outSeqsCapacity);
+
+    switch (testCase) {
+        case EMF_ZERO_SEQS:
+            return 0;
+        case EMF_ONE_BIG_SEQ:
+            outSeqs[0].offset = 0;
+            outSeqs[0].matchLength = 0;
+            outSeqs[0].litLength = (U32)(srcSize);
+            return 1;
+         case EMF_LOTS_OF_SEQS:
+            return simpleExternalMatchFinder(
+                externalMatchState,
+                outSeqs, outSeqsCapacity,
+                src, srcSize,
+                dict, dictSize,
+                compressionLevel,
+                windowSize
+            );
+        case EMF_SMALL_ERROR:
+            return outSeqsCapacity + 1;
+        case EMF_BIG_ERROR:
+        default:
+            return ZSTD_EXTERNAL_MATCHFINDER_ERROR;
+    }
+}
diff --git a/tests/external_matchfinder.h b/tests/external_matchfinder.h
new file mode 100644
index 000000000..041f73e4d
--- /dev/null
+++ b/tests/external_matchfinder.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef EXTERNAL_MATCHFINDER
+#define EXTERNAL_MATCHFINDER
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+/* See external_matchfinder.c for details on each test case */
+typedef enum {
+    EMF_ZERO_SEQS = 0,
+    EMF_ONE_BIG_SEQ = 1,
+    EMF_LOTS_OF_SEQS = 2,
+    EMF_BIG_ERROR = 3,
+    EMF_SMALL_ERROR = 4
+} EMF_testCase;
+
+size_t zstreamExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+#endif // EXTERNAL_MATCHFINDER
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 30c5215ae..664ff632a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -39,7 +39,7 @@
 #include "seqgen.h"
 #include "util.h"
 #include "timefn.h"       /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
-
+#include "external_matchfinder.h"   /* zstreamExternalMatchFinder, EMF_testCase */
 
 /*-************************************
  *  Constants
@@ -1834,6 +1834,97 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : External matchfinder API: ", testNb++);
+    {
+        size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
+        BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize));
+        size_t const checkBufSize = CNBufferSize;
+        BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
+        int enableFallback;
+        EMF_testCase externalMatchState;
+
+        CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
+
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+
+        /* Reference external matchfinder outside the test loop to
+         * check that the reference is preserved across compressions */
+        ZSTD_registerExternalMatchFinder(
+            zc,
+            &externalMatchState,
+            zstreamExternalMatchFinder
+        );
+
+        for (enableFallback = 0; enableFallback < 1; enableFallback++) {
+            size_t testCaseId;
+
+            EMF_testCase const EMF_successCases[] = {
+                EMF_ONE_BIG_SEQ,
+                EMF_LOTS_OF_SEQS,
+            };
+            size_t const EMF_numSuccessCases = 2;
+
+            EMF_testCase const EMF_failureCases[] = {
+                EMF_ZERO_SEQS,
+                EMF_BIG_ERROR,
+                EMF_SMALL_ERROR,
+            };
+            size_t const EMF_numFailureCases = 3;
+
+            /* Test external matchfinder success scenarios */
+            for (testCaseId = 0; testCaseId < EMF_numSuccessCases; testCaseId++) {
+                size_t res;
+                externalMatchState = EMF_successCases[testCaseId];
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+                CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
+                CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
+                CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
+            }
+
+            /* Test external matchfinder failure scenarios */
+            for (testCaseId = 0; testCaseId < EMF_numFailureCases; testCaseId++) {
+                size_t res;
+                externalMatchState = EMF_failureCases[testCaseId];
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+                if (enableFallback) {
+                    CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
+                    CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
+                } else {
+                    CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+                    CHECK(
+                        ZSTD_getErrorCode(res) != ZSTD_error_externalMatchFinder_failed,
+                        "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+                    );
+                }
+            }
+
+            /* Test compression with external matchfinder + empty src buffer */
+            {
+                size_t res;
+                externalMatchState = EMF_ZERO_SEQS;
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
+                CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
+                CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
+            }
+        }
+
+        /* Test that reset clears the external matchfinder */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+        free(dstBuf);
+        free(checkBuf);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);

From 00c85b28e724dca2ceeac8ac8ef319c282ad3f5c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 28 Dec 2022 15:08:18 -0800
Subject: [PATCH 087/197] update ZSTD_CCts_setCParams() inline documentation

specify behavior when changing compression parameters during MT compression,
reported by @embg
---
 lib/zstd.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index b4035686f..3279ee1a7 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1743,7 +1743,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
 ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
 /*! ZSTD_CCtx_setCParams() :
- *  Set all parameters provided within @cparams into the working @cctx
+ *  Set all parameters provided within @cparams into the working @cctx.
+ *  Note : if modifying parameters during compression (MT mode only),
+ *         note that changes to the .windowLog parameter will be ignored.
  * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
 

From 4a1a79a51229bafbd89964043af236ebd4707b38 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 19 Aug 2022 16:04:28 -0700
Subject: [PATCH 088/197] just add some comments to zstd_opt for improved
 clarity

---
 lib/compress/zstd_opt.c | 68 +++++++++++++++++++++++++----------------
 1 file changed, 41 insertions(+), 27 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 6e08dc8c7..f0f3791e7 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -37,11 +37,16 @@
 #  define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
 #endif
 
+/* ZSTD_bitWeight() :
+ * provide estimated "cost" of a stat in full bits only */
 MEM_STATIC U32 ZSTD_bitWeight(U32 stat)
 {
     return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER);
 }
 
+/* ZSTD_fracWeight() :
+ * provide fractional-bit "cost" of a stat,
+ * using linear interpolation approximation */
 MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
 {
     U32 const stat = rawStat + 1;
@@ -50,6 +55,7 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
     U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
     U32 const weight = BWeight + FWeight;
     assert(hb + BITCOST_ACCURACY < 31);
+    assert(FWeight < BITCOST_MULTIPLIER);
     return weight;
 }
 
@@ -91,17 +97,19 @@ static U32 sum_u32(const unsigned table[], size_t nbElts)
 static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift)
 {
     U32 s, sum=0;
-    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift);
+    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
+            (unsigned)lastEltIndex+1, (unsigned)shift );
     assert(shift < 30);
     for (s=0; s<lastEltIndex+1; s++) {
-        table[s] = 1 + (table[s] >> shift);
-        sum += table[s];
+        unsigned newStat = 1 + (table[s] >> shift);
+        sum += newStat;
+        table[s] = newStat;
     }
     return sum;
 }
 
 /* ZSTD_scaleStats() :
- * reduce all elements in table is sum too large
+ * reduce all elt frequencies in table if sum too large
  * return the resulting sum of elements */
 static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
 {
@@ -129,18 +137,22 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
     DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
     optPtr->priceType = zop_dynamic;
 
-    if (optPtr->litLengthSum == 0) {  /* first block : init */
-        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {  /* heuristic */
-            DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
+    if (optPtr->litLengthSum == 0) {  /* no literals stats collected -> first block assumed -> init */
+
+        /* heuristic: use pre-defined stats for too small inputs */
+        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {
+            DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD);
             optPtr->priceType = zop_predef;
         }
 
         assert(optPtr->symbolCosts != NULL);
         if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
-            /* huffman table presumed generated by dictionary */
+
+            /* huffman stats covering the full value set : table presumed generated by dictionary */
             optPtr->priceType = zop_dynamic;
 
             if (compressedLiterals) {
+                /* generate literals statistics from huffman table */
                 unsigned lit;
                 assert(optPtr->litFreq != NULL);
                 optPtr->litSum = 0;
@@ -188,10 +200,11 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                     optPtr->offCodeSum += optPtr->offCodeFreq[of];
             }   }
 
-        } else {  /* not a dictionary */
+        } else {  /* huf.repeatMode != HUF_repeat_valid => presumed not a dictionary */
 
             assert(optPtr->litFreq != NULL);
             if (compressedLiterals) {
+                /* base initial cost of literals on direct frequency within src */
                 unsigned lit = MaxLit;
                 HIST_count_simple(optPtr->litFreq, &lit, src, srcSize);   /* use raw first block to init statistics */
                 optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8);
@@ -224,10 +237,9 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                 optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
             }
 
-
         }
 
-    } else {   /* new block : re-use previous statistics, scaled down */
+    } else {   /* new block : scale down accumulated statistics */
 
         if (compressedLiterals)
             optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
@@ -275,10 +287,11 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
     assert(litLength <= ZSTD_BLOCKSIZE_MAX);
     if (optPtr->priceType == zop_predef)
         return WEIGHT(litLength, optLevel);
-    /* We can't compute the litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
-     * because it isn't representable in the zstd format. So instead just
-     * call it 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. In this case the block
-     * would be all literals.
+
+    /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
+     * because it isn't representable in the zstd format.
+     * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1.
+     * In such a case, the block would be all literals.
      */
     if (litLength == ZSTD_BLOCKSIZE_MAX)
         return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel);
@@ -292,7 +305,7 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
 }
 
 /* ZSTD_getMatchPrice() :
- * Provides the cost of the match part (offset + matchLength) of a sequence
+ * Provides the cost of the match part (offset + matchLength) of a sequence.
  * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
  * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
  * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
@@ -308,8 +321,9 @@ ZSTD_getMatchPrice(U32 const offBase,
     U32 const mlBase = matchLength - MINMATCH;
     assert(matchLength >= MINMATCH);
 
-    if (optPtr->priceType == zop_predef)  /* fixed scheme, do not use statistics */
-        return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER);
+    if (optPtr->priceType == zop_predef)  /* fixed scheme, does not use statistics */
+        return WEIGHT(mlBase, optLevel)
+             + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */
 
     /* dynamic statistics */
     price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
@@ -347,7 +361,7 @@ static void ZSTD_updateStats(optState_t* const optPtr,
         optPtr->litLengthSum++;
     }
 
-    /* offset code : expected to follow storeSeq() numeric representation */
+    /* offset code : follows storeSeq() numeric representation */
     {   U32 const offCode = ZSTD_highbit32(offBase);
         assert(offCode <= MaxOff);
         optPtr->offCodeFreq[offCode]++;
@@ -1352,7 +1366,7 @@ size_t ZSTD_compressBlock_btopt(
 /* ZSTD_initStats_ultra():
  * make a first compression pass, just to seed stats with more accurate starting values.
  * only works on first block, with no dictionary and no ldm.
- * this function cannot error, hence its contract must be respected.
+ * this function cannot error out, its narrow contract must be respected.
  */
 static void
 ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
@@ -1371,7 +1385,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
 
     ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict);   /* generate stats into ms->opt*/
 
-    /* invalidate first scan from history */
+    /* invalidate first scan from history, only keep entropy stats */
     ZSTD_resetSeqStore(seqStore);
     ms->window.base -= srcSize;
     ms->window.dictLimit += (U32)srcSize;
@@ -1395,20 +1409,20 @@ size_t ZSTD_compressBlock_btultra2(
     U32 const curr = (U32)((const BYTE*)src - ms->window.base);
     DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
 
-    /* 2-pass strategy:
+    /* 2-passes strategy:
      * this strategy makes a first pass over first block to collect statistics
-     * and seed next round's statistics with it.
-     * After 1st pass, function forgets everything, and starts a new block.
+     * in order to seed next round's statistics with it.
+     * After 1st pass, function forgets history, and starts a new block.
      * Consequently, this can only work if no data has been previously loaded in tables,
      * aka, no dictionary, no prefix, no ldm preprocessing.
      * The compression ratio gain is generally small (~0.5% on first block),
-     * the cost is 2x cpu time on first block. */
+    ** the cost is 2x cpu time on first block. */
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
     if ( (ms->opt.litLengthSum==0)   /* first block */
       && (seqStore->sequences == seqStore->sequencesStart)  /* no ldm */
       && (ms->window.dictLimit == ms->window.lowLimit)   /* no dictionary */
-      && (curr == ms->window.dictLimit)   /* start of frame, nothing already loaded nor skipped */
-      && (srcSize > ZSTD_PREDEF_THRESHOLD)
+      && (curr == ms->window.dictLimit)    /* start of frame, nothing already loaded nor skipped */
+      && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */
       ) {
         ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
     }

From d07e72bb13f6baab0ba33929c332699b78ca9955 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 28 Dec 2022 17:23:40 -0800
Subject: [PATCH 089/197] fixed incorrect assert

commented Fweight instead
---
 lib/compress/zstd_opt.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index f0f3791e7..9b14093cb 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -26,15 +26,15 @@
 #if 0    /* approximation at bit level (for tests) */
 #  define BITCOST_ACCURACY 0
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat))
+#  define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat))
 #elif 0  /* fractional bit accuracy (for tests) */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat))
+#  define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat))
 #else    /* opt==approx, ultra==accurate */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
+#  define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
 #endif
 
 /* ZSTD_bitWeight() :
@@ -52,10 +52,12 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
     U32 const stat = rawStat + 1;
     U32 const hb = ZSTD_highbit32(stat);
     U32 const BWeight = hb * BITCOST_MULTIPLIER;
+    /* Fweight was meant for "Fractional weight"
+     * but it's effectively a value between 1 and 2
+     * using fixed point arithmetic */
     U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
     U32 const weight = BWeight + FWeight;
     assert(hb + BITCOST_ACCURACY < 31);
-    assert(FWeight < BITCOST_MULTIPLIER);
     return weight;
 }
 

From df714ddb0f2dc076dc841d10c8b72f66ef5af937 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 3 Jan 2023 07:20:21 -0800
Subject: [PATCH 090/197] implement suggestions

---
 lib/compress/huf_compress.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index d7755e0f2..9a4cde6f2 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1264,7 +1264,7 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
         size_t maxBits, hSize, newSize;
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
         const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
-        size_t optSize = ((size_t) ~0);
+        size_t optSize = ((size_t) ~0) - 1;
         unsigned optLogGuess;
 
         if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/
@@ -1275,17 +1275,22 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
 
             if (ERR_isError(maxBits)) continue;
 
+            if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
+
             hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
 
             if (ERR_isError(hSize)) continue;
 
             newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
 
-            if (newSize > optSize) {
+            if (newSize > optSize + 1) {
                 break;
             }
-            optSize = newSize;
-            optLog = optLogGuess;
+
+            if (newSize < optSize) {
+                optSize = newSize;
+                optLog = optLogGuess;
+            }
         }
     }
     assert(optLog <= HUF_TABLELOG_MAX);

From 87becc567d20aeae6265ef6cee0f063aafc05bb8 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 3 Jan 2023 08:41:40 -0800
Subject: [PATCH 091/197] update regression results.csv

---
 tests/regression/results.csv | 60 ++++++++++++++++++------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index ffa9b22e0..4a44dbb00 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -12,9 +12,9 @@ silesia.tar,                        level 7,                            compress
 silesia.tar,                        level 9,                            compress simple,                    4552899
 silesia.tar,                        level 13,                           compress simple,                    4502956
 silesia.tar,                        level 16,                           compress simple,                    4360527
-silesia.tar,                        level 19,                           compress simple,                    4267014
+silesia.tar,                        level 19,                           compress simple,                    4267021
 silesia.tar,                        uncompressed literals,              compress simple,                    4854086
-silesia.tar,                        uncompressed literals optimal,      compress simple,                    4267014
+silesia.tar,                        uncompressed literals optimal,      compress simple,                    4267021
 silesia.tar,                        huffman literals,                   compress simple,                    6179047
 github.tar,                         level -5,                           compress simple,                    52115
 github.tar,                         level -3,                           compress simple,                    45678
@@ -29,9 +29,9 @@ github.tar,                         level 7,                            compress
 github.tar,                         level 9,                            compress simple,                    36760
 github.tar,                         level 13,                           compress simple,                    35501
 github.tar,                         level 16,                           compress simple,                    40471
-github.tar,                         level 19,                           compress simple,                    32127
+github.tar,                         level 19,                           compress simple,                    32149
 github.tar,                         uncompressed literals,              compress simple,                    38831
-github.tar,                         uncompressed literals optimal,      compress simple,                    32127
+github.tar,                         uncompressed literals optimal,      compress simple,                    32149
 github.tar,                         huffman literals,                   compress simple,                    42560
 silesia,                            level -5,                           compress cctx,                      6857372
 silesia,                            level -3,                           compress cctx,                      6503412
@@ -46,7 +46,7 @@ silesia,                            level 7,                            compress
 silesia,                            level 9,                            compress cctx,                      4543018
 silesia,                            level 13,                           compress cctx,                      4493990
 silesia,                            level 16,                           compress cctx,                      4359864
-silesia,                            level 19,                           compress cctx,                      4296439
+silesia,                            level 19,                           compress cctx,                      4296438
 silesia,                            long distance mode,                 compress cctx,                      4842075
 silesia,                            multithreaded,                      compress cctx,                      4842075
 silesia,                            multithreaded long distance mode,   compress cctx,                      4842075
@@ -55,7 +55,7 @@ silesia,                            small hash log,                     compress
 silesia,                            small chain log,                    compress cctx,                      4912197
 silesia,                            explicit params,                    compress cctx,                      4794052
 silesia,                            uncompressed literals,              compress cctx,                      4842075
-silesia,                            uncompressed literals optimal,      compress cctx,                      4296439
+silesia,                            uncompressed literals optimal,      compress cctx,                      4296438
 silesia,                            huffman literals,                   compress cctx,                      6172202
 silesia,                            multithreaded with advanced params, compress cctx,                      4842075
 github,                             level -5,                           compress cctx,                      204407
@@ -110,7 +110,7 @@ silesia,                            level 7,                            zstdcli,
 silesia,                            level 9,                            zstdcli,                            4543066
 silesia,                            level 13,                           zstdcli,                            4494038
 silesia,                            level 16,                           zstdcli,                            4359912
-silesia,                            level 19,                           zstdcli,                            4296487
+silesia,                            level 19,                           zstdcli,                            4296486
 silesia,                            long distance mode,                 zstdcli,                            4833785
 silesia,                            multithreaded,                      zstdcli,                            4842123
 silesia,                            multithreaded long distance mode,   zstdcli,                            4833785
@@ -135,7 +135,7 @@ silesia.tar,                        level 7,                            zstdcli,
 silesia.tar,                        level 9,                            zstdcli,                            4552903
 silesia.tar,                        level 13,                           zstdcli,                            4502960
 silesia.tar,                        level 16,                           zstdcli,                            4360531
-silesia.tar,                        level 19,                           zstdcli,                            4267018
+silesia.tar,                        level 19,                           zstdcli,                            4267025
 silesia.tar,                        no source size,                     zstdcli,                            4854160
 silesia.tar,                        long distance mode,                 zstdcli,                            4845745
 silesia.tar,                        multithreaded,                      zstdcli,                            4854164
@@ -213,7 +213,7 @@ github.tar,                         level 13,                           zstdcli,
 github.tar,                         level 13 with dict,                 zstdcli,                            37134
 github.tar,                         level 16,                           zstdcli,                            40475
 github.tar,                         level 16 with dict,                 zstdcli,                            33378
-github.tar,                         level 19,                           zstdcli,                            32131
+github.tar,                         level 19,                           zstdcli,                            32153
 github.tar,                         level 19 with dict,                 zstdcli,                            32716
 github.tar,                         no source size,                     zstdcli,                            38832
 github.tar,                         no source size with dict,           zstdcli,                            38004
@@ -249,7 +249,7 @@ silesia,                            level 12 row 1,                     advanced
 silesia,                            level 12 row 2,                     advanced one pass,                  4503116
 silesia,                            level 13,                           advanced one pass,                  4493990
 silesia,                            level 16,                           advanced one pass,                  4359864
-silesia,                            level 19,                           advanced one pass,                  4296439
+silesia,                            level 19,                           advanced one pass,                  4296438
 silesia,                            no source size,                     advanced one pass,                  4842075
 silesia,                            long distance mode,                 advanced one pass,                  4833710
 silesia,                            multithreaded,                      advanced one pass,                  4842075
@@ -283,7 +283,7 @@ silesia.tar,                        level 12 row 1,                     advanced
 silesia.tar,                        level 12 row 2,                     advanced one pass,                  4513797
 silesia.tar,                        level 13,                           advanced one pass,                  4502956
 silesia.tar,                        level 16,                           advanced one pass,                  4360527
-silesia.tar,                        level 19,                           advanced one pass,                  4267014
+silesia.tar,                        level 19,                           advanced one pass,                  4267021
 silesia.tar,                        no source size,                     advanced one pass,                  4854086
 silesia.tar,                        long distance mode,                 advanced one pass,                  4840452
 silesia.tar,                        multithreaded,                      advanced one pass,                  4854160
@@ -527,7 +527,7 @@ github.tar,                         level 16 with dict dms,             advanced
 github.tar,                         level 16 with dict dds,             advanced one pass,                  33206
 github.tar,                         level 16 with dict copy,            advanced one pass,                  33374
 github.tar,                         level 16 with dict load,            advanced one pass,                  39081
-github.tar,                         level 19,                           advanced one pass,                  32127
+github.tar,                         level 19,                           advanced one pass,                  32149
 github.tar,                         level 19 with dict,                 advanced one pass,                  32712
 github.tar,                         level 19 with dict dms,             advanced one pass,                  32555
 github.tar,                         level 19 with dict dds,             advanced one pass,                  32555
@@ -567,7 +567,7 @@ silesia,                            level 12 row 1,                     advanced
 silesia,                            level 12 row 2,                     advanced one pass small out,        4503116
 silesia,                            level 13,                           advanced one pass small out,        4493990
 silesia,                            level 16,                           advanced one pass small out,        4359864
-silesia,                            level 19,                           advanced one pass small out,        4296439
+silesia,                            level 19,                           advanced one pass small out,        4296438
 silesia,                            no source size,                     advanced one pass small out,        4842075
 silesia,                            long distance mode,                 advanced one pass small out,        4833710
 silesia,                            multithreaded,                      advanced one pass small out,        4842075
@@ -601,7 +601,7 @@ silesia.tar,                        level 12 row 1,                     advanced
 silesia.tar,                        level 12 row 2,                     advanced one pass small out,        4513797
 silesia.tar,                        level 13,                           advanced one pass small out,        4502956
 silesia.tar,                        level 16,                           advanced one pass small out,        4360527
-silesia.tar,                        level 19,                           advanced one pass small out,        4267014
+silesia.tar,                        level 19,                           advanced one pass small out,        4267021
 silesia.tar,                        no source size,                     advanced one pass small out,        4854086
 silesia.tar,                        long distance mode,                 advanced one pass small out,        4840452
 silesia.tar,                        multithreaded,                      advanced one pass small out,        4854160
@@ -845,7 +845,7 @@ github.tar,                         level 16 with dict dms,             advanced
 github.tar,                         level 16 with dict dds,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict copy,            advanced one pass small out,        33374
 github.tar,                         level 16 with dict load,            advanced one pass small out,        39081
-github.tar,                         level 19,                           advanced one pass small out,        32127
+github.tar,                         level 19,                           advanced one pass small out,        32149
 github.tar,                         level 19 with dict,                 advanced one pass small out,        32712
 github.tar,                         level 19 with dict dms,             advanced one pass small out,        32555
 github.tar,                         level 19 with dict dds,             advanced one pass small out,        32555
@@ -885,7 +885,7 @@ silesia,                            level 12 row 1,                     advanced
 silesia,                            level 12 row 2,                     advanced streaming,                 4503116
 silesia,                            level 13,                           advanced streaming,                 4493990
 silesia,                            level 16,                           advanced streaming,                 4359864
-silesia,                            level 19,                           advanced streaming,                 4296439
+silesia,                            level 19,                           advanced streaming,                 4296438
 silesia,                            no source size,                     advanced streaming,                 4842039
 silesia,                            long distance mode,                 advanced streaming,                 4833710
 silesia,                            multithreaded,                      advanced streaming,                 4842075
@@ -919,7 +919,7 @@ silesia.tar,                        level 12 row 1,                     advanced
 silesia.tar,                        level 12 row 2,                     advanced streaming,                 4513797
 silesia.tar,                        level 13,                           advanced streaming,                 4502956
 silesia.tar,                        level 16,                           advanced streaming,                 4360527
-silesia.tar,                        level 19,                           advanced streaming,                 4267014
+silesia.tar,                        level 19,                           advanced streaming,                 4267021
 silesia.tar,                        no source size,                     advanced streaming,                 4859267
 silesia.tar,                        long distance mode,                 advanced streaming,                 4840452
 silesia.tar,                        multithreaded,                      advanced streaming,                 4854160
@@ -1163,7 +1163,7 @@ github.tar,                         level 16 with dict dms,             advanced
 github.tar,                         level 16 with dict dds,             advanced streaming,                 33206
 github.tar,                         level 16 with dict copy,            advanced streaming,                 33374
 github.tar,                         level 16 with dict load,            advanced streaming,                 39081
-github.tar,                         level 19,                           advanced streaming,                 32127
+github.tar,                         level 19,                           advanced streaming,                 32149
 github.tar,                         level 19 with dict,                 advanced streaming,                 32712
 github.tar,                         level 19 with dict dms,             advanced streaming,                 32555
 github.tar,                         level 19 with dict dds,             advanced streaming,                 32555
@@ -1195,10 +1195,10 @@ silesia,                            level 7,                            old stre
 silesia,                            level 9,                            old streaming,                      4543018
 silesia,                            level 13,                           old streaming,                      4493990
 silesia,                            level 16,                           old streaming,                      4359864
-silesia,                            level 19,                           old streaming,                      4296439
+silesia,                            level 19,                           old streaming,                      4296438
 silesia,                            no source size,                     old streaming,                      4842039
 silesia,                            uncompressed literals,              old streaming,                      4842075
-silesia,                            uncompressed literals optimal,      old streaming,                      4296439
+silesia,                            uncompressed literals optimal,      old streaming,                      4296438
 silesia,                            huffman literals,                   old streaming,                      6172207
 silesia.tar,                        level -5,                           old streaming,                      6856523
 silesia.tar,                        level -3,                           old streaming,                      6505954
@@ -1213,10 +1213,10 @@ silesia.tar,                        level 7,                            old stre
 silesia.tar,                        level 9,                            old streaming,                      4552900
 silesia.tar,                        level 13,                           old streaming,                      4502956
 silesia.tar,                        level 16,                           old streaming,                      4360527
-silesia.tar,                        level 19,                           old streaming,                      4267014
+silesia.tar,                        level 19,                           old streaming,                      4267021
 silesia.tar,                        no source size,                     old streaming,                      4859267
 silesia.tar,                        uncompressed literals,              old streaming,                      4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming,                      4267014
+silesia.tar,                        uncompressed literals optimal,      old streaming,                      4267021
 silesia.tar,                        huffman literals,                   old streaming,                      6179056
 github,                             level -5,                           old streaming,                      204407
 github,                             level -5 with dict,                 old streaming,                      46718
@@ -1277,12 +1277,12 @@ github.tar,                         level 13,                           old stre
 github.tar,                         level 13 with dict,                 old streaming,                      37130
 github.tar,                         level 16,                           old streaming,                      40471
 github.tar,                         level 16 with dict,                 old streaming,                      33374
-github.tar,                         level 19,                           old streaming,                      32127
+github.tar,                         level 19,                           old streaming,                      32149
 github.tar,                         level 19 with dict,                 old streaming,                      32712
 github.tar,                         no source size,                     old streaming,                      38828
 github.tar,                         no source size with dict,           old streaming,                      38000
 github.tar,                         uncompressed literals,              old streaming,                      38831
-github.tar,                         uncompressed literals optimal,      old streaming,                      32127
+github.tar,                         uncompressed literals optimal,      old streaming,                      32149
 github.tar,                         huffman literals,                   old streaming,                      42560
 silesia,                            level -5,                           old streaming advanced,             6854744
 silesia,                            level -3,                           old streaming advanced,             6503319
@@ -1297,7 +1297,7 @@ silesia,                            level 7,                            old stre
 silesia,                            level 9,                            old streaming advanced,             4543018
 silesia,                            level 13,                           old streaming advanced,             4493990
 silesia,                            level 16,                           old streaming advanced,             4359864
-silesia,                            level 19,                           old streaming advanced,             4296439
+silesia,                            level 19,                           old streaming advanced,             4296438
 silesia,                            no source size,                     old streaming advanced,             4842039
 silesia,                            long distance mode,                 old streaming advanced,             4842075
 silesia,                            multithreaded,                      old streaming advanced,             4842075
@@ -1307,7 +1307,7 @@ silesia,                            small hash log,                     old stre
 silesia,                            small chain log,                    old streaming advanced,             4912197
 silesia,                            explicit params,                    old streaming advanced,             4795452
 silesia,                            uncompressed literals,              old streaming advanced,             4842075
-silesia,                            uncompressed literals optimal,      old streaming advanced,             4296439
+silesia,                            uncompressed literals optimal,      old streaming advanced,             4296438
 silesia,                            huffman literals,                   old streaming advanced,             6172207
 silesia,                            multithreaded with advanced params, old streaming advanced,             4842075
 silesia.tar,                        level -5,                           old streaming advanced,             6856523
@@ -1323,7 +1323,7 @@ silesia.tar,                        level 7,                            old stre
 silesia.tar,                        level 9,                            old streaming advanced,             4552900
 silesia.tar,                        level 13,                           old streaming advanced,             4502956
 silesia.tar,                        level 16,                           old streaming advanced,             4360527
-silesia.tar,                        level 19,                           old streaming advanced,             4267014
+silesia.tar,                        level 19,                           old streaming advanced,             4267021
 silesia.tar,                        no source size,                     old streaming advanced,             4859267
 silesia.tar,                        long distance mode,                 old streaming advanced,             4859271
 silesia.tar,                        multithreaded,                      old streaming advanced,             4859271
@@ -1333,7 +1333,7 @@ silesia.tar,                        small hash log,                     old stre
 silesia.tar,                        small chain log,                    old streaming advanced,             4917021
 silesia.tar,                        explicit params,                    old streaming advanced,             4806873
 silesia.tar,                        uncompressed literals,              old streaming advanced,             4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4267014
+silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4267021
 silesia.tar,                        huffman literals,                   old streaming advanced,             6179056
 silesia.tar,                        multithreaded with advanced params, old streaming advanced,             4859271
 github,                             level -5,                           old streaming advanced,             213265
@@ -1403,7 +1403,7 @@ github.tar,                         level 13,                           old stre
 github.tar,                         level 13 with dict,                 old streaming advanced,             35807
 github.tar,                         level 16,                           old streaming advanced,             40471
 github.tar,                         level 16 with dict,                 old streaming advanced,             38578
-github.tar,                         level 19,                           old streaming advanced,             32127
+github.tar,                         level 19,                           old streaming advanced,             32149
 github.tar,                         level 19 with dict,                 old streaming advanced,             32704
 github.tar,                         no source size,                     old streaming advanced,             38828
 github.tar,                         no source size with dict,           old streaming advanced,             38015
@@ -1415,7 +1415,7 @@ github.tar,                         small hash log,                     old stre
 github.tar,                         small chain log,                    old streaming advanced,             41669
 github.tar,                         explicit params,                    old streaming advanced,             41385
 github.tar,                         uncompressed literals,              old streaming advanced,             38831
-github.tar,                         uncompressed literals optimal,      old streaming advanced,             32127
+github.tar,                         uncompressed literals optimal,      old streaming advanced,             32149
 github.tar,                         huffman literals,                   old streaming advanced,             42560
 github.tar,                         multithreaded with advanced params, old streaming advanced,             38831
 github,                             level -5 with dict,                 old streaming cdict,                46718

From 5434de01e21672cdd3ac111a99a969d8c5079297 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 21 Dec 2022 14:58:53 -0800
Subject: [PATCH 092/197] improve compression ratio of small alphabets

fix #3328

In situations where the alphabet size is very small,
the evaluation of literal costs from the Optimal Parser is initially incorrect.
It takes some time to converge, during which compression is less efficient.
This is especially important for small files,
because there will not be enough data to converge,
so most of the parsing is selected based on incorrect metrics.

After this patch, the scenario ##3328 gets fixed,
delivering the expected 29 bytes compressed size (smallest known compressed size).
---
 lib/compress/huf_compress.c           | 16 ++++++--
 lib/compress/zstd_compress.c          | 11 +++---
 lib/compress/zstd_compress_literals.c | 57 ++++++++++++++++++---------
 lib/compress/zstd_compress_literals.h | 18 +++++----
 lib/compress/zstd_opt.c               | 16 +++++---
 5 files changed, 77 insertions(+), 41 deletions(-)

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 0da96d694..007bb15e7 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -386,7 +386,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits
 
         /* renorm totalCost from 2^largestBits to 2^targetNbBits
          * note : totalCost is necessarily a multiple of baseCost */
-        assert((totalCost & (baseCost - 1)) == 0);
+        assert(((U32)totalCost & (baseCost - 1)) == 0);
         totalCost >>= (largestBits - targetNbBits);
         assert(totalCost > 0);
 
@@ -1253,7 +1253,14 @@ unsigned HUF_minTableLog(unsigned symbolCardinality)
     return minBitsSymbols;
 }
 
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
+unsigned HUF_optimalTableLog(
+            unsigned maxTableLog,
+            size_t srcSize,
+            unsigned maxSymbolValue,
+            void* workSpace, size_t wkspSize,
+            HUF_CElt* table,
+      const unsigned* count,
+            HUF_depth_mode depthMode)
 {
     unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
@@ -1267,12 +1274,13 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
         size_t optSize = ((size_t) ~0) - 1;
         unsigned optLogGuess;
 
-        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/
+        DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
+        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /* silently return if workspace is not large enough  */
 
         /* Search until size increases */
         for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+            DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
             maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
-
             if (ERR_isError(maxBits)) continue;
 
             if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 759abe00a..b12cacc77 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2727,15 +2727,14 @@ ZSTD_entropyCompressSeqStore_internal(
         unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
         size_t const litSize = (size_t)(seqStorePtr->lit - literals);
 
-        HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
         size_t const cSize = ZSTD_compressLiterals(
-                                    &prevEntropy->huf, &nextEntropy->huf,
-                                    cctxParams->cParams.strategy,
-                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
                                     op, dstCapacity,
                                     literals, litSize,
                                     entropyWorkspace, entropyWkspSize,
-                                    bmi2, suspectUncompressible, depthMode);
+                                    &prevEntropy->huf, &nextEntropy->huf,
+                                    cctxParams->cParams.strategy,
+                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
+                                    suspectUncompressible, bmi2);
         FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
         assert(cSize <= dstCapacity);
         op += cSize;
@@ -3878,12 +3877,12 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end
     size_t estimatedSecondHalfSize;
     size_t midIdx = (startIdx + endIdx)/2;
 
+    DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
     assert(endIdx >= startIdx);
     if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) {
         DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx);
         return;
     }
-    DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx);
     ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx);
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index 972d40e71..666e5315d 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -92,16 +92,37 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
     return flSize+1;
 }
 
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
-                        const void* src, size_t srcSize,
-                              void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible, HUF_depth_mode depthMode)
+/* ZSTD_minLiteralsToCompress() :
+ * returns minimal amount of literals
+ * for literal compression to even be attempted.
+ * Minimum is made tighter as compression strategy increases.
+ */
+static size_t
+ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
+{
+    assert((int)strategy >= 0);
+    assert((int)strategy <= 9);
+    /* btultra2 : min 8 bytes;
+     * then 2x larger for each successive compression strategy
+     * max threshold 64 bytes */
+    {   int const shift = MIN(9-strategy, 3);
+        size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : 8 << shift;
+        DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
+        return mintc;
+    }
+}
+
+size_t ZSTD_compressLiterals (
+                  void* dst, size_t dstCapacity,
+            const void* src, size_t srcSize,
+                  void* entropyWorkspace, size_t entropyWorkspaceSize,
+            const ZSTD_hufCTables_t* prevHuf,
+                  ZSTD_hufCTables_t* nextHuf,
+                  ZSTD_strategy strategy,
+                  int disableLiteralCompression,
+                  int suspectUncompressible,
+                  int bmi2)
 {
-    size_t const minGain = ZSTD_minGain(srcSize, strategy);
     size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
     BYTE*  const ostart = (BYTE*)dst;
     U32 singleStream = srcSize < 256;
@@ -119,15 +140,14 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     if (disableLiteralCompression)
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
-    /* small ? don't even attempt compression (speed opt) */
-#   define COMPRESS_LITERALS_SIZE_MIN 63
-    {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
-        if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    /* if too small, don't even attempt compression (speed opt) */
+    if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
+        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
     RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
     {   HUF_repeat repeat = prevHuf->repeatMode;
         int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
+        HUF_depth_mode const depthMode = (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD) ? HUF_depth_optimal : HUF_depth_fast;
         typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode);
         huf_compress_f huf_compress;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
@@ -146,10 +166,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
         }
     }
 
-    if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
-        ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    {   size_t const minGain = ZSTD_minGain(srcSize, strategy);
+        if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+    }   }
     if (cLitSize==1) {
         ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
         return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h
index e5da204c8..9eb74729d 100644
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@@ -18,14 +18,18 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
 
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
-/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
+/* ZSTD_compressLiterals():
+ * @entropyWorkspace: must be aligned on 4-bytes boundaries
+ * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
+ * @suspectUncompressible: sampling checks, to potentially skip huffman coding
+ */
+size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
                         const void* src, size_t srcSize,
                               void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible, HUF_depth_mode depthMode);
+                        const ZSTD_hufCTables_t* prevHuf,
+                              ZSTD_hufCTables_t* nextHuf,
+                              ZSTD_strategy strategy, int disableLiteralCompression,
+                              int suspectUncompressible,
+                              int bmi2);
 
 #endif /* ZSTD_COMPRESS_LITERALS_H */
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 9b14093cb..68d24aa7b 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -16,7 +16,7 @@
 #define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
 #define ZSTD_MAX_PRICE     (1<<30)
 
-#define ZSTD_PREDEF_THRESHOLD 1024   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
+#define ZSTD_PREDEF_THRESHOLD 8   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
 
 
 /*-*************************************
@@ -96,14 +96,18 @@ static U32 sum_u32(const unsigned table[], size_t nbElts)
     return total;
 }
 
-static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift)
+typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e;
+
+static U32
+ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1)
 {
     U32 s, sum=0;
     DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
             (unsigned)lastEltIndex+1, (unsigned)shift );
     assert(shift < 30);
     for (s=0; s<lastEltIndex+1; s++) {
-        unsigned newStat = 1 + (table[s] >> shift);
+        unsigned const base = base1 ? 1 : (table[s]>0);
+        unsigned const newStat = base + (table[s] >> shift);
         sum += newStat;
         table[s] = newStat;
     }
@@ -120,7 +124,7 @@ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
     DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
     assert(logTarget < 30);
     if (factor <= 1) return prevsum;
-    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor));
+    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed);
 }
 
 /* ZSTD_rescaleFreqs() :
@@ -202,14 +206,14 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                     optPtr->offCodeSum += optPtr->offCodeFreq[of];
             }   }
 
-        } else {  /* huf.repeatMode != HUF_repeat_valid => presumed not a dictionary */
+        } else {  /* first block, no dictionary */
 
             assert(optPtr->litFreq != NULL);
             if (compressedLiterals) {
                 /* base initial cost of literals on direct frequency within src */
                 unsigned lit = MaxLit;
                 HIST_count_simple(optPtr->litFreq, &lit, src, srcSize);   /* use raw first block to init statistics */
-                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8);
+                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
             }
 
             {   unsigned const baseLLfreqs[MaxLL+1] = {

From ebba9ff4259d874030cce14c6a34e988e5679a50 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 3 Jan 2023 14:04:23 -0800
Subject: [PATCH 093/197] update regression results

---
 lib/compress/huf_compress.c  |  17 ++--
 tests/regression/results.csv | 190 +++++++++++++++++------------------
 2 files changed, 105 insertions(+), 102 deletions(-)

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 007bb15e7..f7ca2d3bb 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1262,20 +1262,23 @@ unsigned HUF_optimalTableLog(
       const unsigned* count,
             HUF_depth_mode depthMode)
 {
-    unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
+    assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
 
-    if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/
-        BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+    if (depthMode != HUF_depth_optimal) {
+        /* cheap evaluation, based on FSE */
+        return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+    }
+
+    {   BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
         size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
         size_t maxBits, hSize, newSize;
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
         const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
         size_t optSize = ((size_t) ~0) - 1;
-        unsigned optLogGuess;
+        unsigned optLog = maxTableLog, optLogGuess;
 
         DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
-        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /* silently return if workspace is not large enough  */
 
         /* Search until size increases */
         for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
@@ -1300,9 +1303,9 @@ unsigned HUF_optimalTableLog(
                 optLog = optLogGuess;
             }
         }
+        assert(optLog <= HUF_TABLELOG_MAX);
+        return optLog;
     }
-    assert(optLog <= HUF_TABLELOG_MAX);
-    return optLog;
 }
 
 /* HUF_compress_internal() :
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 4a44dbb00..188eea900 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -11,10 +11,10 @@ silesia.tar,                        level 6,                            compress
 silesia.tar,                        level 7,                            compress simple,                    4576661
 silesia.tar,                        level 9,                            compress simple,                    4552899
 silesia.tar,                        level 13,                           compress simple,                    4502956
-silesia.tar,                        level 16,                           compress simple,                    4360527
-silesia.tar,                        level 19,                           compress simple,                    4267021
+silesia.tar,                        level 16,                           compress simple,                    4360546
+silesia.tar,                        level 19,                           compress simple,                    4265911
 silesia.tar,                        uncompressed literals,              compress simple,                    4854086
-silesia.tar,                        uncompressed literals optimal,      compress simple,                    4267021
+silesia.tar,                        uncompressed literals optimal,      compress simple,                    4265911
 silesia.tar,                        huffman literals,                   compress simple,                    6179047
 github.tar,                         level -5,                           compress simple,                    52115
 github.tar,                         level -3,                           compress simple,                    45678
@@ -28,10 +28,10 @@ github.tar,                         level 6,                            compress
 github.tar,                         level 7,                            compress simple,                    38110
 github.tar,                         level 9,                            compress simple,                    36760
 github.tar,                         level 13,                           compress simple,                    35501
-github.tar,                         level 16,                           compress simple,                    40471
-github.tar,                         level 19,                           compress simple,                    32149
+github.tar,                         level 16,                           compress simple,                    40466
+github.tar,                         level 19,                           compress simple,                    32276
 github.tar,                         uncompressed literals,              compress simple,                    38831
-github.tar,                         uncompressed literals optimal,      compress simple,                    32149
+github.tar,                         uncompressed literals optimal,      compress simple,                    32276
 github.tar,                         huffman literals,                   compress simple,                    42560
 silesia,                            level -5,                           compress cctx,                      6857372
 silesia,                            level -3,                           compress cctx,                      6503412
@@ -45,8 +45,8 @@ silesia,                            level 6,                            compress
 silesia,                            level 7,                            compress cctx,                      4566984
 silesia,                            level 9,                            compress cctx,                      4543018
 silesia,                            level 13,                           compress cctx,                      4493990
-silesia,                            level 16,                           compress cctx,                      4359864
-silesia,                            level 19,                           compress cctx,                      4296438
+silesia,                            level 16,                           compress cctx,                      4360041
+silesia,                            level 19,                           compress cctx,                      4296055
 silesia,                            long distance mode,                 compress cctx,                      4842075
 silesia,                            multithreaded,                      compress cctx,                      4842075
 silesia,                            multithreaded long distance mode,   compress cctx,                      4842075
@@ -55,7 +55,7 @@ silesia,                            small hash log,                     compress
 silesia,                            small chain log,                    compress cctx,                      4912197
 silesia,                            explicit params,                    compress cctx,                      4794052
 silesia,                            uncompressed literals,              compress cctx,                      4842075
-silesia,                            uncompressed literals optimal,      compress cctx,                      4296438
+silesia,                            uncompressed literals optimal,      compress cctx,                      4296055
 silesia,                            huffman literals,                   compress cctx,                      6172202
 silesia,                            multithreaded with advanced params, compress cctx,                      4842075
 github,                             level -5,                           compress cctx,                      204407
@@ -80,11 +80,11 @@ github,                             level 7,                            compress
 github,                             level 7 with dict,                  compress cctx,                      38755
 github,                             level 9,                            compress cctx,                      135122
 github,                             level 9 with dict,                  compress cctx,                      39398
-github,                             level 13,                           compress cctx,                      132729
+github,                             level 13,                           compress cctx,                      132878
 github,                             level 13 with dict,                 compress cctx,                      39948
-github,                             level 16,                           compress cctx,                      132729
+github,                             level 16,                           compress cctx,                      133209
 github,                             level 16 with dict,                 compress cctx,                      37568
-github,                             level 19,                           compress cctx,                      132729
+github,                             level 19,                           compress cctx,                      132879
 github,                             level 19 with dict,                 compress cctx,                      37567
 github,                             long distance mode,                 compress cctx,                      141069
 github,                             multithreaded,                      compress cctx,                      141069
@@ -94,7 +94,7 @@ github,                             small hash log,                     compress
 github,                             small chain log,                    compress cctx,                      139242
 github,                             explicit params,                    compress cctx,                      140932
 github,                             uncompressed literals,              compress cctx,                      136332
-github,                             uncompressed literals optimal,      compress cctx,                      132729
+github,                             uncompressed literals optimal,      compress cctx,                      132879
 github,                             huffman literals,                   compress cctx,                      175468
 github,                             multithreaded with advanced params, compress cctx,                      141069
 silesia,                            level -5,                           zstdcli,                            6857420
@@ -109,8 +109,8 @@ silesia,                            level 6,                            zstdcli,
 silesia,                            level 7,                            zstdcli,                            4567032
 silesia,                            level 9,                            zstdcli,                            4543066
 silesia,                            level 13,                           zstdcli,                            4494038
-silesia,                            level 16,                           zstdcli,                            4359912
-silesia,                            level 19,                           zstdcli,                            4296486
+silesia,                            level 16,                           zstdcli,                            4360089
+silesia,                            level 19,                           zstdcli,                            4296103
 silesia,                            long distance mode,                 zstdcli,                            4833785
 silesia,                            multithreaded,                      zstdcli,                            4842123
 silesia,                            multithreaded long distance mode,   zstdcli,                            4833785
@@ -134,14 +134,14 @@ silesia.tar,                        level 6,                            zstdcli,
 silesia.tar,                        level 7,                            zstdcli,                            4578719
 silesia.tar,                        level 9,                            zstdcli,                            4552903
 silesia.tar,                        level 13,                           zstdcli,                            4502960
-silesia.tar,                        level 16,                           zstdcli,                            4360531
-silesia.tar,                        level 19,                           zstdcli,                            4267025
+silesia.tar,                        level 16,                           zstdcli,                            4360550
+silesia.tar,                        level 19,                           zstdcli,                            4265915
 silesia.tar,                        no source size,                     zstdcli,                            4854160
 silesia.tar,                        long distance mode,                 zstdcli,                            4845745
 silesia.tar,                        multithreaded,                      zstdcli,                            4854164
 silesia.tar,                        multithreaded long distance mode,   zstdcli,                            4845745
 silesia.tar,                        small window log,                   zstdcli,                            7100701
-silesia.tar,                        small hash log,                     zstdcli,                            6529289
+silesia.tar,                        small hash log,                     zstdcli,                            6529264
 silesia.tar,                        small chain log,                    zstdcli,                            4917022
 silesia.tar,                        explicit params,                    zstdcli,                            4820713
 silesia.tar,                        uncompressed literals,              zstdcli,                            5122571
@@ -170,11 +170,11 @@ github,                             level 7,                            zstdcli,
 github,                             level 7 with dict,                  zstdcli,                            40745
 github,                             level 9,                            zstdcli,                            137122
 github,                             level 9 with dict,                  zstdcli,                            41393
-github,                             level 13,                           zstdcli,                            134729
+github,                             level 13,                           zstdcli,                            134878
 github,                             level 13 with dict,                 zstdcli,                            41900
-github,                             level 16,                           zstdcli,                            134729
+github,                             level 16,                           zstdcli,                            135209
 github,                             level 16 with dict,                 zstdcli,                            39577
-github,                             level 19,                           zstdcli,                            134729
+github,                             level 19,                           zstdcli,                            134879
 github,                             level 19 with dict,                 zstdcli,                            39576
 github,                             long distance mode,                 zstdcli,                            138332
 github,                             multithreaded,                      zstdcli,                            138332
@@ -184,7 +184,7 @@ github,                             small hash log,                     zstdcli,
 github,                             small chain log,                    zstdcli,                            138341
 github,                             explicit params,                    zstdcli,                            136197
 github,                             uncompressed literals,              zstdcli,                            167911
-github,                             uncompressed literals optimal,      zstdcli,                            159227
+github,                             uncompressed literals optimal,      zstdcli,                            154667
 github,                             huffman literals,                   zstdcli,                            144365
 github,                             multithreaded with advanced params, zstdcli,                            167911
 github.tar,                         level -5,                           zstdcli,                            52119
@@ -211,9 +211,9 @@ github.tar,                         level 9,                            zstdcli,
 github.tar,                         level 9 with dict,                  zstdcli,                            36632
 github.tar,                         level 13,                           zstdcli,                            35505
 github.tar,                         level 13 with dict,                 zstdcli,                            37134
-github.tar,                         level 16,                           zstdcli,                            40475
+github.tar,                         level 16,                           zstdcli,                            40470
 github.tar,                         level 16 with dict,                 zstdcli,                            33378
-github.tar,                         level 19,                           zstdcli,                            32153
+github.tar,                         level 19,                           zstdcli,                            32280
 github.tar,                         level 19 with dict,                 zstdcli,                            32716
 github.tar,                         no source size,                     zstdcli,                            38832
 github.tar,                         no source size with dict,           zstdcli,                            38004
@@ -248,8 +248,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced one pass,                  4505046
 silesia,                            level 12 row 2,                     advanced one pass,                  4503116
 silesia,                            level 13,                           advanced one pass,                  4493990
-silesia,                            level 16,                           advanced one pass,                  4359864
-silesia,                            level 19,                           advanced one pass,                  4296438
+silesia,                            level 16,                           advanced one pass,                  4360041
+silesia,                            level 19,                           advanced one pass,                  4296055
 silesia,                            no source size,                     advanced one pass,                  4842075
 silesia,                            long distance mode,                 advanced one pass,                  4833710
 silesia,                            multithreaded,                      advanced one pass,                  4842075
@@ -282,14 +282,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced one pass,                  4514049
 silesia.tar,                        level 12 row 2,                     advanced one pass,                  4513797
 silesia.tar,                        level 13,                           advanced one pass,                  4502956
-silesia.tar,                        level 16,                           advanced one pass,                  4360527
-silesia.tar,                        level 19,                           advanced one pass,                  4267021
+silesia.tar,                        level 16,                           advanced one pass,                  4360546
+silesia.tar,                        level 19,                           advanced one pass,                  4265911
 silesia.tar,                        no source size,                     advanced one pass,                  4854086
 silesia.tar,                        long distance mode,                 advanced one pass,                  4840452
 silesia.tar,                        multithreaded,                      advanced one pass,                  4854160
 silesia.tar,                        multithreaded long distance mode,   advanced one pass,                  4845741
 silesia.tar,                        small window log,                   advanced one pass,                  7100655
-silesia.tar,                        small hash log,                     advanced one pass,                  6529231
+silesia.tar,                        small hash log,                     advanced one pass,                  6529206
 silesia.tar,                        small chain log,                    advanced one pass,                  4917041
 silesia.tar,                        explicit params,                    advanced one pass,                  4806855
 silesia.tar,                        uncompressed literals,              advanced one pass,                  5122473
@@ -370,39 +370,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced one pass,                  39393
 github,                             level 9 with dict copy,             advanced one pass,                  39398
 github,                             level 9 with dict load,             advanced one pass,                  41710
-github,                             level 11 row 1,                     advanced one pass,                  135119
+github,                             level 11 row 1,                     advanced one pass,                  135367
 github,                             level 11 row 1 with dict dms,       advanced one pass,                  39671
 github,                             level 11 row 1 with dict dds,       advanced one pass,                  39671
 github,                             level 11 row 1 with dict copy,      advanced one pass,                  39651
 github,                             level 11 row 1 with dict load,      advanced one pass,                  41360
-github,                             level 11 row 2,                     advanced one pass,                  135119
+github,                             level 11 row 2,                     advanced one pass,                  135367
 github,                             level 11 row 2 with dict dms,       advanced one pass,                  39671
 github,                             level 11 row 2 with dict dds,       advanced one pass,                  39671
 github,                             level 11 row 2 with dict copy,      advanced one pass,                  39651
 github,                             level 11 row 2 with dict load,      advanced one pass,                  41360
-github,                             level 12 row 1,                     advanced one pass,                  134180
+github,                             level 12 row 1,                     advanced one pass,                  134402
 github,                             level 12 row 1 with dict dms,       advanced one pass,                  39677
 github,                             level 12 row 1 with dict dds,       advanced one pass,                  39677
 github,                             level 12 row 1 with dict copy,      advanced one pass,                  39677
 github,                             level 12 row 1 with dict load,      advanced one pass,                  41166
-github,                             level 12 row 2,                     advanced one pass,                  134180
+github,                             level 12 row 2,                     advanced one pass,                  134402
 github,                             level 12 row 2 with dict dms,       advanced one pass,                  39677
 github,                             level 12 row 2 with dict dds,       advanced one pass,                  39677
 github,                             level 12 row 2 with dict copy,      advanced one pass,                  39677
 github,                             level 12 row 2 with dict load,      advanced one pass,                  41166
-github,                             level 13,                           advanced one pass,                  132729
+github,                             level 13,                           advanced one pass,                  132878
 github,                             level 13 with dict,                 advanced one pass,                  39900
 github,                             level 13 with dict dms,             advanced one pass,                  39900
 github,                             level 13 with dict dds,             advanced one pass,                  39900
 github,                             level 13 with dict copy,            advanced one pass,                  39948
 github,                             level 13 with dict load,            advanced one pass,                  42624
-github,                             level 16,                           advanced one pass,                  132729
+github,                             level 16,                           advanced one pass,                  133209
 github,                             level 16 with dict,                 advanced one pass,                  37577
 github,                             level 16 with dict dms,             advanced one pass,                  37577
 github,                             level 16 with dict dds,             advanced one pass,                  37577
 github,                             level 16 with dict copy,            advanced one pass,                  37568
 github,                             level 16 with dict load,            advanced one pass,                  42338
-github,                             level 19,                           advanced one pass,                  132729
+github,                             level 19,                           advanced one pass,                  132879
 github,                             level 19 with dict,                 advanced one pass,                  37576
 github,                             level 19 with dict dms,             advanced one pass,                  37576
 github,                             level 19 with dict dds,             advanced one pass,                  37576
@@ -418,7 +418,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced one pass,                  136341
 github,                             explicit params,                    advanced one pass,                  137727
 github,                             uncompressed literals,              advanced one pass,                  165911
-github,                             uncompressed literals optimal,      advanced one pass,                  157227
+github,                             uncompressed literals optimal,      advanced one pass,                  152667
 github,                             huffman literals,                   advanced one pass,                  142365
 github,                             multithreaded with advanced params, advanced one pass,                  165911
 github.tar,                         level -5,                           advanced one pass,                  52115
@@ -521,13 +521,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced one pass,                  37220
 github.tar,                         level 13 with dict copy,            advanced one pass,                  37130
 github.tar,                         level 13 with dict load,            advanced one pass,                  36010
-github.tar,                         level 16,                           advanced one pass,                  40471
+github.tar,                         level 16,                           advanced one pass,                  40466
 github.tar,                         level 16 with dict,                 advanced one pass,                  33374
 github.tar,                         level 16 with dict dms,             advanced one pass,                  33206
 github.tar,                         level 16 with dict dds,             advanced one pass,                  33206
 github.tar,                         level 16 with dict copy,            advanced one pass,                  33374
 github.tar,                         level 16 with dict load,            advanced one pass,                  39081
-github.tar,                         level 19,                           advanced one pass,                  32149
+github.tar,                         level 19,                           advanced one pass,                  32276
 github.tar,                         level 19 with dict,                 advanced one pass,                  32712
 github.tar,                         level 19 with dict dms,             advanced one pass,                  32555
 github.tar,                         level 19 with dict dds,             advanced one pass,                  32555
@@ -566,8 +566,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced one pass small out,        4505046
 silesia,                            level 12 row 2,                     advanced one pass small out,        4503116
 silesia,                            level 13,                           advanced one pass small out,        4493990
-silesia,                            level 16,                           advanced one pass small out,        4359864
-silesia,                            level 19,                           advanced one pass small out,        4296438
+silesia,                            level 16,                           advanced one pass small out,        4360041
+silesia,                            level 19,                           advanced one pass small out,        4296055
 silesia,                            no source size,                     advanced one pass small out,        4842075
 silesia,                            long distance mode,                 advanced one pass small out,        4833710
 silesia,                            multithreaded,                      advanced one pass small out,        4842075
@@ -600,14 +600,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced one pass small out,        4514049
 silesia.tar,                        level 12 row 2,                     advanced one pass small out,        4513797
 silesia.tar,                        level 13,                           advanced one pass small out,        4502956
-silesia.tar,                        level 16,                           advanced one pass small out,        4360527
-silesia.tar,                        level 19,                           advanced one pass small out,        4267021
+silesia.tar,                        level 16,                           advanced one pass small out,        4360546
+silesia.tar,                        level 19,                           advanced one pass small out,        4265911
 silesia.tar,                        no source size,                     advanced one pass small out,        4854086
 silesia.tar,                        long distance mode,                 advanced one pass small out,        4840452
 silesia.tar,                        multithreaded,                      advanced one pass small out,        4854160
 silesia.tar,                        multithreaded long distance mode,   advanced one pass small out,        4845741
 silesia.tar,                        small window log,                   advanced one pass small out,        7100655
-silesia.tar,                        small hash log,                     advanced one pass small out,        6529231
+silesia.tar,                        small hash log,                     advanced one pass small out,        6529206
 silesia.tar,                        small chain log,                    advanced one pass small out,        4917041
 silesia.tar,                        explicit params,                    advanced one pass small out,        4806855
 silesia.tar,                        uncompressed literals,              advanced one pass small out,        5122473
@@ -688,39 +688,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced one pass small out,        39393
 github,                             level 9 with dict copy,             advanced one pass small out,        39398
 github,                             level 9 with dict load,             advanced one pass small out,        41710
-github,                             level 11 row 1,                     advanced one pass small out,        135119
+github,                             level 11 row 1,                     advanced one pass small out,        135367
 github,                             level 11 row 1 with dict dms,       advanced one pass small out,        39671
 github,                             level 11 row 1 with dict dds,       advanced one pass small out,        39671
 github,                             level 11 row 1 with dict copy,      advanced one pass small out,        39651
 github,                             level 11 row 1 with dict load,      advanced one pass small out,        41360
-github,                             level 11 row 2,                     advanced one pass small out,        135119
+github,                             level 11 row 2,                     advanced one pass small out,        135367
 github,                             level 11 row 2 with dict dms,       advanced one pass small out,        39671
 github,                             level 11 row 2 with dict dds,       advanced one pass small out,        39671
 github,                             level 11 row 2 with dict copy,      advanced one pass small out,        39651
 github,                             level 11 row 2 with dict load,      advanced one pass small out,        41360
-github,                             level 12 row 1,                     advanced one pass small out,        134180
+github,                             level 12 row 1,                     advanced one pass small out,        134402
 github,                             level 12 row 1 with dict dms,       advanced one pass small out,        39677
 github,                             level 12 row 1 with dict dds,       advanced one pass small out,        39677
 github,                             level 12 row 1 with dict copy,      advanced one pass small out,        39677
 github,                             level 12 row 1 with dict load,      advanced one pass small out,        41166
-github,                             level 12 row 2,                     advanced one pass small out,        134180
+github,                             level 12 row 2,                     advanced one pass small out,        134402
 github,                             level 12 row 2 with dict dms,       advanced one pass small out,        39677
 github,                             level 12 row 2 with dict dds,       advanced one pass small out,        39677
 github,                             level 12 row 2 with dict copy,      advanced one pass small out,        39677
 github,                             level 12 row 2 with dict load,      advanced one pass small out,        41166
-github,                             level 13,                           advanced one pass small out,        132729
+github,                             level 13,                           advanced one pass small out,        132878
 github,                             level 13 with dict,                 advanced one pass small out,        39900
 github,                             level 13 with dict dms,             advanced one pass small out,        39900
 github,                             level 13 with dict dds,             advanced one pass small out,        39900
 github,                             level 13 with dict copy,            advanced one pass small out,        39948
 github,                             level 13 with dict load,            advanced one pass small out,        42624
-github,                             level 16,                           advanced one pass small out,        132729
+github,                             level 16,                           advanced one pass small out,        133209
 github,                             level 16 with dict,                 advanced one pass small out,        37577
 github,                             level 16 with dict dms,             advanced one pass small out,        37577
 github,                             level 16 with dict dds,             advanced one pass small out,        37577
 github,                             level 16 with dict copy,            advanced one pass small out,        37568
 github,                             level 16 with dict load,            advanced one pass small out,        42338
-github,                             level 19,                           advanced one pass small out,        132729
+github,                             level 19,                           advanced one pass small out,        132879
 github,                             level 19 with dict,                 advanced one pass small out,        37576
 github,                             level 19 with dict dms,             advanced one pass small out,        37576
 github,                             level 19 with dict dds,             advanced one pass small out,        37576
@@ -736,7 +736,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced one pass small out,        136341
 github,                             explicit params,                    advanced one pass small out,        137727
 github,                             uncompressed literals,              advanced one pass small out,        165911
-github,                             uncompressed literals optimal,      advanced one pass small out,        157227
+github,                             uncompressed literals optimal,      advanced one pass small out,        152667
 github,                             huffman literals,                   advanced one pass small out,        142365
 github,                             multithreaded with advanced params, advanced one pass small out,        165911
 github.tar,                         level -5,                           advanced one pass small out,        52115
@@ -839,13 +839,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced one pass small out,        37220
 github.tar,                         level 13 with dict copy,            advanced one pass small out,        37130
 github.tar,                         level 13 with dict load,            advanced one pass small out,        36010
-github.tar,                         level 16,                           advanced one pass small out,        40471
+github.tar,                         level 16,                           advanced one pass small out,        40466
 github.tar,                         level 16 with dict,                 advanced one pass small out,        33374
 github.tar,                         level 16 with dict dms,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict dds,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict copy,            advanced one pass small out,        33374
 github.tar,                         level 16 with dict load,            advanced one pass small out,        39081
-github.tar,                         level 19,                           advanced one pass small out,        32149
+github.tar,                         level 19,                           advanced one pass small out,        32276
 github.tar,                         level 19 with dict,                 advanced one pass small out,        32712
 github.tar,                         level 19 with dict dms,             advanced one pass small out,        32555
 github.tar,                         level 19 with dict dds,             advanced one pass small out,        32555
@@ -884,8 +884,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced streaming,                 4505046
 silesia,                            level 12 row 2,                     advanced streaming,                 4503116
 silesia,                            level 13,                           advanced streaming,                 4493990
-silesia,                            level 16,                           advanced streaming,                 4359864
-silesia,                            level 19,                           advanced streaming,                 4296438
+silesia,                            level 16,                           advanced streaming,                 4360041
+silesia,                            level 19,                           advanced streaming,                 4296055
 silesia,                            no source size,                     advanced streaming,                 4842039
 silesia,                            long distance mode,                 advanced streaming,                 4833710
 silesia,                            multithreaded,                      advanced streaming,                 4842075
@@ -918,14 +918,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced streaming,                 4514049
 silesia.tar,                        level 12 row 2,                     advanced streaming,                 4513797
 silesia.tar,                        level 13,                           advanced streaming,                 4502956
-silesia.tar,                        level 16,                           advanced streaming,                 4360527
-silesia.tar,                        level 19,                           advanced streaming,                 4267021
+silesia.tar,                        level 16,                           advanced streaming,                 4360546
+silesia.tar,                        level 19,                           advanced streaming,                 4265911
 silesia.tar,                        no source size,                     advanced streaming,                 4859267
 silesia.tar,                        long distance mode,                 advanced streaming,                 4840452
 silesia.tar,                        multithreaded,                      advanced streaming,                 4854160
 silesia.tar,                        multithreaded long distance mode,   advanced streaming,                 4845741
 silesia.tar,                        small window log,                   advanced streaming,                 7117559
-silesia.tar,                        small hash log,                     advanced streaming,                 6529234
+silesia.tar,                        small hash log,                     advanced streaming,                 6529209
 silesia.tar,                        small chain log,                    advanced streaming,                 4917021
 silesia.tar,                        explicit params,                    advanced streaming,                 4806873
 silesia.tar,                        uncompressed literals,              advanced streaming,                 5127423
@@ -1006,39 +1006,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced streaming,                 39393
 github,                             level 9 with dict copy,             advanced streaming,                 39398
 github,                             level 9 with dict load,             advanced streaming,                 41710
-github,                             level 11 row 1,                     advanced streaming,                 135119
+github,                             level 11 row 1,                     advanced streaming,                 135367
 github,                             level 11 row 1 with dict dms,       advanced streaming,                 39671
 github,                             level 11 row 1 with dict dds,       advanced streaming,                 39671
 github,                             level 11 row 1 with dict copy,      advanced streaming,                 39651
 github,                             level 11 row 1 with dict load,      advanced streaming,                 41360
-github,                             level 11 row 2,                     advanced streaming,                 135119
+github,                             level 11 row 2,                     advanced streaming,                 135367
 github,                             level 11 row 2 with dict dms,       advanced streaming,                 39671
 github,                             level 11 row 2 with dict dds,       advanced streaming,                 39671
 github,                             level 11 row 2 with dict copy,      advanced streaming,                 39651
 github,                             level 11 row 2 with dict load,      advanced streaming,                 41360
-github,                             level 12 row 1,                     advanced streaming,                 134180
+github,                             level 12 row 1,                     advanced streaming,                 134402
 github,                             level 12 row 1 with dict dms,       advanced streaming,                 39677
 github,                             level 12 row 1 with dict dds,       advanced streaming,                 39677
 github,                             level 12 row 1 with dict copy,      advanced streaming,                 39677
 github,                             level 12 row 1 with dict load,      advanced streaming,                 41166
-github,                             level 12 row 2,                     advanced streaming,                 134180
+github,                             level 12 row 2,                     advanced streaming,                 134402
 github,                             level 12 row 2 with dict dms,       advanced streaming,                 39677
 github,                             level 12 row 2 with dict dds,       advanced streaming,                 39677
 github,                             level 12 row 2 with dict copy,      advanced streaming,                 39677
 github,                             level 12 row 2 with dict load,      advanced streaming,                 41166
-github,                             level 13,                           advanced streaming,                 132729
+github,                             level 13,                           advanced streaming,                 132878
 github,                             level 13 with dict,                 advanced streaming,                 39900
 github,                             level 13 with dict dms,             advanced streaming,                 39900
 github,                             level 13 with dict dds,             advanced streaming,                 39900
 github,                             level 13 with dict copy,            advanced streaming,                 39948
 github,                             level 13 with dict load,            advanced streaming,                 42624
-github,                             level 16,                           advanced streaming,                 132729
+github,                             level 16,                           advanced streaming,                 133209
 github,                             level 16 with dict,                 advanced streaming,                 37577
 github,                             level 16 with dict dms,             advanced streaming,                 37577
 github,                             level 16 with dict dds,             advanced streaming,                 37577
 github,                             level 16 with dict copy,            advanced streaming,                 37568
 github,                             level 16 with dict load,            advanced streaming,                 42338
-github,                             level 19,                           advanced streaming,                 132729
+github,                             level 19,                           advanced streaming,                 132879
 github,                             level 19 with dict,                 advanced streaming,                 37576
 github,                             level 19 with dict dms,             advanced streaming,                 37576
 github,                             level 19 with dict dds,             advanced streaming,                 37576
@@ -1054,7 +1054,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced streaming,                 136341
 github,                             explicit params,                    advanced streaming,                 137727
 github,                             uncompressed literals,              advanced streaming,                 165911
-github,                             uncompressed literals optimal,      advanced streaming,                 157227
+github,                             uncompressed literals optimal,      advanced streaming,                 152667
 github,                             huffman literals,                   advanced streaming,                 142365
 github,                             multithreaded with advanced params, advanced streaming,                 165911
 github.tar,                         level -5,                           advanced streaming,                 52152
@@ -1157,13 +1157,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced streaming,                 37220
 github.tar,                         level 13 with dict copy,            advanced streaming,                 37130
 github.tar,                         level 13 with dict load,            advanced streaming,                 36010
-github.tar,                         level 16,                           advanced streaming,                 40471
+github.tar,                         level 16,                           advanced streaming,                 40466
 github.tar,                         level 16 with dict,                 advanced streaming,                 33374
 github.tar,                         level 16 with dict dms,             advanced streaming,                 33206
 github.tar,                         level 16 with dict dds,             advanced streaming,                 33206
 github.tar,                         level 16 with dict copy,            advanced streaming,                 33374
 github.tar,                         level 16 with dict load,            advanced streaming,                 39081
-github.tar,                         level 19,                           advanced streaming,                 32149
+github.tar,                         level 19,                           advanced streaming,                 32276
 github.tar,                         level 19 with dict,                 advanced streaming,                 32712
 github.tar,                         level 19 with dict dms,             advanced streaming,                 32555
 github.tar,                         level 19 with dict dds,             advanced streaming,                 32555
@@ -1194,11 +1194,11 @@ silesia,                            level 6,                            old stre
 silesia,                            level 7,                            old streaming,                      4566984
 silesia,                            level 9,                            old streaming,                      4543018
 silesia,                            level 13,                           old streaming,                      4493990
-silesia,                            level 16,                           old streaming,                      4359864
-silesia,                            level 19,                           old streaming,                      4296438
+silesia,                            level 16,                           old streaming,                      4360041
+silesia,                            level 19,                           old streaming,                      4296055
 silesia,                            no source size,                     old streaming,                      4842039
 silesia,                            uncompressed literals,              old streaming,                      4842075
-silesia,                            uncompressed literals optimal,      old streaming,                      4296438
+silesia,                            uncompressed literals optimal,      old streaming,                      4296055
 silesia,                            huffman literals,                   old streaming,                      6172207
 silesia.tar,                        level -5,                           old streaming,                      6856523
 silesia.tar,                        level -3,                           old streaming,                      6505954
@@ -1212,11 +1212,11 @@ silesia.tar,                        level 6,                            old stre
 silesia.tar,                        level 7,                            old streaming,                      4576664
 silesia.tar,                        level 9,                            old streaming,                      4552900
 silesia.tar,                        level 13,                           old streaming,                      4502956
-silesia.tar,                        level 16,                           old streaming,                      4360527
-silesia.tar,                        level 19,                           old streaming,                      4267021
+silesia.tar,                        level 16,                           old streaming,                      4360546
+silesia.tar,                        level 19,                           old streaming,                      4265911
 silesia.tar,                        no source size,                     old streaming,                      4859267
 silesia.tar,                        uncompressed literals,              old streaming,                      4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming,                      4267021
+silesia.tar,                        uncompressed literals optimal,      old streaming,                      4265911
 silesia.tar,                        huffman literals,                   old streaming,                      6179056
 github,                             level -5,                           old streaming,                      204407
 github,                             level -5 with dict,                 old streaming,                      46718
@@ -1240,16 +1240,16 @@ github,                             level 7,                            old stre
 github,                             level 7 with dict,                  old streaming,                      38758
 github,                             level 9,                            old streaming,                      135122
 github,                             level 9 with dict,                  old streaming,                      39437
-github,                             level 13,                           old streaming,                      132729
+github,                             level 13,                           old streaming,                      132878
 github,                             level 13 with dict,                 old streaming,                      39900
-github,                             level 16,                           old streaming,                      132729
+github,                             level 16,                           old streaming,                      133209
 github,                             level 16 with dict,                 old streaming,                      37577
-github,                             level 19,                           old streaming,                      132729
+github,                             level 19,                           old streaming,                      132879
 github,                             level 19 with dict,                 old streaming,                      37576
 github,                             no source size,                     old streaming,                      140599
 github,                             no source size with dict,           old streaming,                      40654
 github,                             uncompressed literals,              old streaming,                      136332
-github,                             uncompressed literals optimal,      old streaming,                      132729
+github,                             uncompressed literals optimal,      old streaming,                      132879
 github,                             huffman literals,                   old streaming,                      175468
 github.tar,                         level -5,                           old streaming,                      52152
 github.tar,                         level -5 with dict,                 old streaming,                      51045
@@ -1275,14 +1275,14 @@ github.tar,                         level 9,                            old stre
 github.tar,                         level 9 with dict,                  old streaming,                      36484
 github.tar,                         level 13,                           old streaming,                      35501
 github.tar,                         level 13 with dict,                 old streaming,                      37130
-github.tar,                         level 16,                           old streaming,                      40471
+github.tar,                         level 16,                           old streaming,                      40466
 github.tar,                         level 16 with dict,                 old streaming,                      33374
-github.tar,                         level 19,                           old streaming,                      32149
+github.tar,                         level 19,                           old streaming,                      32276
 github.tar,                         level 19 with dict,                 old streaming,                      32712
 github.tar,                         no source size,                     old streaming,                      38828
 github.tar,                         no source size with dict,           old streaming,                      38000
 github.tar,                         uncompressed literals,              old streaming,                      38831
-github.tar,                         uncompressed literals optimal,      old streaming,                      32149
+github.tar,                         uncompressed literals optimal,      old streaming,                      32276
 github.tar,                         huffman literals,                   old streaming,                      42560
 silesia,                            level -5,                           old streaming advanced,             6854744
 silesia,                            level -3,                           old streaming advanced,             6503319
@@ -1296,8 +1296,8 @@ silesia,                            level 6,                            old stre
 silesia,                            level 7,                            old streaming advanced,             4566984
 silesia,                            level 9,                            old streaming advanced,             4543018
 silesia,                            level 13,                           old streaming advanced,             4493990
-silesia,                            level 16,                           old streaming advanced,             4359864
-silesia,                            level 19,                           old streaming advanced,             4296438
+silesia,                            level 16,                           old streaming advanced,             4360041
+silesia,                            level 19,                           old streaming advanced,             4296055
 silesia,                            no source size,                     old streaming advanced,             4842039
 silesia,                            long distance mode,                 old streaming advanced,             4842075
 silesia,                            multithreaded,                      old streaming advanced,             4842075
@@ -1307,7 +1307,7 @@ silesia,                            small hash log,                     old stre
 silesia,                            small chain log,                    old streaming advanced,             4912197
 silesia,                            explicit params,                    old streaming advanced,             4795452
 silesia,                            uncompressed literals,              old streaming advanced,             4842075
-silesia,                            uncompressed literals optimal,      old streaming advanced,             4296438
+silesia,                            uncompressed literals optimal,      old streaming advanced,             4296055
 silesia,                            huffman literals,                   old streaming advanced,             6172207
 silesia,                            multithreaded with advanced params, old streaming advanced,             4842075
 silesia.tar,                        level -5,                           old streaming advanced,             6856523
@@ -1322,18 +1322,18 @@ silesia.tar,                        level 6,                            old stre
 silesia.tar,                        level 7,                            old streaming advanced,             4576664
 silesia.tar,                        level 9,                            old streaming advanced,             4552900
 silesia.tar,                        level 13,                           old streaming advanced,             4502956
-silesia.tar,                        level 16,                           old streaming advanced,             4360527
-silesia.tar,                        level 19,                           old streaming advanced,             4267021
+silesia.tar,                        level 16,                           old streaming advanced,             4360546
+silesia.tar,                        level 19,                           old streaming advanced,             4265911
 silesia.tar,                        no source size,                     old streaming advanced,             4859267
 silesia.tar,                        long distance mode,                 old streaming advanced,             4859271
 silesia.tar,                        multithreaded,                      old streaming advanced,             4859271
 silesia.tar,                        multithreaded long distance mode,   old streaming advanced,             4859271
 silesia.tar,                        small window log,                   old streaming advanced,             7117562
-silesia.tar,                        small hash log,                     old streaming advanced,             6529234
+silesia.tar,                        small hash log,                     old streaming advanced,             6529209
 silesia.tar,                        small chain log,                    old streaming advanced,             4917021
 silesia.tar,                        explicit params,                    old streaming advanced,             4806873
 silesia.tar,                        uncompressed literals,              old streaming advanced,             4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4267021
+silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4265911
 silesia.tar,                        huffman literals,                   old streaming advanced,             6179056
 silesia.tar,                        multithreaded with advanced params, old streaming advanced,             4859271
 github,                             level -5,                           old streaming advanced,             213265
@@ -1360,9 +1360,9 @@ github,                             level 9,                            old stre
 github,                             level 9 with dict,                  old streaming advanced,             38981
 github,                             level 13,                           old streaming advanced,             138676
 github,                             level 13 with dict,                 old streaming advanced,             39725
-github,                             level 16,                           old streaming advanced,             138676
+github,                             level 16,                           old streaming advanced,             138575
 github,                             level 16 with dict,                 old streaming advanced,             40789
-github,                             level 19,                           old streaming advanced,             132729
+github,                             level 19,                           old streaming advanced,             132879
 github,                             level 19 with dict,                 old streaming advanced,             37576
 github,                             no source size,                     old streaming advanced,             140599
 github,                             no source size with dict,           old streaming advanced,             40608
@@ -1374,7 +1374,7 @@ github,                             small hash log,                     old stre
 github,                             small chain log,                    old streaming advanced,             139275
 github,                             explicit params,                    old streaming advanced,             140937
 github,                             uncompressed literals,              old streaming advanced,             141104
-github,                             uncompressed literals optimal,      old streaming advanced,             132729
+github,                             uncompressed literals optimal,      old streaming advanced,             132879
 github,                             huffman literals,                   old streaming advanced,             181107
 github,                             multithreaded with advanced params, old streaming advanced,             141104
 github.tar,                         level -5,                           old streaming advanced,             52152
@@ -1401,9 +1401,9 @@ github.tar,                         level 9,                            old stre
 github.tar,                         level 9 with dict,                  old streaming advanced,             36312
 github.tar,                         level 13,                           old streaming advanced,             35501
 github.tar,                         level 13 with dict,                 old streaming advanced,             35807
-github.tar,                         level 16,                           old streaming advanced,             40471
+github.tar,                         level 16,                           old streaming advanced,             40466
 github.tar,                         level 16 with dict,                 old streaming advanced,             38578
-github.tar,                         level 19,                           old streaming advanced,             32149
+github.tar,                         level 19,                           old streaming advanced,             32276
 github.tar,                         level 19 with dict,                 old streaming advanced,             32704
 github.tar,                         no source size,                     old streaming advanced,             38828
 github.tar,                         no source size with dict,           old streaming advanced,             38015
@@ -1415,7 +1415,7 @@ github.tar,                         small hash log,                     old stre
 github.tar,                         small chain log,                    old streaming advanced,             41669
 github.tar,                         explicit params,                    old streaming advanced,             41385
 github.tar,                         uncompressed literals,              old streaming advanced,             38831
-github.tar,                         uncompressed literals optimal,      old streaming advanced,             32149
+github.tar,                         uncompressed literals optimal,      old streaming advanced,             32276
 github.tar,                         huffman literals,                   old streaming advanced,             42560
 github.tar,                         multithreaded with advanced params, old streaming advanced,             38831
 github,                             level -5 with dict,                 old streaming cdict,                46718

From c79fb4d78d986f5e85cd79370ecf1cccec473261 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 21 Dec 2022 16:21:29 -0800
Subject: [PATCH 094/197] update levels.sh test

comparing level 19 to level 22 and expecting a stricter better result from level 22
is not that guaranteed,
because level 19 and 22 are very close to each other,
especially for small files,
so any noise in the final compression result
result in failing this test.

Level 22 could be compared to something much lower, like level 15,
But level 19 is required anyway, because there is a clamping test which depends on it.

Removed level 22, kept level 19
---
 tests/cli-tests/compression/levels.sh              | 4 +---
 tests/cli-tests/compression/levels.sh.stderr.exact | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh
index 4837790ce..47ac2029f 100755
--- a/tests/cli-tests/compression/levels.sh
+++ b/tests/cli-tests/compression/levels.sh
@@ -10,11 +10,9 @@ zstd --fast=10 file -o file-f10.zst
 zstd --fast=1 file -o file-f1.zst
 zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
-zstd -22 --ultra file -o file-22.zst
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
 
-cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst
diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact
index cb00433e6..5664f854b 100644
--- a/tests/cli-tests/compression/levels.sh.stderr.exact
+++ b/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -6,11 +6,9 @@ zstd --fast=10 file -o file-f10.zst
 zstd --fast=1 file -o file-f1.zst
 zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
-zstd -22 --ultra file -o file-22.zst
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
 
-cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst

From 908e8127335ccc6acf23fbaec601aecfba30e96f Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Wed, 4 Jan 2023 13:01:54 -0800
Subject: [PATCH 095/197] initial commit

---
 .github/workflows/dev-long-tests.yml          |  46 +-
 .github/workflows/dev-short-tests.yml         | 146 +++-
 .../workflows/publish-release-artifacts.yml   |   2 +-
 .github/workflows/scorecards.yml              |   8 +-
 LICENSE                                       |   8 +-
 Makefile                                      |   4 +-
 build/VS2010/libzstd-dll/libzstd-dll.rc       |   4 +-
 build/VS2010/zstd/zstd.rc                     |   4 +-
 build/cmake/CMakeLists.txt                    |  22 +-
 .../AddZstdCompilationFlags.cmake             |  42 +-
 build/cmake/contrib/CMakeLists.txt            |   2 +-
 build/cmake/contrib/gen_html/CMakeLists.txt   |   2 +-
 build/cmake/contrib/pzstd/CMakeLists.txt      |   2 +-
 build/cmake/lib/CMakeLists.txt                |   8 +-
 build/cmake/programs/CMakeLists.txt           |   2 +-
 build/cmake/tests/CMakeLists.txt              |   9 +-
 build/meson/tests/meson.build                 |  55 +-
 build/single_file_libs/zstd-in.c              |   2 +-
 build/single_file_libs/zstddeclib-in.c        |   2 +-
 contrib/diagnose_corruption/Makefile          |   2 +-
 .../diagnose_corruption/check_flipped_bits.c  |   2 +-
 contrib/externalMatchfinder/.gitignore        |   2 +
 contrib/externalMatchfinder/Makefile          |  40 +
 contrib/externalMatchfinder/README.md         |  14 +
 contrib/externalMatchfinder/main.c            | 107 +++
 contrib/externalMatchfinder/matchfinder.c     |  80 ++
 contrib/externalMatchfinder/matchfinder.h     |  26 +
 contrib/freestanding_lib/freestanding.py      |   2 +-
 contrib/gen_html/Makefile                     |   2 +-
 contrib/gen_html/gen_html.cpp                 |   2 +-
 contrib/largeNbDicts/Makefile                 |   2 +-
 contrib/largeNbDicts/largeNbDicts.c           |   6 +-
 contrib/linux-kernel/Makefile                 |   6 +-
 contrib/linux-kernel/decompress_sources.h     |   2 +-
 contrib/linux-kernel/linux.mk                 |   2 +-
 contrib/linux-kernel/linux_zstd.h             |   2 +-
 contrib/linux-kernel/mem.h                    |   2 +-
 contrib/linux-kernel/test/Makefile            |   2 +-
 .../test/include/linux/compiler.h             |   2 +-
 .../linux-kernel/test/include/linux/errno.h   |   2 +-
 .../linux-kernel/test/include/linux/kernel.h  |   2 +-
 .../linux-kernel/test/include/linux/limits.h  |   2 +-
 .../linux-kernel/test/include/linux/math64.h  |   2 +-
 .../linux-kernel/test/include/linux/module.h  |   2 +-
 .../linux-kernel/test/include/linux/printk.h  |   2 +-
 .../linux-kernel/test/include/linux/stddef.h  |   2 +-
 .../linux-kernel/test/include/linux/swab.h    |   2 +-
 .../linux-kernel/test/include/linux/types.h   |   2 +-
 contrib/linux-kernel/test/static_test.c       |   2 +-
 contrib/linux-kernel/test/test.c              |   2 +-
 contrib/linux-kernel/zstd_common_module.c     |   2 +-
 contrib/linux-kernel/zstd_compress_module.c   |   2 +-
 contrib/linux-kernel/zstd_decompress_module.c |   2 +-
 contrib/linux-kernel/zstd_deps.h              |   2 +-
 contrib/match_finders/zstd_edist.c            |   2 +-
 contrib/match_finders/zstd_edist.h            |   2 +-
 contrib/pzstd/ErrorHolder.h                   |   2 +-
 contrib/pzstd/Logging.h                       |   2 +-
 contrib/pzstd/Makefile                        |   2 +-
 contrib/pzstd/Options.cpp                     |   2 +-
 contrib/pzstd/Options.h                       |   2 +-
 contrib/pzstd/Pzstd.cpp                       |  11 +-
 contrib/pzstd/Pzstd.h                         |   2 +-
 contrib/pzstd/SkippableFrame.cpp              |   2 +-
 contrib/pzstd/SkippableFrame.h                |   2 +-
 contrib/pzstd/main.cpp                        |   2 +-
 contrib/pzstd/test/OptionsTest.cpp            |   2 +-
 contrib/pzstd/test/PzstdTest.cpp              |   2 +-
 contrib/pzstd/test/RoundTrip.h                |   2 +-
 contrib/pzstd/test/RoundTripTest.cpp          |   2 +-
 contrib/pzstd/utils/Buffer.h                  |   2 +-
 contrib/pzstd/utils/FileSystem.h              |   8 +-
 contrib/pzstd/utils/Likely.h                  |   2 +-
 contrib/pzstd/utils/Portability.h             |  16 +
 contrib/pzstd/utils/Range.h                   |   4 +-
 contrib/pzstd/utils/ResourcePool.h            |   2 +-
 contrib/pzstd/utils/ScopeGuard.h              |   2 +-
 contrib/pzstd/utils/ThreadPool.h              |   2 +-
 contrib/pzstd/utils/WorkQueue.h               |   2 +-
 contrib/pzstd/utils/test/BufferTest.cpp       |   2 +-
 contrib/pzstd/utils/test/RangeTest.cpp        |   2 +-
 contrib/pzstd/utils/test/ResourcePoolTest.cpp |   2 +-
 contrib/pzstd/utils/test/ScopeGuardTest.cpp   |   2 +-
 contrib/pzstd/utils/test/ThreadPoolTest.cpp   |   2 +-
 contrib/pzstd/utils/test/WorkQueueTest.cpp    |   2 +-
 contrib/recovery/Makefile                     |   2 +-
 contrib/recovery/recover_directory.c          |   2 +-
 contrib/seekable_format/examples/Makefile     |   2 +-
 .../examples/parallel_compression.c           |   2 +-
 .../examples/parallel_processing.c            |   2 +-
 .../examples/seekable_compression.c           |   2 +-
 .../examples/seekable_decompression.c         |   2 +-
 .../examples/seekable_decompression_mem.c     |   2 +-
 contrib/seekable_format/tests/Makefile        |   2 +-
 .../zstd_seekable_compression_format.md       |   2 +-
 contrib/seekable_format/zstdseek_compress.c   |   2 +-
 contrib/seekable_format/zstdseek_decompress.c |   2 +-
 contrib/seqBench/Makefile                     |   2 +-
 doc/educational_decoder/Makefile              |   2 +-
 doc/educational_decoder/harness.c             |   2 +-
 doc/educational_decoder/zstd_decompress.c     |   2 +-
 doc/educational_decoder/zstd_decompress.h     |   2 +-
 doc/zstd_compression_format.md                |  26 +-
 examples/Makefile                             |   2 +-
 examples/common.h                             |   2 +-
 examples/dictionary_compression.c             |   2 +-
 examples/dictionary_decompression.c           |   2 +-
 examples/multiple_simple_compression.c        |   2 +-
 examples/multiple_streaming_compression.c     |   2 +-
 examples/simple_compression.c                 |   2 +-
 examples/simple_decompression.c               |   2 +-
 examples/streaming_compression.c              |   2 +-
 examples/streaming_compression_thread_pool.c  |   2 +-
 examples/streaming_decompression.c            |   2 +-
 examples/streaming_memory_usage.c             |   2 +-
 lib/Makefile                                  |   2 +-
 lib/README.md                                 |   7 +
 lib/common/bits.h                             |   2 +-
 lib/common/bitstream.h                        |   2 +-
 lib/common/compiler.h                         |   2 +-
 lib/common/cpu.h                              |   2 +-
 lib/common/debug.c                            |   2 +-
 lib/common/debug.h                            |   2 +-
 lib/common/entropy_common.c                   |   2 +-
 lib/common/error_private.c                    |   5 +-
 lib/common/error_private.h                    |   2 +-
 lib/common/fse.h                              |   2 +-
 lib/common/fse_decompress.c                   |   2 +-
 lib/common/huf.h                              |   8 +-
 lib/common/mem.h                              |   2 +-
 lib/common/pool.c                             |   4 +-
 lib/common/pool.h                             |   2 +-
 lib/common/portability_macros.h               |   2 +-
 lib/common/threading.c                        |  79 +-
 lib/common/threading.h                        |  12 +-
 lib/common/xxhash.c                           |   2 +-
 lib/common/xxhash.h                           |   2 +-
 lib/common/zstd_common.c                      |   2 +-
 lib/common/zstd_deps.h                        |   2 +-
 lib/common/zstd_internal.h                    |   3 +-
 lib/common/zstd_trace.h                       |   2 +-
 lib/compress/clevels.h                        |   2 +-
 lib/compress/fse_compress.c                   |   2 +-
 lib/compress/hist.c                           |   2 +-
 lib/compress/hist.h                           |   2 +-
 lib/compress/huf_compress.c                   |  54 +-
 lib/compress/zstd_compress.c                  | 716 ++++++++++++------
 lib/compress/zstd_compress_internal.h         |  71 +-
 lib/compress/zstd_compress_literals.c         |  62 +-
 lib/compress/zstd_compress_literals.h         |  20 +-
 lib/compress/zstd_compress_sequences.c        |   2 +-
 lib/compress/zstd_compress_sequences.h        |   2 +-
 lib/compress/zstd_compress_superblock.c       |   2 +-
 lib/compress/zstd_compress_superblock.h       |   2 +-
 lib/compress/zstd_cwksp.h                     |   2 +-
 lib/compress/zstd_double_fast.c               |   2 +-
 lib/compress/zstd_double_fast.h               |   2 +-
 lib/compress/zstd_fast.c                      |   2 +-
 lib/compress/zstd_fast.h                      |   2 +-
 lib/compress/zstd_lazy.c                      |   2 +-
 lib/compress/zstd_lazy.h                      |   2 +-
 lib/compress/zstd_ldm.c                       |   2 +-
 lib/compress/zstd_ldm.h                       |   2 +-
 lib/compress/zstd_ldm_geartab.h               |   2 +-
 lib/compress/zstd_opt.c                       |  90 ++-
 lib/compress/zstd_opt.h                       |   2 +-
 lib/compress/zstdmt_compress.c                |   2 +-
 lib/compress/zstdmt_compress.h                |   2 +-
 lib/decompress/huf_decompress.c               |  42 +-
 lib/decompress/huf_decompress_amd64.S         |   2 +-
 lib/decompress/zstd_ddict.c                   |   2 +-
 lib/decompress/zstd_ddict.h                   |   2 +-
 lib/decompress/zstd_decompress.c              |   4 +-
 lib/decompress/zstd_decompress_block.c        |  22 +-
 lib/decompress/zstd_decompress_block.h        |   2 +-
 lib/decompress/zstd_decompress_internal.h     |   2 +-
 lib/deprecated/zbuff.h                        |   2 +-
 lib/deprecated/zbuff_common.c                 |   2 +-
 lib/deprecated/zbuff_compress.c               |   2 +-
 lib/deprecated/zbuff_decompress.c             |   2 +-
 lib/dictBuilder/cover.c                       |   6 +-
 lib/dictBuilder/cover.h                       |   2 +-
 lib/dictBuilder/fastcover.c                   |   2 +-
 lib/dictBuilder/zdict.c                       |   6 +-
 lib/dll/example/Makefile                      |   2 +-
 lib/legacy/zstd_legacy.h                      |   2 +-
 lib/legacy/zstd_v01.c                         |   2 +-
 lib/legacy/zstd_v01.h                         |   2 +-
 lib/legacy/zstd_v02.c                         |   2 +-
 lib/legacy/zstd_v02.h                         |   2 +-
 lib/legacy/zstd_v03.c                         |   2 +-
 lib/legacy/zstd_v03.h                         |   2 +-
 lib/legacy/zstd_v04.c                         |   2 +-
 lib/legacy/zstd_v04.h                         |   2 +-
 lib/legacy/zstd_v05.c                         |   2 +-
 lib/legacy/zstd_v05.h                         |   2 +-
 lib/legacy/zstd_v06.c                         |   2 +-
 lib/legacy/zstd_v06.h                         |   2 +-
 lib/legacy/zstd_v07.c                         |   2 +-
 lib/legacy/zstd_v07.h                         |   2 +-
 lib/libzstd.mk                                |  42 +-
 lib/libzstd.pc.in                             |   2 +-
 lib/module.modulemap                          |  16 +-
 lib/zdict.h                                   |  80 +-
 lib/zstd.h                                    | 251 +++++-
 lib/zstd_errors.h                             |  31 +-
 programs/Makefile                             |   2 +-
 programs/benchfn.c                            |   8 +-
 programs/benchfn.h                            |   2 +-
 programs/benchzstd.c                          |   2 +-
 programs/benchzstd.h                          |   2 +-
 programs/datagen.c                            |   2 +-
 programs/datagen.h                            |   2 +-
 programs/dibio.c                              |   2 +-
 programs/dibio.h                              |   2 +-
 programs/fileio.c                             |  14 +-
 programs/fileio.h                             |   2 +-
 programs/fileio_asyncio.c                     |   2 +-
 programs/fileio_asyncio.h                     |   2 +-
 programs/fileio_common.h                      |   2 +-
 programs/fileio_types.h                       |   2 +-
 programs/platform.h                           |   2 +-
 programs/timefn.c                             |   2 +-
 programs/timefn.h                             |   2 +-
 programs/util.c                               | 167 +++-
 programs/util.h                               |   8 +-
 programs/windres/verrsrc.h                    |   2 +-
 programs/windres/zstd.rc                      |   4 +-
 programs/zstd.1                               | 115 ++-
 programs/zstd.1.md                            | 142 ++--
 programs/zstdcli.c                            | 199 ++---
 programs/zstdcli_trace.c                      |   2 +-
 programs/zstdcli_trace.h                      |   2 +-
 programs/zstdgrep.1.md                        |  10 +-
 programs/zstdless.1.md                        |   6 +-
 tests/DEPRECATED-test-zstd-speed.py           |   2 +-
 tests/Makefile                                |   4 +-
 tests/automated_benchmarking.py               |   2 +-
 tests/bigdict.c                               |   2 +-
 tests/checkTag.c                              |   2 +-
 tests/check_size.py                           |  31 +
 tests/cli-tests/README.md                     |  10 +
 tests/cli-tests/basic/help.sh.stdout.glob     |  41 +-
 tests/cli-tests/basic/version.sh.stdout.glob  |   4 +-
 tests/cli-tests/compression/levels.sh         |   4 +-
 .../compression/levels.sh.stderr.exact        |   4 +-
 .../file-stat/compress-file-to-file.sh        |   8 +
 .../compress-file-to-file.sh.stderr.exact     |  36 +
 .../file-stat/compress-file-to-stdout.sh      |   8 +
 .../compress-file-to-stdout.sh.stderr.exact   |  22 +
 .../file-stat/compress-stdin-to-file.sh       |   8 +
 .../compress-stdin-to-file.sh.stderr.exact    |  28 +
 .../file-stat/compress-stdin-to-stdout.sh     |   8 +
 .../compress-stdin-to-stdout.sh.stderr.exact  |  20 +
 .../file-stat/decompress-file-to-file.sh      |   7 +
 .../decompress-file-to-file.sh.stderr.exact   |  30 +
 .../file-stat/decompress-file-to-stdout.sh    |   7 +
 .../decompress-file-to-stdout.sh.stderr.exact |  14 +
 .../file-stat/decompress-stdin-to-file.sh     |   7 +
 .../decompress-stdin-to-file.sh.stderr.exact  |  20 +
 .../file-stat/decompress-stdin-to-stdout.sh   |   7 +
 ...decompress-stdin-to-stdout.sh.stderr.exact |  12 +
 .../progress/no-progress.sh.stderr.glob       |  20 +-
 tests/cli-tests/run.py                        |  20 +-
 tests/datagencli.c                            |   2 +-
 tests/decodecorpus.c                          |   8 +-
 tests/external_matchfinder.c                  | 117 +++
 tests/external_matchfinder.h                  |  35 +
 tests/fullbench.c                             |   2 +-
 tests/fuzz/Makefile                           |   2 +-
 tests/fuzz/block_decompress.c                 |   2 +-
 tests/fuzz/block_round_trip.c                 |   2 +-
 tests/fuzz/decompress_dstSize_tooSmall.c      |   2 +-
 tests/fuzz/dictionary_decompress.c            |   2 +-
 tests/fuzz/dictionary_loader.c                |   2 +-
 tests/fuzz/dictionary_round_trip.c            |   2 +-
 tests/fuzz/dictionary_stream_round_trip.c     |   2 +-
 tests/fuzz/fse_read_ncount.c                  |   2 +-
 tests/fuzz/fuzz.h                             |   2 +-
 tests/fuzz/fuzz.py                            |   2 +-
 tests/fuzz/fuzz_data_producer.c               |   2 +-
 tests/fuzz/fuzz_data_producer.h               |   2 +-
 tests/fuzz/fuzz_helpers.c                     |   2 +-
 tests/fuzz/fuzz_helpers.h                     |   2 +-
 tests/fuzz/huf_decompress.c                   |   2 +-
 tests/fuzz/huf_round_trip.c                   |   2 +-
 tests/fuzz/raw_dictionary_round_trip.c        |   2 +-
 tests/fuzz/regression_driver.c                |   2 +-
 tests/fuzz/seekable_roundtrip.c               |   2 +-
 tests/fuzz/sequence_compression_api.c         |   2 +-
 tests/fuzz/simple_compress.c                  |   2 +-
 tests/fuzz/simple_decompress.c                |   2 +-
 tests/fuzz/simple_round_trip.c                |   2 +-
 tests/fuzz/stream_decompress.c                |   2 +-
 tests/fuzz/stream_round_trip.c                |   2 +-
 tests/fuzz/zstd_frame_info.c                  |   2 +-
 tests/fuzz/zstd_helpers.c                     |   2 +-
 tests/fuzz/zstd_helpers.h                     |   2 +-
 tests/fuzzer.c                                | 270 ++++---
 tests/gzip/Makefile                           |   2 +-
 tests/invalidDictionaries.c                   |   2 +-
 tests/legacy.c                                |   2 +-
 tests/longmatch.c                             |  35 +-
 tests/paramgrill.c                            |  40 +-
 tests/poolTests.c                             |   2 +-
 tests/rateLimiter.py                          |   2 +-
 tests/regression/Makefile                     |   2 +-
 tests/regression/config.c                     |   2 +-
 tests/regression/config.h                     |   2 +-
 tests/regression/data.c                       |   2 +-
 tests/regression/data.h                       |   2 +-
 tests/regression/levels.h                     |   2 +-
 tests/regression/method.c                     |   2 +-
 tests/regression/method.h                     |   2 +-
 tests/regression/result.c                     |   2 +-
 tests/regression/result.h                     |   2 +-
 tests/regression/results.csv                  | 190 ++---
 tests/regression/test.c                       |   2 +-
 tests/roundTripCrash.c                        |   2 +-
 tests/seqgen.c                                |   2 +-
 tests/seqgen.h                                |   2 +-
 tests/test-license.py                         |   6 +-
 tests/test-zstd-versions.py                   | 104 +--
 tests/zstreamtest.c                           |  95 ++-
 zlibWrapper/examples/zwrapbench.c             |  12 +-
 zlibWrapper/gzcompatibility.h                 |   2 +-
 zlibWrapper/zstd_zlibwrapper.c                |   2 +-
 zlibWrapper/zstd_zlibwrapper.h                |   2 +-
 328 files changed, 3340 insertions(+), 1426 deletions(-)
 create mode 100644 contrib/externalMatchfinder/.gitignore
 create mode 100644 contrib/externalMatchfinder/Makefile
 create mode 100644 contrib/externalMatchfinder/README.md
 create mode 100644 contrib/externalMatchfinder/main.c
 create mode 100644 contrib/externalMatchfinder/matchfinder.c
 create mode 100644 contrib/externalMatchfinder/matchfinder.h
 create mode 100644 contrib/pzstd/utils/Portability.h
 create mode 100755 tests/check_size.py
 create mode 100755 tests/cli-tests/file-stat/compress-file-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/compress-file-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/compress-stdin-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-file-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-file-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-stdin-to-file.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
 create mode 100755 tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
 create mode 100644 tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
 create mode 100644 tests/external_matchfinder.c
 create mode 100644 tests/external_matchfinder.h

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index ac34c7165..f75f61f8e 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -13,7 +13,7 @@ jobs:
   make-all:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make all
       run: make all
 
@@ -24,7 +24,7 @@ jobs:
       DEVNULLRIGHTS: 1
       READFROMBLOCKDEVICE: 1
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make test
       run: make test
 
@@ -32,28 +32,28 @@ jobs:
   make-test-osx:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: OS-X test
       run: make test # make -c lib all doesn't work because of the fact that it's not a tty
 
   no-intrinsics-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: no intrinsics fuzztest
       run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
 
   tsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
   ubsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: undefined behavior sanitizer zstreamtest
       run: CC=clang make uasan-test-zstream
 
@@ -61,7 +61,7 @@ jobs:
   tsan-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: thread sanitizer fuzztest
       run: CC=clang make tsan-fuzztest
 
@@ -69,7 +69,7 @@ jobs:
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: gcc-8 + ASan + UBSan + Test Zstd
       # See https://askubuntu.com/a/1428822
       run: |
@@ -81,14 +81,14 @@ jobs:
   clang-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Test Zstd
       run: CC=clang make -j uasan-test-zstd </dev/null V=1
 
   gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -102,7 +102,7 @@ jobs:
   gcc-8-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: gcc-8 + ASan + UBSan + Fuzz Test
       # See https://askubuntu.com/a/1428822
       run: |
@@ -114,14 +114,14 @@ jobs:
   clang-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Fuzz Test
       run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
@@ -131,7 +131,7 @@ jobs:
   clang-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
@@ -141,28 +141,28 @@ jobs:
   asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ASan + UBSan + Regression Test
       run: make -j uasanregressiontest
 
   clang-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + ASan + UBSan + Regression Test
       run: CC=clang make -j uasanregressiontest
 
   msan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: MSan + Regression Test
       run: make -j msanregressiontest
 
   clang-msan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + MSan + Fuzz Test
       run: |
         sudo apt-get -qqq update
@@ -173,7 +173,7 @@ jobs:
   clang-msan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: clang + MSan + Test Zstd
       run: |
         sudo apt-get update
@@ -183,7 +183,7 @@ jobs:
   armfuzz:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: Qemu ARM emulation + Fuzz Test
         run: |
           sudo apt-get -qqq update
@@ -193,7 +193,7 @@ jobs:
   valgrind-fuzz-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: valgrind + fuzz test stack mode    # ~ 7mn
       shell: 'script -q -e -c "bash {0}"'
       run: |
@@ -213,7 +213,7 @@ jobs:
           { compiler: gcc, platform: x64, action: test, script: ""},
         ]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Mingw long test
       run: |
         $env:PATH_ORIGINAL = $env:PATH
@@ -279,7 +279,7 @@ jobs:
         dry-run: false
         sanitizer: ${{ matrix.sanitizer }}
     - name: Upload Crash
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
       if: failure() && steps.build.outcome == 'success'
       with:
         name: ${{ matrix.sanitizer }}-artifacts
diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 08df1c07c..b9998e024 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -14,21 +14,21 @@ jobs:
   linux-kernel:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: linux kernel, library + build + test
       run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
 
   benchmarking:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make benchmarking
       run: make benchmarking
 
   check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make check on 32-bit
       run: |
         sudo apt update
@@ -38,7 +38,7 @@ jobs:
   check-x32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
       env:
         CHECK_CONSTRAINED_MEM: true
@@ -50,7 +50,7 @@ jobs:
   gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: gcc-7 + libzstdmt compilation
       # See https://askubuntu.com/a/1428822
       run: |
@@ -67,7 +67,7 @@ jobs:
   cmake-build-and-test-check:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: cmake build and test check
       run: |
         FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
@@ -78,7 +78,7 @@ jobs:
   cpp-gnu90-c99-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: C++, gnu90 and c99 compatibility
       run: |
         make cxxtest
@@ -92,7 +92,7 @@ jobs:
   mingw-cross-compilation:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: mingw cross-compilation
       run: |
         # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@@ -103,7 +103,7 @@ jobs:
   armbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: ARM Build Test
       run: |
         sudo apt-get -qqq update
@@ -113,7 +113,7 @@ jobs:
   bourne-shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Bourne shell compatibility (shellcheck)
       run: |
         wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
@@ -123,7 +123,7 @@ jobs:
   zlib-wrapper:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: zlib wrapper test
       run: |
         sudo apt-get -qqq update
@@ -134,7 +134,7 @@ jobs:
   lz4-threadpool-libs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: LZ4, thread pool, and libs build testslib wrapper test
       run: |
         make lz4install
@@ -148,7 +148,7 @@ jobs:
   gcc-make-tests-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Make all, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -158,7 +158,7 @@ jobs:
   gcc-8-make:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: gcc-8 build
         # See https://askubuntu.com/a/1428822
         run: |
@@ -170,7 +170,7 @@ jobs:
   implicit-fall-through:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: -Wimplicit-fallthrough build
         run: |
           make clean
@@ -178,6 +178,51 @@ jobs:
           make clean
           CC=clang MOREFLAGS="-Werror -Wimplicit-fallthrough -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
 
+  meson-linux:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - name: Install packages
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev
+          pip install --pre meson
+      - name: Build with Meson
+        run: |
+          meson setup \
+            --buildtype=debugoptimized \
+            -Db_lundef=false \
+            -Dauto_features=enabled \
+            -Dbin_programs=true \
+            -Dbin_tests=true \
+            -Dbin_contrib=true \
+            -Ddefault_library=both \
+            build/meson builddir
+          ninja -C builddir/
+          meson test -C builddir/ --print-errorlogs
+          meson install -C builddir --destdir staging/
+
+  meson-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - name: Install packages
+        run: pip install --pre meson
+      - name: Initialize the MSVC dev command prompt
+        uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d
+      - name: Configure with Meson
+        run: |
+          meson setup build/meson/ builddir -Dbin_tests=true -Dbin_programs=true -Dbin_contrib=true
+      - name: Build with Meson
+        run: |
+          ninja -C builddir/
+      - name: Test with Meson
+        run: |
+          meson test -C builddir/ --print-errorlogs
+      - name: Install with Meson
+        run: |
+          meson install -C builddir --destdir staging/
+
   cmake-visual-2019:
     runs-on: windows-2019
     strategy:
@@ -189,9 +234,9 @@ jobs:
             flags: "-A Win32"
           - generator: "MinGW Makefiles"
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
@@ -208,9 +253,9 @@ jobs:
         platform: [x64, Win32]
         configuration: [Debug, Release]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@@ -227,19 +272,33 @@ jobs:
 #        platform: [x64, Win32]
 #        configuration: [Debug, Release]
 #    steps:
-#    - uses: actions/checkout@v3
+#    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
 #    - name: Add MSBuild to PATH
-#      uses: microsoft/setup-msbuild@v1.1.3
+#      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
 #    - name: Build
 #      working-directory: ${{env.GITHUB_WORKSPACE}}
 #      run: >
 #        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140
 #        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
 
+  # This tests that we don't accidently grow the size too much.
+  # If the size grows intentionally, you can raise these numbers.
+  # But we do need to think about binary size, since it is a concern.
+  libzstd-size:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - name: libzstd size test
+      run: |
+        make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
+        make clean && make -j -C lib libzstd ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 && ./tests/check_size.py lib/libzstd.so 400000
+        make clean && make -j -C lib libzstd ZSTD_LIB_MINIFY=1 && ./tests/check_size.py lib/libzstd.so 300000
+        make clean && make -j -C lib libzstd ZSTD_LIB_MINIFY=1 ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 && ./tests/check_size.py lib/libzstd.so 80000
+
   minimal-decompressor-macros:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: minimal decompressor macros
       run: |
         make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -254,7 +313,7 @@ jobs:
   dynamic-bmi2:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: dynamic bmi2 tests
       run: |
         make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@@ -266,7 +325,7 @@ jobs:
   test-variants:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: make all variants & validate
       run: |
         make -j -C programs allVariants MOREFLAGS=-O0
@@ -292,7 +351,7 @@ jobs:
       XCC: ${{ matrix.xcc }}
       XEMU: ${{ matrix.xemu }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: apt update & install
       run: |
         sudo apt-get update
@@ -345,7 +404,7 @@ jobs:
           { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
         ]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Mingw short test
       run: |
         ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
@@ -378,9 +437,9 @@ jobs:
         platform: [x64, Win32]
         configuration: [Release]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build and run tests
       working-directory: ${{env.GITHUB_WORKSPACE}}
       env:
@@ -398,7 +457,7 @@ jobs:
   intel-cet-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
     - name: Build Zstd
       run: |
         make -j zstd V=1
@@ -419,7 +478,7 @@ jobs:
     container:
       image: debian:testing
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
       - name: Install dependencies
         run: |
           apt -y update
@@ -431,18 +490,21 @@ jobs:
           cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd)
           ./simple LICENSE
 
+  versions-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - name: Versions Compatibility Test
+      run: |
+        make -C tests versionsTest
 
-# This test currently fails on Github Actions specifically.
-# Possible reason : TTY emulation.
-# Note that the same test works fine locally and on travisCI.
-# This will have to be fixed before transferring the test to GA.
-#  versions-compatibility:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: actions/checkout@v3
-#    - name: Versions Compatibility Test
-#      run: |
-#        make -C tests versionsTest
+  clangbuild:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - name: make clangbuild
+      run: |
+        make clangbuild
 
 
 # For reference : icc tests
@@ -463,7 +525,7 @@ jobs:
 #        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
 #        sudo apt-get update
 #        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@v3
+#    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
 #    - name: make check
 #      run: |
 #        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml
index 4a70fb8c6..0903f0d3c 100644
--- a/.github/workflows/publish-release-artifacts.yml
+++ b/.github/workflows/publish-release-artifacts.yml
@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
 
       - name: Archive
         env:
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 1c19b48b2..afdf4d63f 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -27,12 +27,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.0.0
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6
+        uses: ossf/scorecard-action@937ffa90d79c7d720498178154ad4c7ba1e4ad8c # tag=v2.1.0
         with:
           results_file: results.sarif
           results_format: sarif
@@ -51,7 +51,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # tag=v3.0.0
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
         with:
           name: SARIF file
           path: results.sarif
@@ -59,6 +59,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@5f532563584d71fdef14ee64d17bafb34f751ce5 # tag=v1.0.26
+        uses: github/codeql-action/upload-sarif@959cbb7472c4d4ad70cdfe6f4976053fe48ab394 # tag=v2.1.37
         with:
           sarif_file: results.sarif
diff --git a/LICENSE b/LICENSE
index a793a8028..75800288c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -2,7 +2,7 @@ BSD License
 
 For Zstandard software
 
-Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
@@ -14,9 +14,9 @@ are permitted provided that the following conditions are met:
    this list of conditions and the following disclaimer in the documentation
    and/or other materials provided with the distribution.
 
- * Neither the name Facebook nor the names of its contributors may be used to
-   endorse or promote products derived from this software without specific
-   prior written permission.
+ * Neither the name Facebook, nor Meta, nor the names of its contributors may
+   be used to endorse or promote products derived from this software without
+   specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
diff --git a/Makefile b/Makefile
index 429c90ff1..d87fc76eb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -123,6 +123,7 @@ contrib: lib
 	$(MAKE) -C contrib/seekable_format/examples all
 	$(MAKE) -C contrib/seekable_format/tests test
 	$(MAKE) -C contrib/largeNbDicts all
+	$(MAKE) -C contrib/externalMatchfinder all
 	cd build/single_file_libs/ ; ./build_decoder_test.sh
 	cd build/single_file_libs/ ; ./build_library_test.sh
 
@@ -142,6 +143,7 @@ clean:
 	$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
 	$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
 	$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/externalMatchfinder $@ > $(VOID)
 	$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	$(Q)$(RM) -r lz4
 	@echo Cleaning completed
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.rc b/build/VS2010/libzstd-dll/libzstd-dll.rc
index ee9f56280..13e8746ff 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.rc
+++ b/build/VS2010/libzstd-dll/libzstd-dll.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "CompanyName", "Meta Platforms, Inc."
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "libzstd.dll"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+            VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "libzstd.dll"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/build/VS2010/zstd/zstd.rc b/build/VS2010/zstd/zstd.rc
index f5e404730..a2118c2df 100644
--- a/build/VS2010/zstd/zstd.rc
+++ b/build/VS2010/zstd/zstd.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "CompanyName", "Meta Platforms, Inc."
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+            VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index f1eac9ada..0bffc87d9 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -8,18 +8,18 @@
 # ################################################################
 
 cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
-  
-# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. 
-# Set and use the newest cmake policies that are validated to work 
-set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") 
+
+# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
+# Set and use the newest cmake policies that are validated to work
+set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
 set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
 if("${CMAKE_MAJOR_VERSION}" LESS 3)
-  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
-elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND 
+  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
+elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
         "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
-    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
-else() 
-    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") 
+    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
+else()
+    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
 endif()
 cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
 
@@ -40,11 +40,13 @@ if( CMAKE_MAJOR_VERSION LESS 3 )
   set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
   set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
   enable_language(C)   # Main library is in C
+  enable_language(ASM) # And ASM
   enable_language(CXX) # Testing contributed code also utilizes CXX
 else()
   project(zstd
     VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
     LANGUAGES C   # Main library is in C
+              ASM # And ASM
               CXX # Testing contributed code also utilizes CXX
     )
 endif()
diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
index 8d04458c3..0265349fb 100644
--- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
@@ -1,7 +1,8 @@
 include(CheckCXXCompilerFlag)
 include(CheckCCompilerFlag)
+include(CheckLinkerFlag)
 
-function(EnableCompilerFlag _flag _C _CXX)
+function(EnableCompilerFlag _flag _C _CXX _LD)
     string(REGEX REPLACE "\\+" "PLUS" varname "${_flag}")
     string(REGEX REPLACE "[^A-Za-z0-9]+" "_" varname "${varname}")
     string(REGEX REPLACE "^_+" "" varname "${varname}")
@@ -18,6 +19,13 @@ function(EnableCompilerFlag _flag _C _CXX)
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
+    if (_LD)
+        CHECK_LINKER_FLAG(C ${_flag} LD_FLAG_${varname})
+        if (LD_FLAG_${varname})
+            set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
+            set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
+        endif ()
+    endif ()
 endfunction()
 
 macro(ADD_ZSTD_COMPILATION_FLAGS)
@@ -30,33 +38,39 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
         # EnableCompilerFlag("-std=c99" true false)   # Set C compiation to c99 standard
         if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
             # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" true true)
+            EnableCompilerFlag("/clang:-Wall" true true false)
         else ()
-            EnableCompilerFlag("-Wall" true true)
+            EnableCompilerFlag("-Wall" true true false)
         endif ()
-        EnableCompilerFlag("-Wextra" true true)
-        EnableCompilerFlag("-Wundef" true true)
-        EnableCompilerFlag("-Wshadow" true true)
-        EnableCompilerFlag("-Wcast-align" true true)
-        EnableCompilerFlag("-Wcast-qual" true true)
-        EnableCompilerFlag("-Wstrict-prototypes" true false)
+        EnableCompilerFlag("-Wextra" true true false)
+        EnableCompilerFlag("-Wundef" true true false)
+        EnableCompilerFlag("-Wshadow" true true false)
+        EnableCompilerFlag("-Wcast-align" true true false)
+        EnableCompilerFlag("-Wcast-qual" true true false)
+        EnableCompilerFlag("-Wstrict-prototypes" true false false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("-DDEBUGLEVEL=1" true true)
+            EnableCompilerFlag("-DDEBUGLEVEL=1" true true false)
         endif ()
+        # Add noexecstack flags
+        # LDFLAGS
+        EnableCompilerFlag("-z noexecstack" false false true)
+        # CFLAGS & CXXFLAGS
+        EnableCompilerFlag("-Qunused-arguments" true true false)
+        EnableCompilerFlag("-Wa,--noexecstack" true true false)
     elseif (MSVC) # Add specific compilation flags for Windows Visual
 
         set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
         if (CMAKE_GENERATOR MATCHES "Visual Studio" AND ACTIVATE_MULTITHREADED_COMPILATION)
-            EnableCompilerFlag("/MP" true true)
+            EnableCompilerFlag("/MP" true true false)
         endif ()
 
         # UNICODE SUPPORT
-        EnableCompilerFlag("/D_UNICODE" true true)
-        EnableCompilerFlag("/DUNICODE" true true)
+        EnableCompilerFlag("/D_UNICODE" true true false)
+        EnableCompilerFlag("/DUNICODE" true true false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("/DDEBUGLEVEL=1" true true)
+            EnableCompilerFlag("/DDEBUGLEVEL=1" true true false)
         endif ()
     endif ()
 
diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt
index f7631d08c..8df2a17b3 100644
--- a/build/cmake/contrib/CMakeLists.txt
+++ b/build/cmake/contrib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt
index 8fdd61131..d1ff6c64b 100644
--- a/build/cmake/contrib/gen_html/CMakeLists.txt
+++ b/build/cmake/contrib/gen_html/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt
index 27af86c88..f7098fa0f 100644
--- a/build/cmake/contrib/pzstd/CMakeLists.txt
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index cf7af0f8c..30349586b 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -77,6 +77,12 @@ if (MSVC)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
 endif ()
 
+# Explicitly set the language to C for all files, including ASM files.
+# Our assembly expects to be compiled by a C compiler, and is only enabled for
+# __GNUC__ compatible compilers. Otherwise all the ASM code is disabled by
+# macros.
+set_source_files_properties(${Sources} PROPERTIES LANGUAGE C)
+
 # Split project to static and shared libraries build
 set(library_targets)
 if (ZSTD_BUILD_SHARED)
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index 28b1e1d16..58d998e42 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 02444c803..250f0508f 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -1,6 +1,6 @@
 # ################################################################
 # zstd - Makefile
-# Copyright (C) Yann Collet 2014-present
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # BSD license
@@ -81,7 +81,7 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
 #
 # zstreamtest
 #
-add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
+add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c ${TESTS_DIR}/external_matchfinder.c)
 if (NOT MSVC)
     target_compile_options(zstreamtest PRIVATE "-Wno-deprecated-declarations")
 endif()
@@ -96,13 +96,14 @@ add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
 AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "$ENV{PLAYTESTS_FLAGS}"
     "Semicolon-separated list of flags to pass to the playTests.sh test")
 add_test(NAME playTests COMMAND sh -c "\"${TESTS_DIR}/playTests.sh\" ${ZSTD_PLAYTESTS_FLAGS}")
-if (ZSTD_BUILD_PROGRAMS)
+find_program(UNAME uname) # Run script only in unix shell environments
+if (ZSTD_BUILD_PROGRAMS AND UNAME)
     set_property(TEST playTests APPEND PROPERTY ENVIRONMENT
         "ZSTD_BIN=$<TARGET_FILE:zstd>"
         "DATAGEN_BIN=$<TARGET_FILE:datagen>"
         )
 else()
-    message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled")
+    message(STATUS "Disabling playTests.sh test because requirements not met")
     set_tests_properties(playTests PROPERTIES DISABLED YES)
 endif()
 
diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index 22f43209a..e70b73432 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -21,7 +21,6 @@ FUZZER_FLAGS = ['--no-big-tests']
 FUZZERTEST = '-T200s'
 ZSTREAM_TESTTIME = '-T90s'
 DECODECORPUS_TESTTIME = '-T30'
-ZSTDRTTEST = ['--test-large-data']
 
 # =============================================================================
 # Executables
@@ -66,8 +65,10 @@ fuzzer = executable('fuzzer',
   dependencies: [ testcommon_dep, thread_dep ],
   install: false)
 
-zstreamtest_sources = [join_paths(zstd_rootdir, 'tests/seqgen.c'),
-  join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
+zstreamtest_sources = [
+  join_paths(zstd_rootdir, 'tests/seqgen.c'),
+  join_paths(zstd_rootdir, 'tests/zstreamtest.c'),
+  join_paths(zstd_rootdir, 'tests/external_matchfinder.c')]
 zstreamtest = executable('zstreamtest',
   zstreamtest_sources,
   include_directories: test_includes,
@@ -134,24 +135,38 @@ checkTag = executable('checkTag',
 # =============================================================================
 
 if tests_supported_oses.contains(host_machine_os)
-  valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
+  valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: false)
   valgrindTest_py = files('valgrindTest.py')
-  test('valgrindTest',
-    valgrindTest_py,
-    args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
-    depends: [zstd, datagen, fuzzer, fullbench],
-    timeout: 600) # Timeout should work on HDD drive
+  if valgrind_prog.found()
+    test('valgrindTest',
+      valgrindTest_py,
+      args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
+      depends: [zstd, datagen, fuzzer, fullbench],
+      timeout: 600) # Timeout should work on HDD drive
+  endif
 endif
 
 if host_machine_os != os_windows
   playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
-  test('test-zstd',
-    playTests_sh,
-    args: ZSTDRTTEST,
-    env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
-    depends: [datagen],
-    workdir: meson.current_build_dir(),
-    timeout: 2800) # Timeout should work on HDD drive
+
+  # add slow tests only if the meson version is new enough to support
+  # test setups with default-excluded suites
+  if meson.version().version_compare('>=0.57.0')
+    matrix = {'fast': [], 'slow': ['--test-large-data']}
+  else
+    matrix = {'fast': []}
+  endif
+
+  foreach suite, opt: matrix
+    test('test-zstd-'+suite,
+      playTests_sh,
+      args: opt,
+      env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
+      depends: [datagen],
+      suite: suite,
+      workdir: meson.current_build_dir(),
+      timeout: 2800) # Timeout should work on HDD drive
+  endforeach
 endif
 
 test('test-fullbench-1',
@@ -190,3 +205,11 @@ test('test-decodecorpus',
   args: ['-t', DECODECORPUS_TESTTIME],
   timeout: 60)
 test('test-poolTests', poolTests) # should be fast
+
+if meson.version().version_compare('>=0.57.0')
+  add_test_setup('fast',
+    is_default: true,
+    exclude_suites: ['slow'])
+  add_test_setup('slow',
+    exclude_suites: ['fast'])
+endif
diff --git a/build/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c
index 59c6b5f19..e6fca9e4a 100644
--- a/build/single_file_libs/zstd-in.c
+++ b/build/single_file_libs/zstd-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/build/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c
index 5a58589c9..8d9c1f54b 100644
--- a/build/single_file_libs/zstddeclib-in.c
+++ b/build/single_file_libs/zstddeclib-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/Makefile b/contrib/diagnose_corruption/Makefile
index a21a00212..ecc9e6395 100644
--- a/contrib/diagnose_corruption/Makefile
+++ b/contrib/diagnose_corruption/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/check_flipped_bits.c b/contrib/diagnose_corruption/check_flipped_bits.c
index cc40ab84b..09ddd4674 100644
--- a/contrib/diagnose_corruption/check_flipped_bits.c
+++ b/contrib/diagnose_corruption/check_flipped_bits.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/externalMatchfinder/.gitignore b/contrib/externalMatchfinder/.gitignore
new file mode 100644
index 000000000..46357ef58
--- /dev/null
+++ b/contrib/externalMatchfinder/.gitignore
@@ -0,0 +1,2 @@
+# build artifacts
+externalMatchfinder
diff --git a/contrib/externalMatchfinder/Makefile b/contrib/externalMatchfinder/Makefile
new file mode 100644
index 000000000..2baa558cb
--- /dev/null
+++ b/contrib/externalMatchfinder/Makefile
@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) Yann Collet, Meta Platforms, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common
+
+CFLAGS  ?= -O3
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+default: externalMatchfinder
+
+all: externalMatchfinder
+
+externalMatchfinder: matchfinder.c main.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) externalMatchfinder
diff --git a/contrib/externalMatchfinder/README.md b/contrib/externalMatchfinder/README.md
new file mode 100644
index 000000000..cb7d49d97
--- /dev/null
+++ b/contrib/externalMatchfinder/README.md
@@ -0,0 +1,14 @@
+externalMatchfinder
+=====================
+
+`externalMatchfinder` is a test tool for the external matchfinder API.
+It demonstrates how to use the API to perform a simple round-trip test.
+
+A sample matchfinder is provided in matchfinder.c, but the user can swap
+this out with a different one if desired. The sample matchfinder implements
+LZ compression with a 1KB hashtable. Dictionary compression is not currently supported.
+
+Command line :
+```
+externalMatchfinder filename
+```
diff --git a/contrib/externalMatchfinder/main.c b/contrib/externalMatchfinder/main.c
new file mode 100644
index 000000000..6971a46c7
--- /dev/null
+++ b/contrib/externalMatchfinder/main.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#include "zstd_errors.h"
+#include "matchfinder.h" // simpleExternalMatchFinder
+
+#define CHECK(res)                                      \
+do {                                                    \
+    if (ZSTD_isError(res)) {                            \
+        printf("ERROR: %s\n", ZSTD_getErrorName(res));  \
+        return 1;                                       \
+    }                                                   \
+} while (0)                                             \
+
+int main(int argc, char *argv[]) {
+    if (argc != 2) {
+        printf("Usage: exampleMatchfinder <file>\n");
+        return 1;
+    }
+
+    ZSTD_CCtx* const zc = ZSTD_createCCtx();
+
+    int simpleExternalMatchState = 0xdeadbeef;
+
+    // Here is the crucial bit of code!
+    ZSTD_registerExternalMatchFinder(
+        zc,
+        &simpleExternalMatchState,
+        simpleExternalMatchFinder
+    );
+
+    {
+        size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 1);
+        CHECK(res);
+    }
+
+    FILE *f = fopen(argv[1], "rb");
+    assert(f);
+    {
+        int const ret = fseek(f, 0, SEEK_END);
+        assert(ret == 0);
+    }
+    size_t const srcSize = ftell(f);
+    {
+        int const ret = fseek(f, 0, SEEK_SET);
+        assert(ret == 0);
+    }
+
+    char* const src = malloc(srcSize + 1);
+    assert(src);
+    {
+        size_t const ret = fread(src, srcSize, 1, f);
+        assert(ret == 1);
+        int const ret2 = fclose(f);
+        assert(ret2 == 0);
+    }
+
+    size_t const dstSize = ZSTD_compressBound(srcSize);
+    char* const dst = malloc(dstSize);
+    assert(dst);
+
+    size_t const cSize = ZSTD_compress2(zc, dst, dstSize, src, srcSize);
+    CHECK(cSize);
+
+    char* const val = malloc(srcSize);
+    assert(val);
+
+    {
+        size_t const res = ZSTD_decompress(val, srcSize, dst, cSize);
+        CHECK(res);
+    }
+
+    if (memcmp(src, val, srcSize) == 0) {
+        printf("Compression and decompression were successful!\n");
+        printf("Original size: %lu\n", srcSize);
+        printf("Compressed size: %lu\n", cSize);
+    } else {
+        printf("ERROR: input and validation buffers don't match!\n");
+        for (size_t i = 0; i < srcSize; i++) {
+            if (src[i] != val[i]) {
+                printf("First bad index: %zu\n", i);
+                break;
+            }
+        }
+        return 1;
+    }
+
+    ZSTD_freeCCtx(zc);
+    free(src);
+    free(dst);
+    free(val);
+    return 0;
+}
diff --git a/contrib/externalMatchfinder/matchfinder.c b/contrib/externalMatchfinder/matchfinder.c
new file mode 100644
index 000000000..f119193ef
--- /dev/null
+++ b/contrib/externalMatchfinder/matchfinder.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "matchfinder.h"
+
+#define HSIZE 1024
+static U32 const HLOG = 10;
+static U32 const MLS = 4;
+static U32 const BADIDX = 0xffffffff;
+
+size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    size_t seqCount = 0;
+    U32 hashTable[HSIZE];
+
+    (void)externalMatchState;
+    (void)dict;
+    (void)dictSize;
+    (void)outSeqsCapacity;
+    (void)compressionLevel;
+
+    {   int i;
+        for (i=0; i < HSIZE; i++) {
+            hashTable[i] = BADIDX;
+    }   }
+
+    while (ip + MLS < iend) {
+        size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS);
+        U32 const matchIndex = hashTable[hash];
+        hashTable[hash] = (U32)(ip - istart);
+
+        if (matchIndex != BADIDX) {
+            const BYTE* const match = istart + matchIndex;
+            U32 const matchLen = (U32)ZSTD_count(ip, match, iend);
+            if (matchLen >= ZSTD_MINMATCH_MIN) {
+                U32 const litLen = (U32)(ip - anchor);
+                U32 const offset = (U32)(ip - match);
+                ZSTD_Sequence const seq = {
+                    offset, litLen, matchLen, 0
+                };
+
+                /* Note: it's crucial to stay within the window size! */
+                if (offset <= windowSize) {
+                    outSeqs[seqCount++] = seq;
+                    ip += matchLen;
+                    anchor = ip;
+                    continue;
+                }
+            }
+        }
+
+        ip++;
+    }
+
+    {   ZSTD_Sequence const finalSeq = {
+            0, (U32)(iend - anchor), 0, 0
+        };
+        outSeqs[seqCount++] = finalSeq;
+    }
+
+    return seqCount;
+}
diff --git a/contrib/externalMatchfinder/matchfinder.h b/contrib/externalMatchfinder/matchfinder.h
new file mode 100644
index 000000000..f8ba1c965
--- /dev/null
+++ b/contrib/externalMatchfinder/matchfinder.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MATCHFINDER_H
+#define MATCHFINDER_H
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+#endif
diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index 4e0a944f1..df6983245 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) 2021-2021, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/Makefile b/contrib/gen_html/Makefile
index 425f266c4..26e870399 100644
--- a/contrib/gen_html/Makefile
+++ b/contrib/gen_html/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp
index 90d5b21a3..1da5879ad 100644
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile
index 4c055b0ed..40734e62e 100644
--- a/contrib/largeNbDicts/Makefile
+++ b/contrib/largeNbDicts/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index cc489000f..7b2af0bae 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -856,7 +856,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary,
     CONTROL(cTotalSizeNoDict != 0);
     DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f  (%u bytes) \n",
                     clevel,
-                    (double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
+                    (double)totalSrcSlicesSize / (double)cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
 
     size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
     CONTROL(cSizes != NULL);
@@ -865,7 +865,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary,
     CONTROL(cTotalSize != 0);
     DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f  (%u bytes) \n",
                     (unsigned)dictBuffer.size,
-                    (double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
+                    (double)totalSrcSlicesSize / (double)cTotalSize, (unsigned)cTotalSize);
 
     /* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
     shrinkSizes(dstSlices, cSizes);
diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index f80ee8653..61a52ce20 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -49,8 +49,8 @@ libzstd:
 		-UZSTD_MULTITHREAD \
 		-U_MSC_VER \
 		-U_WIN32 \
-		-RZSTDLIB_VISIBILITY= \
-		-RZSTDERRORLIB_VISIBILITY= \
+		-RZSTDLIB_VISIBLE= \
+		-RZSTDERRORLIB_VISIBLE= \
 		-RZSTD_FALLTHROUGH=fallthrough \
 		-DZSTD_HAVE_WEAK_SYMBOLS=0 \
 		-DZSTD_TRACE=0 \
diff --git a/contrib/linux-kernel/decompress_sources.h b/contrib/linux-kernel/decompress_sources.h
index a06ca187a..8a47eb2a4 100644
--- a/contrib/linux-kernel/decompress_sources.h
+++ b/contrib/linux-kernel/decompress_sources.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/linux.mk b/contrib/linux-kernel/linux.mk
index 20f08c644..464c410b2 100644
--- a/contrib/linux-kernel/linux.mk
+++ b/contrib/linux-kernel/linux.mk
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/linux_zstd.h b/contrib/linux-kernel/linux_zstd.h
index 113408eef..f109d49f4 100644
--- a/contrib/linux-kernel/linux_zstd.h
+++ b/contrib/linux-kernel/linux_zstd.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/mem.h b/contrib/linux-kernel/mem.h
index 1d9cc0392..a7231822b 100644
--- a/contrib/linux-kernel/mem.h
+++ b/contrib/linux-kernel/mem.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile
index 53b0c2a65..67b55e665 100644
--- a/contrib/linux-kernel/test/Makefile
+++ b/contrib/linux-kernel/test/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h
index de43edb69..988ce4a20 100644
--- a/contrib/linux-kernel/test/include/linux/compiler.h
+++ b/contrib/linux-kernel/test/include/linux/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h
index b24752257..b4bdcba0e 100644
--- a/contrib/linux-kernel/test/include/linux/errno.h
+++ b/contrib/linux-kernel/test/include/linux/errno.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h
index 1f702abac..a4d791cd1 100644
--- a/contrib/linux-kernel/test/include/linux/kernel.h
+++ b/contrib/linux-kernel/test/include/linux/kernel.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h
index db9c09904..574aa7b34 100644
--- a/contrib/linux-kernel/test/include/linux/limits.h
+++ b/contrib/linux-kernel/test/include/linux/limits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h
index 8eefa2d5c..7f6713e73 100644
--- a/contrib/linux-kernel/test/include/linux/math64.h
+++ b/contrib/linux-kernel/test/include/linux/math64.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h
index 63a28d57b..06ef56f9e 100644
--- a/contrib/linux-kernel/test/include/linux/module.h
+++ b/contrib/linux-kernel/test/include/linux/module.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h
index eab08e0c4..92a25278e 100644
--- a/contrib/linux-kernel/test/include/linux/printk.h
+++ b/contrib/linux-kernel/test/include/linux/printk.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h
index 8538eb3e4..15c7408fc 100644
--- a/contrib/linux-kernel/test/include/linux/stddef.h
+++ b/contrib/linux-kernel/test/include/linux/stddef.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h
index 783046b42..2b48b434c 100644
--- a/contrib/linux-kernel/test/include/linux/swab.h
+++ b/contrib/linux-kernel/test/include/linux/swab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h
index 459a45700..b413db6f9 100644
--- a/contrib/linux-kernel/test/include/linux/types.h
+++ b/contrib/linux-kernel/test/include/linux/types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/static_test.c b/contrib/linux-kernel/test/static_test.c
index d2b8b5a32..ba4a420d4 100644
--- a/contrib/linux-kernel/test/static_test.c
+++ b/contrib/linux-kernel/test/static_test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c
index 67d248e0c..0f4ba3f45 100644
--- a/contrib/linux-kernel/test/test.c
+++ b/contrib/linux-kernel/test/test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_common_module.c b/contrib/linux-kernel/zstd_common_module.c
index 22686e367..2fead39eb 100644
--- a/contrib/linux-kernel/zstd_common_module.c
+++ b/contrib/linux-kernel/zstd_common_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_compress_module.c b/contrib/linux-kernel/zstd_compress_module.c
index 04e1b5c01..8ecf43226 100644
--- a/contrib/linux-kernel/zstd_compress_module.c
+++ b/contrib/linux-kernel/zstd_compress_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_decompress_module.c b/contrib/linux-kernel/zstd_decompress_module.c
index f4ed952ed..eb1c49e69 100644
--- a/contrib/linux-kernel/zstd_decompress_module.c
+++ b/contrib/linux-kernel/zstd_decompress_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_deps.h b/contrib/linux-kernel/zstd_deps.h
index 7a5bf4483..925161416 100644
--- a/contrib/linux-kernel/zstd_deps.h
+++ b/contrib/linux-kernel/zstd_deps.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.c b/contrib/match_finders/zstd_edist.c
index d63a7cf8d..d685cdd9e 100644
--- a/contrib/match_finders/zstd_edist.c
+++ b/contrib/match_finders/zstd_edist.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/match_finders/zstd_edist.h b/contrib/match_finders/zstd_edist.h
index a947649b2..c739e2abc 100644
--- a/contrib/match_finders/zstd_edist.h
+++ b/contrib/match_finders/zstd_edist.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/ErrorHolder.h b/contrib/pzstd/ErrorHolder.h
index 829651c59..2c2797ede 100644
--- a/contrib/pzstd/ErrorHolder.h
+++ b/contrib/pzstd/ErrorHolder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h
index beb160b64..84a08d200 100644
--- a/contrib/pzstd/Logging.h
+++ b/contrib/pzstd/Logging.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index 3d930cca9..830053cd7 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index 90f9d571f..a77824edb 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h
index 924543abf..92c18a350 100644
--- a/contrib/pzstd/Options.h
+++ b/contrib/pzstd/Options.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 2c09bda7a..67b941991 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -10,11 +10,13 @@
 #include "Pzstd.h"
 #include "SkippableFrame.h"
 #include "utils/FileSystem.h"
+#include "utils/Portability.h"
 #include "utils/Range.h"
 #include "utils/ScopeGuard.h"
 #include "utils/ThreadPool.h"
 #include "utils/WorkQueue.h"
 
+#include <algorithm>
 #include <chrono>
 #include <cinttypes>
 #include <cstddef>
@@ -336,6 +338,10 @@ static size_t calculateStep(
     const ZSTD_parameters &params) {
   (void)size;
   (void)numThreads;
+  // Not validated to work correctly for window logs > 23.
+  // It will definitely fail if windowLog + 2 is >= 4GB because
+  // the skippable frame can only store sizes up to 4GB.
+  assert(params.cParams.windowLog <= 23);
   return size_t{1} << (params.cParams.windowLog + 2);
 }
 
@@ -587,7 +593,8 @@ std::uint64_t writeFile(
       // start writing before compression is done because we need to know the
       // compressed size.
       // Wait for the compressed size to be available and write skippable frame
-      SkippableFrame frame(out->size());
+      assert(uint64_t(out->size()) < uint64_t(1) << 32);
+      SkippableFrame frame(uint32_t(out->size()));
       if (!writeData(frame.data(), outputFd)) {
         errorHolder.setError("Failed to write output");
         return bytesWritten;
diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h
index c667c887d..3645e5942 100644
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.cpp b/contrib/pzstd/SkippableFrame.cpp
index 769866dfc..3bea4eb65 100644
--- a/contrib/pzstd/SkippableFrame.cpp
+++ b/contrib/pzstd/SkippableFrame.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/SkippableFrame.h b/contrib/pzstd/SkippableFrame.h
index 60deed040..817415e92 100644
--- a/contrib/pzstd/SkippableFrame.h
+++ b/contrib/pzstd/SkippableFrame.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp
index b93f043b1..422b4a56a 100644
--- a/contrib/pzstd/main.cpp
+++ b/contrib/pzstd/main.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp
index e60114825..91e39750d 100644
--- a/contrib/pzstd/test/OptionsTest.cpp
+++ b/contrib/pzstd/test/OptionsTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp
index 5c7d66310..75453f5f6 100644
--- a/contrib/pzstd/test/PzstdTest.cpp
+++ b/contrib/pzstd/test/PzstdTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h
index c6364ecb4..f777622a3 100644
--- a/contrib/pzstd/test/RoundTrip.h
+++ b/contrib/pzstd/test/RoundTrip.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp
index 36af0673a..c37646d1c 100644
--- a/contrib/pzstd/test/RoundTripTest.cpp
+++ b/contrib/pzstd/test/RoundTripTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Buffer.h b/contrib/pzstd/utils/Buffer.h
index d17ad2f2c..a85f770ba 100644
--- a/contrib/pzstd/utils/Buffer.h
+++ b/contrib/pzstd/utils/Buffer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h
index 8898b0357..8d57d05f0 100644
--- a/contrib/pzstd/utils/FileSystem.h
+++ b/contrib/pzstd/utils/FileSystem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -8,11 +8,13 @@
  */
 #pragma once
 
+#include "utils/Portability.h"
 #include "utils/Range.h"
 
 #include <sys/stat.h>
 #include <cerrno>
 #include <cstdint>
+#include <limits>
 #include <system_error>
 
 // A small subset of `std::filesystem`.
@@ -82,11 +84,11 @@ inline std::uintmax_t file_size(
     std::error_code& ec) noexcept {
   auto stat = status(path, ec);
   if (ec) {
-    return -1;
+    return std::numeric_limits<uintmax_t>::max();
   }
   if (!is_regular_file(stat)) {
     ec.assign(ENOTSUP, std::generic_category());
-    return -1;
+    return std::numeric_limits<uintmax_t>::max();
   }
   ec.clear();
   return stat.st_size;
diff --git a/contrib/pzstd/utils/Likely.h b/contrib/pzstd/utils/Likely.h
index 7cea8da27..52243a64e 100644
--- a/contrib/pzstd/utils/Likely.h
+++ b/contrib/pzstd/utils/Likely.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/Portability.h b/contrib/pzstd/utils/Portability.h
new file mode 100644
index 000000000..ef1f86e51
--- /dev/null
+++ b/contrib/pzstd/utils/Portability.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+#pragma once
+
+#include <algorithm>
+
+// Required for windows, which defines min/max, but we want the std:: version.
+#undef min
+#undef max
diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h
index 6a850ad4e..0fd8f9f86 100644
--- a/contrib/pzstd/utils/Range.h
+++ b/contrib/pzstd/utils/Range.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,9 @@
 #pragma once
 
 #include "utils/Likely.h"
+#include "utils/Portability.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <cstring>
 #include <stdexcept>
diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h
index 8dfcdd765..7c4bb6235 100644
--- a/contrib/pzstd/utils/ResourcePool.h
+++ b/contrib/pzstd/utils/ResourcePool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ScopeGuard.h b/contrib/pzstd/utils/ScopeGuard.h
index c26f911bf..911fd9842 100644
--- a/contrib/pzstd/utils/ScopeGuard.h
+++ b/contrib/pzstd/utils/ScopeGuard.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h
index 8ece8e0da..a087d7c1c 100644
--- a/contrib/pzstd/utils/ThreadPool.h
+++ b/contrib/pzstd/utils/ThreadPool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h
index 1d14d922c..d7947b814 100644
--- a/contrib/pzstd/utils/WorkQueue.h
+++ b/contrib/pzstd/utils/WorkQueue.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/BufferTest.cpp b/contrib/pzstd/utils/test/BufferTest.cpp
index fbba74e82..58bf08dcd 100644
--- a/contrib/pzstd/utils/test/BufferTest.cpp
+++ b/contrib/pzstd/utils/test/BufferTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/RangeTest.cpp b/contrib/pzstd/utils/test/RangeTest.cpp
index 755b50fa6..8b7dee271 100644
--- a/contrib/pzstd/utils/test/RangeTest.cpp
+++ b/contrib/pzstd/utils/test/RangeTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
index 6fe145180..750ee084b 100644
--- a/contrib/pzstd/utils/test/ResourcePoolTest.cpp
+++ b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ScopeGuardTest.cpp b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
index 7bc624da7..0f77cdf38 100644
--- a/contrib/pzstd/utils/test/ScopeGuardTest.cpp
+++ b/contrib/pzstd/utils/test/ScopeGuardTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
index 703fd4c9c..a01052e60 100644
--- a/contrib/pzstd/utils/test/ThreadPoolTest.cpp
+++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp
index 14cf77304..16600bb60 100644
--- a/contrib/pzstd/utils/test/WorkQueueTest.cpp
+++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/Makefile b/contrib/recovery/Makefile
index 9a9f4f2e8..be6ea4b0e 100644
--- a/contrib/recovery/Makefile
+++ b/contrib/recovery/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/recovery/recover_directory.c b/contrib/recovery/recover_directory.c
index 13f83fd10..b9bd7ab49 100644
--- a/contrib/recovery/recover_directory.c
+++ b/contrib/recovery/recover_directory.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile
index 9df6b75fb..fcc04587c 100644
--- a/contrib/seekable_format/examples/Makefile
+++ b/contrib/seekable_format/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_compression.c b/contrib/seekable_format/examples/parallel_compression.c
index 4118b0ad7..0ec9fbd20 100644
--- a/contrib/seekable_format/examples/parallel_compression.c
+++ b/contrib/seekable_format/examples/parallel_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/parallel_processing.c b/contrib/seekable_format/examples/parallel_processing.c
index 36226b49f..b1709db77 100644
--- a/contrib/seekable_format/examples/parallel_processing.c
+++ b/contrib/seekable_format/examples/parallel_processing.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c
index 9a331a895..182b46f64 100644
--- a/contrib/seekable_format/examples/seekable_compression.c
+++ b/contrib/seekable_format/examples/seekable_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression.c b/contrib/seekable_format/examples/seekable_decompression.c
index e9e201333..2c4f3ba0c 100644
--- a/contrib/seekable_format/examples/seekable_decompression.c
+++ b/contrib/seekable_format/examples/seekable_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/examples/seekable_decompression_mem.c b/contrib/seekable_format/examples/seekable_decompression_mem.c
index e7b1c6505..44a06fbbf 100644
--- a/contrib/seekable_format/examples/seekable_decompression_mem.c
+++ b/contrib/seekable_format/examples/seekable_decompression_mem.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile
index d51deb3ea..a81f2229f 100644
--- a/contrib/seekable_format/tests/Makefile
+++ b/contrib/seekable_format/tests/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstd_seekable_compression_format.md b/contrib/seekable_format/zstd_seekable_compression_format.md
index 55aebfd2e..7bd0790e8 100644
--- a/contrib/seekable_format/zstd_seekable_compression_format.md
+++ b/contrib/seekable_format/zstd_seekable_compression_format.md
@@ -2,7 +2,7 @@
 
 ### Notices
 
-Copyright (c) 2017-present Facebook, Inc.
+Copyright (c) Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c
index 7ec9bb577..113f6f99c 100644
--- a/contrib/seekable_format/zstdseek_compress.c
+++ b/contrib/seekable_format/zstdseek_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index 24083aa52..7128b9d22 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/seqBench/Makefile b/contrib/seqBench/Makefile
index 0782961eb..e7f08a42c 100644
--- a/contrib/seqBench/Makefile
+++ b/contrib/seqBench/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index a9c601ebc..f6deeb13d 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index 935f60da8..12c5a801b 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index 54c059352..9ade76502 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h
index d89c83523..c13c8134d 100644
--- a/doc/educational_decoder/zstd_decompress.h
+++ b/doc/educational_decoder/zstd_decompress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 20c4a10e3..e40677a1d 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -3,7 +3,7 @@ Zstandard Compression Format
 
 ### Notices
 
-Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
+Copyright (c) Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,
@@ -435,7 +435,7 @@ They can be decoded first, and then copied during [Sequence Execution],
 or they can be decoded on the flow during [Sequence Execution].
 
 Literals can be stored uncompressed or compressed using Huffman prefix codes.
-When compressed, an optional tree description can be present,
+When compressed, a tree description may optionally be present,
 followed by 1 or 4 streams.
 
 | `Literals_Section_Header` | [`Huffman_Tree_Description`] | [jumpTable] | Stream1 | [Stream2] | [Stream3] | [Stream4] |
@@ -510,7 +510,7 @@ Its value is : `Size_Format = (Literals_Section_Header[0]>>2) & 3`
                `Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)`
 
 Only Stream1 is present for these cases.
-Note : it's allowed to represent a short value (for example `13`)
+Note : it's allowed to represent a short value (for example `27`)
 using a long format, even if it's less efficient.
 
 __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__ :
@@ -521,19 +521,33 @@ __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__
                Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023).
                `Literals_Section_Header` uses 3 bytes.
 - `Size_Format` == 01 : 4 streams.
-               Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023).
+               Both `Regenerated_Size` and `Compressed_Size` use 10 bits (6-1023).
                `Literals_Section_Header` uses 3 bytes.
 - `Size_Format` == 10 : 4 streams.
-               Both `Regenerated_Size` and `Compressed_Size` use 14 bits (0-16383).
+               Both `Regenerated_Size` and `Compressed_Size` use 14 bits (6-16383).
                `Literals_Section_Header` uses 4 bytes.
 - `Size_Format` == 11 : 4 streams.
-               Both `Regenerated_Size` and `Compressed_Size` use 18 bits (0-262143).
+               Both `Regenerated_Size` and `Compressed_Size` use 18 bits (6-262143).
                `Literals_Section_Header` uses 5 bytes.
 
 Both `Compressed_Size` and `Regenerated_Size` fields follow __little-endian__ convention.
 Note: `Compressed_Size` __includes__ the size of the Huffman Tree description
 _when_ it is present.
 
+4 streams is superior to 1 stream in decompression speed,
+by exploiting instruction level parallelism.
+But it's also more expensive,
+costing on average ~7.3 bytes more than the 1 stream mode, mostly from the jump table.
+
+In general, use the 4 streams mode when there are more literals to decode,
+to favor higher decompression speeds.
+Beyond 1KB, the 4 streams mode is compulsory anyway.
+
+Note that a minimum of 6 bytes is required for the 4 streams mode.
+That's a technical minimum, but it's not recommended to employ the 4 streams mode
+for such a small quantity, that would be wasteful.
+A more practical lower bound would be around ~256 bytes.
+
 #### Raw Literals Block
 The data in Stream1 is `Regenerated_Size` bytes long,
 it contains the raw literals data to be used during [Sequence Execution].
diff --git a/examples/Makefile b/examples/Makefile
index 8d7361dd8..31f52d357 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/common.h b/examples/common.h
index 9b09030eb..4873e877a 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c
index 7c7d34288..83edc1cad 100644
--- a/examples/dictionary_compression.c
+++ b/examples/dictionary_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c
index 107cfc1ee..e6c999964 100644
--- a/examples/dictionary_decompression.c
+++ b/examples/dictionary_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index 5d2a28fcd..bf77ca133 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index d4efc8e57..b12ad03dc 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_compression.c b/examples/simple_compression.c
index 27a65b17f..7c880725f 100644
--- a/examples/simple_compression.c
+++ b/examples/simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index 59c1fd414..f499156f6 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index ff1875829..ed0a3a69c 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression_thread_pool.c b/examples/streaming_compression_thread_pool.c
index 21cb3d549..a1a024129 100644
--- a/examples/streaming_compression_thread_pool.c
+++ b/examples/streaming_compression_thread_pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Martin Liska, SUSE, Facebook, Inc.
+ * Copyright (c) Martin Liska, SUSE, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 6dc4c2267..95fa11227 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c
index a5219ef1e..957acb61a 100644
--- a/examples/streaming_memory_usage.c
+++ b/examples/streaming_memory_usage.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/Makefile b/lib/Makefile
index 74c6f1baf..a4cf61ab1 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/README.md b/lib/README.md
index 015cde567..c3b5d1817 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -161,6 +161,13 @@ The file structure is designed to make this selection manually achievable for an
   `ZSTD_DCtx` decompression contexts,
   but might also result in a small decompression speed cost.
 
+- The C compiler macros `ZSTDLIB_VISIBLE`, `ZSTDERRORLIB_VISIBLE` and `ZDICTLIB_VISIBLE`
+  can be overridden to control the visibility of zstd's API. Additionally,
+  `ZSTDLIB_STATIC_API` and `ZDICTLIB_STATIC_API` can be overridden to control the visibility
+  of zstd's static API. Specifically, it can be set to `ZSTDLIB_HIDDEN` to hide the symbols
+  from the shared library. These macros default to `ZSTDLIB_VISIBILITY`,
+  `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility
+  with the old macro names.
 
 #### Windows : using MinGW+MSYS to create DLL
 
diff --git a/lib/common/bits.h b/lib/common/bits.h
index c0e917750..7939f3d0f 100644
--- a/lib/common/bits.h
+++ b/lib/common/bits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index 841778626..db1b4cf13 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * bitstream
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 42f289e0b..e423f2467 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/cpu.h b/lib/common/cpu.h
index 8acd33be3..8bc34a36d 100644
--- a/lib/common/cpu.h
+++ b/lib/common/cpu.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/debug.c b/lib/common/debug.c
index bb863c9ea..ebf7bfccf 100644
--- a/lib/common/debug.c
+++ b/lib/common/debug.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/debug.h b/lib/common/debug.h
index 3b2a320a1..0e9817ea6 100644
--- a/lib/common/debug.h
+++ b/lib/common/debug.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index 98bd4238d..4bf8242ff 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Common functions of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 1b67500f3..fb4d70596 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -29,7 +29,9 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
     case PREFIX(corruption_detected): return "Data corruption detected";
     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
     case PREFIX(parameter_unsupported): return "Unsupported parameter";
+    case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
     case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
@@ -50,6 +52,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
     case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
     case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
+    case PREFIX(externalMatchFinder_failed): return "External matchfinder returned an error code";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index 007d81066..325daad40 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/fse.h b/lib/common/fse.h
index 466a07281..2ef8df459 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
index 7034fd97b..ca894eb14 100644
--- a/lib/common/fse_decompress.c
+++ b/lib/common/fse_decompress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 595b2f6db..f7316f9b2 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -88,8 +88,10 @@ HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
                                unsigned maxSymbolValue, unsigned tableLog);
 
 /** HUF_compress4X_wksp() :
- *  Same as HUF_compress2(), but uses externally allocated `workSpace`.
- * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
+ *  Same as HUF_compress2(), but uses externally allocated @workSpace.
+ * @workSpace's size, aka @wkspSize, must be >= HUF_WORKSPACE_SIZE
+ * @srcSize must be >= 6
+ */
 #define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
 #define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
 HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
diff --git a/lib/common/mem.h b/lib/common/mem.h
index 493782f6f..98dd47a04 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/pool.c b/lib/common/pool.c
index bf21c57ed..3466b85b8 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -173,7 +173,7 @@ static void POOL_join(POOL_ctx* ctx) {
     /* Join all of the threads */
     {   size_t i;
         for (i = 0; i < ctx->threadCapacity; ++i) {
-            ZSTD_pthread_join(ctx->threads[i], NULL);  /* note : could fail */
+            ZSTD_pthread_join(ctx->threads[i]);  /* note : could fail */
     }   }
 }
 
diff --git a/lib/common/pool.h b/lib/common/pool.h
index b86a3452e..eb22ff509 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/portability_macros.h b/lib/common/portability_macros.h
index 1650fa3d8..bf3e2b7b1 100644
--- a/lib/common/portability_macros.h
+++ b/lib/common/portability_macros.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/threading.c b/lib/common/threading.c
index c62c65d57..f2341105a 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -34,39 +34,92 @@ int g_ZSTD_threading_useless_symbol;
 
 /* ===  Implementation  === */
 
+typedef struct {
+    void* (*start_routine)(void*);
+    void* arg;
+    int initialized;
+    ZSTD_pthread_cond_t initialized_cond;
+    ZSTD_pthread_mutex_t initialized_mutex;
+} ZSTD_thread_params_t;
+
 static unsigned __stdcall worker(void *arg)
 {
-    ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
-    thread->arg = thread->start_routine(thread->arg);
+    void* (*start_routine)(void*);
+    void* thread_arg;
+
+    /* Inialized thread_arg and start_routine and signal main thread that we don't need it
+     * to wait any longer.
+     */
+    {
+        ZSTD_thread_params_t*  thread_param = (ZSTD_thread_params_t*)arg;
+        thread_arg = thread_param->arg;
+        start_routine = thread_param->start_routine;
+
+        /* Signal main thread that we are running and do not depend on its memory anymore */
+        ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex);
+        thread_param->initialized = 1;
+        ZSTD_pthread_cond_signal(&thread_param->initialized_cond);
+        ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex);
+    }
+
+    start_routine(thread_arg);
+
     return 0;
 }
 
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
             void* (*start_routine) (void*), void* arg)
 {
+    ZSTD_thread_params_t thread_param;
     (void)unused;
-    thread->arg = arg;
-    thread->start_routine = start_routine;
-    thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
 
-    if (!thread->handle)
+    thread_param.start_routine = start_routine;
+    thread_param.arg = arg;
+    thread_param.initialized = 0;
+    *thread = NULL;
+
+    /* Setup thread initialization synchronization */
+    if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) {
+        /* Should never happen on Windows */
+        return -1;
+    }
+    if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) {
+        /* Should never happen on Windows */
+        ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
+        return -1;
+    }
+
+    /* Spawn thread */
+    *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL);
+    if (!thread) {
+        ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
+        ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
         return errno;
-    else
-        return 0;
+    }
+
+    /* Wait for thread to be initialized */
+    ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex);
+    while(!thread_param.initialized) {
+        ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex);
+    }
+    ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex);
+    ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
+    ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
+
+    return 0;
 }
 
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
+int ZSTD_pthread_join(ZSTD_pthread_t thread)
 {
     DWORD result;
 
-    if (!thread.handle) return 0;
+    if (!thread) return 0;
 
-    result = WaitForSingleObject(thread.handle, INFINITE);
-    CloseHandle(thread.handle);
+    result = WaitForSingleObject(thread, INFINITE);
+    CloseHandle(thread);
 
     switch (result) {
     case WAIT_OBJECT_0:
-        if (value_ptr) *value_ptr = thread.arg;
         return 0;
     case WAIT_ABANDONED:
         return EINVAL;
diff --git a/lib/common/threading.h b/lib/common/threading.h
index b1458054e..fb5c1c878 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -61,16 +61,12 @@ extern "C" {
 #define ZSTD_pthread_cond_broadcast(a)  WakeAllConditionVariable((a))
 
 /* ZSTD_pthread_create() and ZSTD_pthread_join() */
-typedef struct {
-    HANDLE handle;
-    void* (*start_routine)(void*);
-    void* arg;
-} ZSTD_pthread_t;
+typedef HANDLE ZSTD_pthread_t;
 
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
                    void* (*start_routine) (void*), void* arg);
 
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
+int ZSTD_pthread_join(ZSTD_pthread_t thread);
 
 /**
  * add here more wrappers as required
@@ -98,7 +94,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
 
 #define ZSTD_pthread_t                  pthread_t
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+#define ZSTD_pthread_join(a)         pthread_join((a),NULL)
 
 #else /* DEBUGLEVEL >= 1 */
 
@@ -123,7 +119,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
 
 #define ZSTD_pthread_t                  pthread_t
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+#define ZSTD_pthread_join(a)         pthread_join((a),NULL)
 
 #endif
 
diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index c2d61ca64..fd237c906 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -1,6 +1,6 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Facebook, Inc.
+ *  Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - xxHash homepage: https://cyan4973.github.io/xxHash/
diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index 901bc0832..b8b73290b 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -1,6 +1,6 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Facebook, Inc.
+ *  Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - xxHash homepage: https://cyan4973.github.io/xxHash/
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 3d7e35b30..320855247 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_deps.h b/lib/common/zstd_deps.h
index 14211344a..4d767ae9b 100644
--- a/lib/common/zstd_deps.h
+++ b/lib/common/zstd_deps.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index e89226702..48558873d 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -94,6 +94,7 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 
 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
 #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */)   /* for a non-null block */
+#define MIN_LITERALS_FOR_4_STREAMS 6
 
 typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
 
diff --git a/lib/common/zstd_trace.h b/lib/common/zstd_trace.h
index 6215f1e70..da20534eb 100644
--- a/lib/common/zstd_trace.h
+++ b/lib/common/zstd_trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/clevels.h b/lib/compress/clevels.h
index 7ed2e0049..c18da465f 100644
--- a/lib/compress/clevels.h
+++ b/lib/compress/clevels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index c2271f6ee..6a9aa11e5 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.c b/lib/compress/hist.c
index 073c57e75..e2fb431f0 100644
--- a/lib/compress/hist.c
+++ b/lib/compress/hist.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.h b/lib/compress/hist.h
index 228ed48a7..887896b81 100644
--- a/lib/compress/hist.h
+++ b/lib/compress/hist.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 9d2b521bc..f7ca2d3bb 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -386,7 +386,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits
 
         /* renorm totalCost from 2^largestBits to 2^targetNbBits
          * note : totalCost is necessarily a multiple of baseCost */
-        assert((totalCost & (baseCost - 1)) == 0);
+        assert(((U32)totalCost & (baseCost - 1)) == 0);
         totalCost >>= (largestBits - targetNbBits);
         assert(totalCost > 0);
 
@@ -1253,41 +1253,59 @@ unsigned HUF_minTableLog(unsigned symbolCardinality)
     return minBitsSymbols;
 }
 
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
+unsigned HUF_optimalTableLog(
+            unsigned maxTableLog,
+            size_t srcSize,
+            unsigned maxSymbolValue,
+            void* workSpace, size_t wkspSize,
+            HUF_CElt* table,
+      const unsigned* count,
+            HUF_depth_mode depthMode)
 {
-    unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
+    assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
 
-    if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/
-        BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+    if (depthMode != HUF_depth_optimal) {
+        /* cheap evaluation, based on FSE */
+        return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+    }
+
+    {   BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
         size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
-        size_t optSize = ((size_t) ~0);
-        unsigned huffLog;
         size_t maxBits, hSize, newSize;
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
+        const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
+        size_t optSize = ((size_t) ~0) - 1;
+        unsigned optLog = maxTableLog, optLogGuess;
 
-        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog;
+        DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
 
-        for (huffLog = HUF_minTableLog(symbolCardinality); huffLog <= maxTableLog; huffLog++) {
-            maxBits = HUF_buildCTable_wksp(table, count,
-                                            maxSymbolValue, huffLog,
-                                            workSpace, wkspSize);
+        /* Search until size increases */
+        for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+            DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
+            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
             if (ERR_isError(maxBits)) continue;
 
-            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits,
-                                              workSpace, wkspSize);
+            if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
+
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
+
             if (ERR_isError(hSize)) continue;
 
             newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
 
+            if (newSize > optSize + 1) {
+                break;
+            }
+
             if (newSize < optSize) {
                 optSize = newSize;
-                optLog = huffLog;
+                optLog = optLogGuess;
             }
         }
+        assert(optLog <= HUF_TABLELOG_MAX);
+        return optLog;
     }
-    assert(optLog <= HUF_TABLELOG_MAX);
-    return optLog;
 }
 
 /* HUF_compress_internal() :
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index d2761974c..b9b056cd5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -59,14 +59,17 @@
 *  Helper functions
 ***************************************/
 /* ZSTD_compressBound()
- * Note that the result from this function is only compatible with the "normal"
- * full-block strategy.
- * When there are a lot of small blocks due to frequent flush in streaming mode
- * the overhead of headers can make the compressed data to be larger than the
- * return value of ZSTD_compressBound().
+ * Note that the result from this function is only valid for
+ * the one-pass compression functions.
+ * When employing the streaming mode,
+ * if flushes are frequently altering the size of blocks,
+ * the overhead from block headers can make the compressed data larger
+ * than the return value of ZSTD_compressBound().
  */
 size_t ZSTD_compressBound(size_t srcSize) {
-    return ZSTD_COMPRESSBOUND(srcSize);
+    size_t const r = ZSTD_COMPRESSBOUND(srcSize);
+    if (r==0) return ERROR(srcSize_wrong);
+    return r;
 }
 
 
@@ -275,6 +278,16 @@ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
     return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
 }
 
+/* Enables validation for external sequences in debug builds. */
+static int ZSTD_resolveExternalSequenceValidation(int mode) {
+#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2)
+    (void)mode;
+    return 1;
+#else
+    return mode;
+#endif
+}
+
 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
  * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
@@ -298,6 +311,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     }
     cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+    cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -343,10 +357,13 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
 #define ZSTD_NO_CLEVEL 0
 
 /**
- * Initializes the cctxParams from params and compressionLevel.
+ * Initializes `cctxParams` from `params` and `compressionLevel`.
  * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL.
  */
-static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel)
+static void
+ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
+                        const ZSTD_parameters* params,
+                              int compressionLevel)
 {
     assert(!ZSTD_checkCParams(params->cParams));
     ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
@@ -359,6 +376,7 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par
     cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
     cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
+    cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -373,7 +391,7 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete
 
 /**
  * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone.
- * @param param Validated zstd parameters.
+ * @param params Validated zstd parameters.
  */
 static void ZSTD_CCtxParams_setZstdParams(
         ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
@@ -581,6 +599,16 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
+    case ZSTD_c_enableMatchFinderFallback:
+        bounds.lowerBound = 0;
+        bounds.upperBound = 1;
+        return bounds;
+
+    case ZSTD_c_maxBlockSize:
+        bounds.lowerBound = 1;
+        bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+        return bounds;
+
     default:
         bounds.error = ERROR(parameter_unsupported);
         return bounds;
@@ -646,6 +674,8 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
     default:
         return 0;
     }
@@ -658,7 +688,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
         if (ZSTD_isUpdateAuthorized(param)) {
             cctx->cParamsChanged = 1;
         } else {
-            RETURN_ERROR(stage_wrong, "can only set params in ctx init stage");
+            RETURN_ERROR(stage_wrong, "can only set params in cctx init stage");
     }   }
 
     switch(param)
@@ -702,6 +732,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
         break;
 
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -793,14 +825,14 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
 
     case ZSTD_c_forceAttachDict : {
         const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
-        BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
+        BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref);
         CCtxParams->attachDictPref = pref;
         return CCtxParams->attachDictPref;
     }
 
     case ZSTD_c_literalCompressionMode : {
         const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
-        BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
+        BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm);
         CCtxParams->literalCompressionMode = lcm;
         return CCtxParams->literalCompressionMode;
     }
@@ -934,6 +966,16 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
         return CCtxParams->prefetchCDictTables;
 
+    case ZSTD_c_enableMatchFinderFallback:
+        BOUNDCHECK(ZSTD_c_enableMatchFinderFallback, value);
+        CCtxParams->enableMatchFinderFallback = value;
+        return CCtxParams->enableMatchFinderFallback;
+
+    case ZSTD_c_maxBlockSize:
+        BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+        CCtxParams->maxBlockSize = value;
+        return CCtxParams->prefetchCDictTables;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -1069,6 +1111,12 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_prefetchCDictTables:
         *value = (int)CCtxParams->prefetchCDictTables;
         break;
+    case ZSTD_c_enableMatchFinderFallback:
+        *value = CCtxParams->enableMatchFinderFallback;
+        break;
+    case ZSTD_c_maxBlockSize:
+        *value = (int)CCtxParams->maxBlockSize;
+        break;
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
@@ -1095,6 +1143,21 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
     return 0;
 }
 
+size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
+{
+    DEBUGLOG(4, "ZSTD_CCtx_setCParams");
+    assert(cctx != NULL);
+    if (cctx->streamStage != zcss_init) {
+        /* All parameters in @cparams are allowed to be updated during MT compression.
+         * This must be signaled, so that MT compression picks up the changes */
+        cctx->cParamsChanged = 1;
+    }
+    /* only update if parameters are valid */
+    FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
+    cctx->requestedParams.cParams = cparams;
+    return 0;
+}
+
 size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
@@ -1240,6 +1303,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
         RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                         "Can't reset parameters only when not in init stage.");
         ZSTD_clearAllDicts(cctx);
+        ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx));
         return ZSTD_CCtxParams_reset(&cctx->requestedParams);
     }
     return 0;
@@ -1482,6 +1546,13 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
     return tableSpace + optSpace + slackSpace + lazyAdditionalSpace;
 }
 
+/* Helper function for calculating memory requirements.
+ * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */
+static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useExternalMatchFinder) {
+    U32 const divider = (minMatch==3 || useExternalMatchFinder) ? 3 : 4;
+    return blockSize / divider;
+}
+
 static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_compressionParameters* cParams,
         const ldmParams_t* ldmParams,
@@ -1489,12 +1560,12 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_paramSwitch_e useRowMatchFinder,
         const size_t buffInSize,
         const size_t buffOutSize,
-        const U64 pledgedSrcSize)
+        const U64 pledgedSrcSize,
+        int useExternalMatchFinder)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
     size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-    U32    const divider = (cParams->minMatch==3) ? 3 : 4;
-    size_t const maxNbSeq = blockSize / divider;
+    size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
                             + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
@@ -1513,6 +1584,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 
     size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
 
+    size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+    size_t const externalSeqSpace = useExternalMatchFinder
+        ? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+        : 0;
+
     size_t const neededSpace =
         cctxSpace +
         entropySpace +
@@ -1521,7 +1597,8 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         ldmSeqSpace +
         matchStateSize +
         tokenSpace +
-        bufferSpace;
+        bufferSpace +
+        externalSeqSpace;
 
     DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
     return neededSpace;
@@ -1539,7 +1616,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
      * be needed. However, we still allocate two 0-sized buffers, which can
      * take space under ASAN. */
     return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
-        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN);
+        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
 }
 
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1600,7 +1677,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
-            ZSTD_CONTENTSIZE_UNKNOWN);
+            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
     }
 }
 
@@ -1883,8 +1960,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
         size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-        U32    const divider = (params->cParams.minMatch==3) ? 3 : 4;
-        size_t const maxNbSeq = blockSize / divider;
+        size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
         size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
                 : 0;
@@ -1901,7 +1977,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         size_t const neededSpace =
             ZSTD_estimateCCtxSize_usingCCtxParams_internal(
                 &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
-                buffInSize, buffOutSize, pledgedSrcSize);
+                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder);
         int resizeWorkspace;
 
         FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
@@ -2014,6 +2090,14 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
             zc->ldmState.loadedDictEnd = 0;
         }
 
+        /* reserve space for block-level external sequences */
+        if (params->useExternalMatchFinder) {
+            size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+            zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
+            zc->externalMatchCtx.seqBuffer =
+                (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
+        }
+
         DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
         assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
 
@@ -2493,11 +2577,13 @@ typedef struct {
  * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
-                        const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
-                              BYTE* dst, const BYTE* const dstEnd,
-                              ZSTD_strategy strategy, unsigned* countWorkspace,
-                              void* entropyWorkspace, size_t entropyWkspSize) {
+ZSTD_buildSequencesStatistics(
+                const seqStore_t* seqStorePtr, size_t nbSeq,
+                const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
+                      BYTE* dst, const BYTE* const dstEnd,
+                      ZSTD_strategy strategy, unsigned* countWorkspace,
+                      void* entropyWorkspace, size_t entropyWkspSize)
+{
     BYTE* const ostart = dst;
     const BYTE* const oend = dstEnd;
     BYTE* op = ostart;
@@ -2616,13 +2702,14 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
  */
 #define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
-                          const ZSTD_entropyCTables_t* prevEntropy,
-                                ZSTD_entropyCTables_t* nextEntropy,
-                          const ZSTD_CCtx_params* cctxParams,
-                                void* dst, size_t dstCapacity,
-                                void* entropyWorkspace, size_t entropyWkspSize,
-                          const int bmi2)
+ZSTD_entropyCompressSeqStore_internal(
+                        const seqStore_t* seqStorePtr,
+                        const ZSTD_entropyCTables_t* prevEntropy,
+                              ZSTD_entropyCTables_t* nextEntropy,
+                        const ZSTD_CCtx_params* cctxParams,
+                              void* dst, size_t dstCapacity,
+                              void* entropyWorkspace, size_t entropyWkspSize,
+                        const int bmi2)
 {
     const int longOffsets = cctxParams->cParams.windowLog >= STREAM_ACCUMULATOR_MIN;
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
@@ -2631,7 +2718,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
     FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
     FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
     const seqDef* const sequences = seqStorePtr->sequencesStart;
-    const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
     const BYTE* const llCodeTable = seqStorePtr->llCode;
     const BYTE* const mlCodeTable = seqStorePtr->mlCode;
@@ -2649,21 +2736,20 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
 
     /* Compress literals */
     {   const BYTE* const literals = seqStorePtr->litStart;
-        size_t const numSequences = seqStorePtr->sequences - seqStorePtr->sequencesStart;
-        size_t const numLiterals = seqStorePtr->lit - seqStorePtr->litStart;
+        size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+        size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
         /* Base suspicion of uncompressibility on ratio of literals to sequences */
         unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
         size_t const litSize = (size_t)(seqStorePtr->lit - literals);
 
-        HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
         size_t const cSize = ZSTD_compressLiterals(
-                                    &prevEntropy->huf, &nextEntropy->huf,
-                                    cctxParams->cParams.strategy,
-                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
                                     op, dstCapacity,
                                     literals, litSize,
                                     entropyWorkspace, entropyWkspSize,
-                                    bmi2, suspectUncompressible, depthMode);
+                                    &prevEntropy->huf, &nextEntropy->huf,
+                                    cctxParams->cParams.strategy,
+                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
+                                    suspectUncompressible, bmi2);
         FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
         assert(cSize <= dstCapacity);
         op += cSize;
@@ -2735,14 +2821,15 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
 }
 
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
-                       const ZSTD_entropyCTables_t* prevEntropy,
-                             ZSTD_entropyCTables_t* nextEntropy,
-                       const ZSTD_CCtx_params* cctxParams,
-                             void* dst, size_t dstCapacity,
-                             size_t srcSize,
-                             void* entropyWorkspace, size_t entropyWkspSize,
-                             int bmi2)
+ZSTD_entropyCompressSeqStore(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          void* dst, size_t dstCapacity,
+                          size_t srcSize,
+                          void* entropyWorkspace, size_t entropyWkspSize,
+                          int bmi2)
 {
     size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
                             seqStorePtr, prevEntropy, nextEntropy, cctxParams,
@@ -2763,6 +2850,10 @@ ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
         if (cSize >= maxCSize) return 0;  /* block not compressed */
     }
     DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
+    /* libzstd decoder before  > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
+     * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
+     */
+    assert(cSize < ZSTD_BLOCKSIZE_MAX);
     return cSize;
 }
 
@@ -2857,6 +2948,55 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
     ssPtr->longLengthType = ZSTD_llt_none;
 }
 
+/* ZSTD_postProcessExternalMatchFinderResult() :
+ * Validates and post-processes sequences obtained through the external matchfinder API:
+ *   - Checks whether nbExternalSeqs represents an error condition.
+ *   - Appends a block delimiter to outSeqs if one is not already present.
+ *     See zstd.h for context regarding block delimiters.
+ * Returns the number of sequences after post-processing, or an error code. */
+static size_t ZSTD_postProcessExternalMatchFinderResult(
+    ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize
+) {
+    RETURN_ERROR_IF(
+        nbExternalSeqs > outSeqsCapacity,
+        externalMatchFinder_failed,
+        "External matchfinder returned error code %lu",
+        (unsigned long)nbExternalSeqs
+    );
+
+    RETURN_ERROR_IF(
+        nbExternalSeqs == 0 && srcSize > 0,
+        externalMatchFinder_failed,
+        "External matchfinder produced zero sequences for a non-empty src buffer!"
+    );
+
+    if (srcSize == 0) {
+        ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence));
+        return 1;
+    }
+
+    {
+        ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1];
+
+        /* We can return early if lastSeq is already a block delimiter. */
+        if (lastSeq.offset == 0 && lastSeq.matchLength == 0) {
+            return nbExternalSeqs;
+        }
+
+        /* This error condition is only possible if the external matchfinder
+         * produced an invalid parse, by definition of ZSTD_sequenceBound(). */
+        RETURN_ERROR_IF(
+            nbExternalSeqs == outSeqsCapacity,
+            externalMatchFinder_failed,
+            "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!"
+        );
+
+        /* lastSeq is not a block delimiter, so we need to append one. */
+        ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence));
+        return nbExternalSeqs + 1;
+    }
+}
+
 typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
 
 static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
@@ -2904,6 +3044,15 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         }
         if (zc->externSeqStore.pos < zc->externSeqStore.size) {
             assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
+
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useExternalMatchFinder,
+                parameter_combination_unsupported,
+                "Long-distance matching with external matchfinder enabled is not currently supported."
+            );
+
             /* Updates ldmSeqStore.pos */
             lastLLSize =
                 ZSTD_ldm_blockCompress(&zc->externSeqStore,
@@ -2915,6 +3064,14 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
             rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
 
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useExternalMatchFinder,
+                parameter_combination_unsupported,
+                "Long-distance matching with external matchfinder enabled is not currently supported."
+            );
+
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
             /* Updates ldmSeqStore.size */
@@ -2929,10 +3086,64 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                                        zc->appliedParams.useRowMatchFinder,
                                        src, srcSize);
             assert(ldmSeqStore.pos == ldmSeqStore.size);
-        } else {   /* not long range mode */
+        } else if (zc->appliedParams.useExternalMatchFinder) {
+            assert(
+                zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize)
+            );
+            assert(zc->externalMatchCtx.mFinder != NULL);
+
+            {   U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog;
+
+                size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)(
+                    zc->externalMatchCtx.mState,
+                    zc->externalMatchCtx.seqBuffer,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    src, srcSize,
+                    NULL, 0,  /* dict and dictSize, currently not supported */
+                    zc->appliedParams.compressionLevel,
+                    windowSize
+                );
+
+                size_t const nbPostProcessedSeqs = ZSTD_postProcessExternalMatchFinderResult(
+                    zc->externalMatchCtx.seqBuffer,
+                    nbExternalSeqs,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    srcSize
+                );
+
+                /* Return early if there is no error, since we don't need to worry about last literals */
+                if (!ZSTD_isError(nbPostProcessedSeqs)) {
+                    ZSTD_sequencePosition seqPos = {0,0,0};
+                    ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
+                        zc, &seqPos, zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs, src, srcSize
+                    );
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(5, "Copied %lu sequences from external matchfinder to internal seqStore.", (unsigned long)nbExternalSeqs);
+                    return ZSTDbss_compress;
+                }
+
+                /* Propagate the error if fallback is disabled */
+                if (!zc->appliedParams.enableMatchFinderFallback) {
+                    return nbPostProcessedSeqs;
+                }
+
+                /* Fallback to software matchfinder */
+                {   ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+                                                                                            zc->appliedParams.useRowMatchFinder,
+                                                                                            dictMode);
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(
+                        5,
+                        "External matchfinder returned error code %lu. Falling back to internal matchfinder.",
+                        (unsigned long)nbExternalSeqs
+                    );
+                    lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+            }   }
+        } else {   /* not long range mode and no external matchfinder */
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
                                                                                     zc->appliedParams.useRowMatchFinder,
                                                                                     dictMode);
+            assert(zc->externalMatchCtx.mFinder == NULL);
             ms->ldmSeqStore = NULL;
             lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
         }
@@ -3055,19 +3266,17 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) {
     const size_t unrollMask = unrollSize - 1;
     const size_t prefixLength = length & unrollMask;
     size_t i;
-    size_t u;
     if (length == 1) return 1;
     /* Check if prefix is RLE first before using unrolled loop */
     if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
         return 0;
     }
     for (i = prefixLength; i != length; i += unrollSize) {
+        size_t u;
         for (u = 0; u < unrollSize; u += sizeof(size_t)) {
             if (MEM_readST(ip + i + u) != valueST) {
                 return 0;
-            }
-        }
-    }
+    }   }   }
     return 1;
 }
 
@@ -3083,7 +3292,8 @@ static int ZSTD_maybeRLE(seqStore_t const* seqStore)
     return nbSeqs < 4 && nbLits < 10;
 }
 
-static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
+static void
+ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
 {
     ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock;
     bs->prevCBlock = bs->nextCBlock;
@@ -3091,7 +3301,9 @@ static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* c
 }
 
 /* Writes the block header */
-static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) {
+static void
+writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
+{
     U32 const cBlockHeader = cSize == 1 ?
                         lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
                         lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
@@ -3104,13 +3316,16 @@ static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastB
  *  Stores literals block type (raw, rle, compressed, repeat) and
  *  huffman description table to hufMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE workspace
- *  @return : size of huffman description table or error code */
-static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
-                                            const ZSTD_hufCTables_t* prevHuf,
-                                                  ZSTD_hufCTables_t* nextHuf,
-                                                  ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                  const int literalsCompressionIsDisabled,
-                                                  void* workspace, size_t wkspSize, HUF_depth_mode depthMode)
+ * @return : size of huffman description table, or an error code
+ */
+static size_t
+ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
+                               const ZSTD_hufCTables_t* prevHuf,
+                                     ZSTD_hufCTables_t* nextHuf,
+                                     ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               const int literalsCompressionIsDisabled,
+                                     void* workspace, size_t wkspSize,
+                                     HUF_depth_mode depthMode)
 {
     BYTE* const wkspStart = (BYTE*)workspace;
     BYTE* const wkspEnd = wkspStart + wkspSize;
@@ -3135,33 +3350,37 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
 
     /* small ? don't even attempt compression (speed opt) */
 #ifndef COMPRESS_LITERALS_SIZE_MIN
-#define COMPRESS_LITERALS_SIZE_MIN 63
+# define COMPRESS_LITERALS_SIZE_MIN 63  /* heuristic */
 #endif
     {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
         if (srcSize <= minLitSize) {
             DEBUGLOG(5, "set_basic - too small");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Scan input and build symbol stats */
-    {   size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
+    {   size_t const largest =
+            HIST_count_wksp (countWksp, &maxSymbolValue,
+                            (const BYTE*)src, srcSize,
+                            workspace, wkspSize);
         FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
         if (largest == srcSize) {
+            /* only one literal symbol */
             DEBUGLOG(5, "set_rle");
             hufMetadata->hType = set_rle;
             return 0;
         }
         if (largest <= (srcSize >> 7)+4) {
+            /* heuristic: likely not compressible */
             DEBUGLOG(5, "set_basic - no gain");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Validate the previous Huffman table */
-    if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
+    if (repeat == HUF_repeat_check
+      && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
         repeat = HUF_repeat_none;
     }
 
@@ -3174,35 +3393,34 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
                                                     nodeWksp, nodeWkspSize);
         FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
         huffLog = (U32)maxBits;
-        {   /* Build and write the CTable */
-            size_t const newCSize = HUF_estimateCompressedSize(
-                    (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
-            size_t const hSize = HUF_writeCTable_wksp(
-                    hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
-                    (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
-                    nodeWksp, nodeWkspSize);
-            /* Check against repeating the previous CTable */
-            if (repeat != HUF_repeat_none) {
-                size_t const oldCSize = HUF_estimateCompressedSize(
-                        (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
-                if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
-                    DEBUGLOG(5, "set_repeat - smaller");
-                    ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                    hufMetadata->hType = set_repeat;
-                    return 0;
-                }
-            }
-            if (newCSize + hSize >= srcSize) {
-                DEBUGLOG(5, "set_basic - no gains");
+    }
+    {   /* Build and write the CTable */
+        size_t const newCSize = HUF_estimateCompressedSize(
+                (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
+        size_t const hSize = HUF_writeCTable_wksp(
+                hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
+                (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
+                nodeWksp, nodeWkspSize);
+        /* Check against repeating the previous CTable */
+        if (repeat != HUF_repeat_none) {
+            size_t const oldCSize = HUF_estimateCompressedSize(
+                    (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
+            if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
+                DEBUGLOG(5, "set_repeat - smaller");
                 ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                hufMetadata->hType = set_basic;
+                hufMetadata->hType = set_repeat;
                 return 0;
-            }
-            DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
-            hufMetadata->hType = set_compressed;
-            nextHuf->repeatMode = HUF_repeat_check;
-            return hSize;
+        }   }
+        if (newCSize + hSize >= srcSize) {
+            DEBUGLOG(5, "set_basic - no gains");
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            hufMetadata->hType = set_basic;
+            return 0;
         }
+        DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
+        hufMetadata->hType = set_compressed;
+        nextHuf->repeatMode = HUF_repeat_check;
+        return hSize;
     }
 }
 
@@ -3212,7 +3430,8 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
  * and updates nextEntropy to the appropriate repeatMode.
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
+ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
+{
     ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0};
     nextEntropy->litlength_repeatMode = FSE_repeat_none;
     nextEntropy->offcode_repeatMode = FSE_repeat_none;
@@ -3224,16 +3443,18 @@ ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
  *  Builds entropy for the sequences.
  *  Stores symbol compression modes and fse table to fseMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE wksp.
- *  @return : size of fse tables or error code */
-static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
-                                              const ZSTD_fseCTables_t* prevEntropy,
-                                                    ZSTD_fseCTables_t* nextEntropy,
-                                              const ZSTD_CCtx_params* cctxParams,
-                                                    ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                    void* workspace, size_t wkspSize)
+ * @return : size of fse tables or error code */
+static size_t
+ZSTD_buildBlockEntropyStats_sequences(
+                const seqStore_t* seqStorePtr,
+                const ZSTD_fseCTables_t* prevEntropy,
+                      ZSTD_fseCTables_t* nextEntropy,
+                const ZSTD_CCtx_params* cctxParams,
+                      ZSTD_fseCTablesMetadata_t* fseMetadata,
+                      void* workspace, size_t wkspSize)
 {
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
-    size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     BYTE* const ostart = fseMetadata->fseTablesBuffer;
     BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
     BYTE* op = ostart;
@@ -3260,18 +3481,20 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
 /** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  Requires workspace size ENTROPY_WORKSPACE_SIZE
- *
- *  @return : 0 on success or error code
+ * @return : 0 on success, or an error code
+ *  Note : also employed in superblock
  */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize)
+size_t ZSTD_buildBlockEntropyStats(
+            const seqStore_t* seqStorePtr,
+            const ZSTD_entropyCTables_t* prevEntropy,
+                  ZSTD_entropyCTables_t* nextEntropy,
+            const ZSTD_CCtx_params* cctxParams,
+                  ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                  void* workspace, size_t wkspSize)
 {
-    size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
-    HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
+    size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+    int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
+    HUF_depth_mode const depthMode = huf_useOptDepth ? HUF_depth_optimal : HUF_depth_fast;
 
     entropyMetadata->hufMetadata.hufDesSize =
         ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
@@ -3292,11 +3515,12 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
 }
 
 /* Returns the size estimate for the literals section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
-                                                const ZSTD_hufCTables_t* huf,
-                                                const ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                void* workspace, size_t wkspSize,
-                                                int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
+                               const ZSTD_hufCTables_t* huf,
+                               const ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               void* workspace, size_t wkspSize,
+                               int writeEntropy)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
@@ -3318,12 +3542,13 @@ static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSiz
 }
 
 /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
-static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
-                        const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
-                        const FSE_CTable* fseCTable,
-                        const U8* additionalBits,
-                        short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
-                        void* workspace, size_t wkspSize)
+static size_t
+ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
+                    const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
+                    const FSE_CTable* fseCTable,
+                    const U8* additionalBits,
+                    short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+                    void* workspace, size_t wkspSize)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     const BYTE* ctp = codeTable;
@@ -3355,99 +3580,107 @@ static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
 }
 
 /* Returns the size estimate for the sequences section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
-                                                  const BYTE* llCodeTable,
-                                                  const BYTE* mlCodeTable,
-                                                  size_t nbSeq,
-                                                  const ZSTD_fseCTables_t* fseTables,
-                                                  const ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                  void* workspace, size_t wkspSize,
-                                                  int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
+                                 const BYTE* llCodeTable,
+                                 const BYTE* mlCodeTable,
+                                 size_t nbSeq,
+                                 const ZSTD_fseCTables_t* fseTables,
+                                 const ZSTD_fseCTablesMetadata_t* fseMetadata,
+                                 void* workspace, size_t wkspSize,
+                                 int writeEntropy)
 {
     size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ);
     size_t cSeqSizeEstimate = 0;
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff,
-                                         fseTables->offcodeCTable, NULL,
-                                         OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
-                                         workspace, wkspSize);
+                                    fseTables->offcodeCTable, NULL,
+                                    OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL,
-                                         fseTables->litlengthCTable, LL_bits,
-                                         LL_defaultNorm, LL_defaultNormLog, MaxLL,
-                                         workspace, wkspSize);
+                                    fseTables->litlengthCTable, LL_bits,
+                                    LL_defaultNorm, LL_defaultNormLog, MaxLL,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML,
-                                         fseTables->matchlengthCTable, ML_bits,
-                                         ML_defaultNorm, ML_defaultNormLog, MaxML,
-                                         workspace, wkspSize);
+                                    fseTables->matchlengthCTable, ML_bits,
+                                    ML_defaultNorm, ML_defaultNormLog, MaxML,
+                                    workspace, wkspSize);
     if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
     return cSeqSizeEstimate + sequencesSectionHeaderSize;
 }
 
 /* Returns the size estimate for a given stream of literals, of, ll, ml */
-static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
-                                     const BYTE* ofCodeTable,
-                                     const BYTE* llCodeTable,
-                                     const BYTE* mlCodeTable,
-                                     size_t nbSeq,
-                                     const ZSTD_entropyCTables_t* entropy,
-                                     const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                     void* workspace, size_t wkspSize,
-                                     int writeLitEntropy, int writeSeqEntropy) {
+static size_t
+ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
+                       const BYTE* ofCodeTable,
+                       const BYTE* llCodeTable,
+                       const BYTE* mlCodeTable,
+                       size_t nbSeq,
+                       const ZSTD_entropyCTables_t* entropy,
+                       const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                       void* workspace, size_t wkspSize,
+                       int writeLitEntropy, int writeSeqEntropy)
+{
     size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize,
-                                                         &entropy->huf, &entropyMetadata->hufMetadata,
-                                                         workspace, wkspSize, writeLitEntropy);
+                                    &entropy->huf, &entropyMetadata->hufMetadata,
+                                    workspace, wkspSize, writeLitEntropy);
     size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
-                                                         nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
-                                                         workspace, wkspSize, writeSeqEntropy);
+                                    nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+                                    workspace, wkspSize, writeSeqEntropy);
     return seqSize + literalsSize + ZSTD_blockHeaderSize;
 }
 
 /* Builds entropy statistics and uses them for blocksize estimation.
  *
- * Returns the estimated compressed size of the seqStore, or a zstd error.
+ * @return: estimated compressed size of the seqStore, or a zstd error.
  */
-static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) {
-    ZSTD_entropyCTablesMetadata_t* entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
+static size_t
+ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc)
+{
+    ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
     DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
     FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
                     &zc->blockState.prevCBlock->entropy,
                     &zc->blockState.nextCBlock->entropy,
                     &zc->appliedParams,
                     entropyMetadata,
-                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
-    return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), "");
+    return ZSTD_estimateBlockSize(
+                    seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
                     seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
                     (size_t)(seqStore->sequences - seqStore->sequencesStart),
-                    &zc->blockState.nextCBlock->entropy, entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
+                    &zc->blockState.nextCBlock->entropy,
+                    entropyMetadata,
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
                     (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
 }
 
 /* Returns literals bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore)
+{
     size_t literalsBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
-        seqDef seq = seqStore->sequencesStart[i];
+        seqDef const seq = seqStore->sequencesStart[i];
         literalsBytes += seq.litLength;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
             literalsBytes += 0x10000;
-        }
-    }
+    }   }
     return literalsBytes;
 }
 
 /* Returns match bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore)
+{
     size_t matchBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
         seqDef seq = seqStore->sequencesStart[i];
         matchBytes += seq.mlBase + MINMATCH;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
             matchBytes += 0x10000;
-        }
-    }
+    }   }
     return matchBytes;
 }
 
@@ -3496,6 +3729,7 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, c
     U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0;  /* [ 0 - 3 ] */
     assert(OFFBASE_IS_REPCODE(offBase));
     if (adjustedRepCode == ZSTD_REP_NUM) {
+        assert(ll0);
         /* litlength == 0 and offCode == 2 implies selection of first repcode - 1
          * This is only valid if it results in a valid offset value, aka > 0.
          * Note : it may happen that `rep[0]==1` in exceptional circumstances.
@@ -3521,14 +3755,16 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, c
  *        1-3 : repcode 1-3
  *        4+ : real_offset+3
  */
-static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
-                                          seqStore_t* const seqStore, U32 const nbSeq) {
+static void
+ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
+                        const seqStore_t* const seqStore, U32 const nbSeq)
+{
     U32 idx = 0;
     for (; idx < nbSeq; ++idx) {
         seqDef* const seq = seqStore->sequencesStart + idx;
         U32 const ll0 = (seq->litLength == 0);
         U32 const offBase = seq->offBase;
-        assert(seq->offBase > 0);
+        assert(offBase > 0);
         if (OFFBASE_IS_REPCODE(offBase)) {
             U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0);
             U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0);
@@ -3537,7 +3773,7 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
              * repcode history.
              */
             if (dRawOffset != cRawOffset) {
-                seq->offBase = cRawOffset + ZSTD_REP_NUM;
+                seq->offBase = OFFSET_TO_OFFBASE(cRawOffset);
             }
         }
         /* Compression repcode history is always updated with values directly from the unmodified seqStore.
@@ -3554,10 +3790,11 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
  * Returns the total size of that block (including header) or a ZSTD error code.
  */
 static size_t
-ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore,
+ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
+                            const seqStore_t* const seqStore,
                                   repcodes_t* const dRep, repcodes_t* const cRep,
                                   void* dst, size_t dstCapacity,
-                                  const void* src, size_t srcSize,
+                            const void* src, size_t srcSize,
                                   U32 lastBlock, U32 isPartition)
 {
     const U32 rleMaxLength = 25;
@@ -3631,10 +3868,11 @@ typedef struct {
 
 /* Helper function to perform the recursive search for block splits.
  * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half.
- * If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then
- * we do not recurse.
+ * If advantageous to split, then we recurse down the two sub-blocks.
+ * If not, or if an error occurred in estimation, then we do not recurse.
  *
- * Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING.
+ * Note: The recursion depth is capped by a heuristic minimum number of sequences,
+ * defined by MIN_SEQUENCES_BLOCK_SPLITTING.
  * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING).
  * In practice, recursion depth usually doesn't go beyond 4.
  *
@@ -3646,19 +3884,20 @@ static void
 ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
                              ZSTD_CCtx* zc, const seqStore_t* origSeqStore)
 {
-    seqStore_t* fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
-    seqStore_t* firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
-    seqStore_t* secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
+    seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
+    seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
+    seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
     size_t estimatedOriginalSize;
     size_t estimatedFirstHalfSize;
     size_t estimatedSecondHalfSize;
     size_t midIdx = (startIdx + endIdx)/2;
 
+    DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
+    assert(endIdx >= startIdx);
     if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) {
-        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences");
+        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx);
         return;
     }
-    DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx);
     ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx);
@@ -3679,15 +3918,16 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end
     }
 }
 
-/* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio.
+/* Base recursive function.
+ * Populates a table with intra-block partition indices that can improve compression ratio.
  *
- * Returns the number of splits made (which equals the size of the partition table - 1).
+ * @return: number of splits made (which equals the size of the partition table - 1).
  */
 static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
 {
     seqStoreSplits splits = {partitions, 0};
     if (nbSeq <= 4) {
-        DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split");
+        DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq);
         /* Refuse to try and split anything with less than 4 sequences */
         return 0;
     }
@@ -3703,18 +3943,20 @@ static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
  * Returns combined size of all blocks (which includes headers), or a ZSTD error code.
  */
 static size_t
-ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity,
-                                       const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq)
+ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
+                                    void* dst, size_t dstCapacity,
+                              const void* src, size_t blockSize,
+                                    U32 lastBlock, U32 nbSeq)
 {
     size_t cSize = 0;
     const BYTE* ip = (const BYTE*)src;
     BYTE* op = (BYTE*)dst;
     size_t i = 0;
     size_t srcBytesTotal = 0;
-    U32* partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
-    seqStore_t* nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
-    seqStore_t* currSeqStore = &zc->blockSplitCtx.currSeqStore;
-    size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
+    U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
+    seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
+    seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
+    size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
 
     /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
      * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
@@ -3736,19 +3978,21 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
     ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
     ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t));
 
-    DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+    DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
                 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
                 (unsigned)zc->blockState.matchState.nextToUpdate);
 
     if (numSplits == 0) {
-        size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
-                                                                   &dRep, &cRep,
-                                                                    op, dstCapacity,
-                                                                    ip, blockSize,
-                                                                    lastBlock, 0 /* isPartition */);
+        size_t cSizeSingleBlock =
+            ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
+                                            &dRep, &cRep,
+                                            op, dstCapacity,
+                                            ip, blockSize,
+                                            lastBlock, 0 /* isPartition */);
         FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
         DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
-        assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX);
+        assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize);
         return cSizeSingleBlock;
     }
 
@@ -3773,7 +4017,8 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
                                                        op, dstCapacity,
                                                        ip, srcBytes,
                                                        lastBlockEntireSrc, 1 /* isPartition */);
-        DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
+        DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size",
+                    ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
         FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
 
         ip += srcBytes;
@@ -3781,10 +4026,10 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
         dstCapacity -= cSizeChunk;
         cSize += cSizeChunk;
         *currSeqStore = *nextSeqStore;
-        assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize);
     }
-    /* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes
-     * for the next block.
+    /* cRep and dRep may have diverged during the compression.
+     * If so, we use the dRep repcodes for the next block.
      */
     ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
     return cSize;
@@ -3795,8 +4040,6 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
                               const void* src, size_t srcSize, U32 lastBlock)
 {
-    const BYTE* ip = (const BYTE*)src;
-    BYTE* op = (BYTE*)dst;
     U32 nbSeq;
     size_t cSize;
     DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
@@ -3807,7 +4050,7 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
         if (bss == ZSTDbss_noCompress) {
             if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
                 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
-            cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
+            cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
             FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
             DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
             return cSize;
@@ -3919,10 +4162,11 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
          *   * cSize >= blockBound(srcSize): We have expanded the block too much so
          *     emit an uncompressed block.
          */
-        {
-            size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
+        {   size_t const cSize =
+                ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
             if (cSize != ERROR(dstSize_tooSmall)) {
-                size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
+                size_t const maxCSize =
+                    srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
                 FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
                 if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
                     ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
@@ -3930,7 +4174,7 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
                 }
             }
         }
-    }
+    } /* if (bss == ZSTDbss_compress)*/
 
     DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
     /* Superblock compression failed, attempt to emit a single no compress block.
@@ -3988,7 +4232,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
 *   All blocks will be terminated, all input will be consumed.
 *   Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
 *   Frame is supposed already started (header already produced)
-*   @return : compressed size, or an error code
+*  @return : compressed size, or an error code
 */
 static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                                      void* dst, size_t dstCapacity,
@@ -4053,7 +4297,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                     MEM_writeLE24(op, cBlockHeader);
                     cSize += ZSTD_blockHeaderSize;
                 }
-            }
+            }  /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
 
 
             ip += blockSize;
@@ -5682,6 +5926,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, &params.cParams);
     params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
+    params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
 
 #ifdef ZSTD_MULTITHREAD
     if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
@@ -5883,12 +6128,6 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
     }
 }
 
-typedef struct {
-    U32 idx;            /* Index in array of ZSTD_Sequence */
-    U32 posInSequence;  /* Position within sequence at idx */
-    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
-} ZSTD_sequencePosition;
-
 /* ZSTD_validateSequence() :
  * @offCode : is presumed to follow format required by ZSTD_storeSeq()
  * @returns a ZSTD error code if sequence is not valid
@@ -5926,10 +6165,7 @@ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32
     return offBase;
 }
 
-/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
- * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                               ZSTD_sequencePosition* seqPos,
                                         const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
@@ -5983,19 +6219,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
     return 0;
 }
 
-/* Returns the number of bytes to move the current read position back by.
- * Only non-zero if we ended up splitting a sequence.
- * Otherwise, it may return a ZSTD error if something went wrong.
- *
- * This function will attempt to scan through blockSize bytes
- * represented by the sequences in @inSeqs,
- * storing any (partial) sequences.
- *
- * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
- * avoid splitting a match, or to avoid splitting a match such that it would produce a match
- * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                    const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
                                    const void* src, size_t blockSize)
@@ -6528,3 +6752,19 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
     if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
     return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
 }
+
+void ZSTD_registerExternalMatchFinder(
+    ZSTD_CCtx* zc, void* mState,
+    ZSTD_externalMatchFinder_F* mFinder
+) {
+    ZSTD_externalMatchCtx emctx = {
+        mState,
+        mFinder,
+
+        /* seqBuffer is allocated later (from the cwskp) */
+        NULL, /* seqBuffer */
+        0 /* seqBufferCapacity */
+    };
+    zc->externalMatchCtx = emctx;
+    zc->requestedParams.useExternalMatchFinder = 1;
+}
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index baa726f7d..300c2707d 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -118,12 +118,13 @@ typedef struct {
 /** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  @return : 0 on success or error code */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize);
+size_t ZSTD_buildBlockEntropyStats(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                          void* workspace, size_t wkspSize);
 
 /*********************************
 *  Compression internals structs *
@@ -149,6 +150,12 @@ typedef struct {
   size_t capacity;      /* The capacity starting from `seq` pointer */
 } rawSeqStore_t;
 
+typedef struct {
+    U32 idx;            /* Index in array of ZSTD_Sequence */
+    U32 posInSequence;  /* Position within sequence at idx */
+    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
+} ZSTD_sequencePosition;
+
 UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
 
 typedef struct {
@@ -339,6 +346,18 @@ struct ZSTD_CCtx_params_s {
 
     /* Controls prefetching in some dictMatchState matchfinders */
     ZSTD_paramSwitch_e prefetchCDictTables;
+
+    /* Controls whether zstd will fall back to an internal matchfinder
+     * if the external matchfinder returns an error code. */
+    int enableMatchFinderFallback;
+
+    /* Indicates whether an external matchfinder has been referenced.
+     * Users can't set this externally.
+     * It is set internally in ZSTD_registerExternalMatchFinder(). */
+    int useExternalMatchFinder;
+
+    /* Adjust the max block size*/
+    size_t maxBlockSize;
 };  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
 
 #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
@@ -370,6 +389,14 @@ typedef struct {
     ZSTD_entropyCTablesMetadata_t entropyMetadata;
 } ZSTD_blockSplitCtx;
 
+/* Context for block-level external matchfinder API */
+typedef struct {
+  void* mState;
+  ZSTD_externalMatchFinder_F* mFinder;
+  ZSTD_Sequence* seqBuffer;
+  size_t seqBufferCapacity;
+} ZSTD_externalMatchCtx;
+
 struct ZSTD_CCtx_s {
     ZSTD_compressionStage_e stage;
     int cParamsChanged;                  /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
@@ -439,6 +466,9 @@ struct ZSTD_CCtx_s {
 
     /* Workspace for block splitter */
     ZSTD_blockSplitCtx blockSplitCtx;
+
+    /* Workspace for external matchfinder */
+    ZSTD_externalMatchCtx externalMatchCtx;
 };
 
 typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
@@ -1410,4 +1440,31 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
  */
 void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
 
+/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
+ * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
+ * Note that the block delimiter must include the last literals of the block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+                                              ZSTD_sequencePosition* seqPos,
+                                        const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                        const void* src, size_t blockSize);
+
+/* Returns the number of bytes to move the current read position back by.
+ * Only non-zero if we ended up splitting a sequence.
+ * Otherwise, it may return a ZSTD error if something went wrong.
+ *
+ * This function will attempt to scan through blockSize bytes
+ * represented by the sequences in @inSeqs,
+ * storing any (partial) sequences.
+ *
+ * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
+ * avoid splitting a match, or to avoid splitting a match such that it would produce a match
+ * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+                                   const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                   const void* src, size_t blockSize);
+
 #endif /* ZSTD_COMPRESS_H */
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index ea80a45c8..666e5315d 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -92,16 +92,37 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
     return flSize+1;
 }
 
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
-                        const void* src, size_t srcSize,
-                              void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible, HUF_depth_mode depthMode)
+/* ZSTD_minLiteralsToCompress() :
+ * returns minimal amount of literals
+ * for literal compression to even be attempted.
+ * Minimum is made tighter as compression strategy increases.
+ */
+static size_t
+ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
+{
+    assert((int)strategy >= 0);
+    assert((int)strategy <= 9);
+    /* btultra2 : min 8 bytes;
+     * then 2x larger for each successive compression strategy
+     * max threshold 64 bytes */
+    {   int const shift = MIN(9-strategy, 3);
+        size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : 8 << shift;
+        DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
+        return mintc;
+    }
+}
+
+size_t ZSTD_compressLiterals (
+                  void* dst, size_t dstCapacity,
+            const void* src, size_t srcSize,
+                  void* entropyWorkspace, size_t entropyWorkspaceSize,
+            const ZSTD_hufCTables_t* prevHuf,
+                  ZSTD_hufCTables_t* nextHuf,
+                  ZSTD_strategy strategy,
+                  int disableLiteralCompression,
+                  int suspectUncompressible,
+                  int bmi2)
 {
-    size_t const minGain = ZSTD_minGain(srcSize, strategy);
     size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
     BYTE*  const ostart = (BYTE*)dst;
     U32 singleStream = srcSize < 256;
@@ -119,15 +140,14 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     if (disableLiteralCompression)
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
-    /* small ? don't even attempt compression (speed opt) */
-#   define COMPRESS_LITERALS_SIZE_MIN 63
-    {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
-        if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    /* if too small, don't even attempt compression (speed opt) */
+    if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
+        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
     RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
     {   HUF_repeat repeat = prevHuf->repeatMode;
         int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
+        HUF_depth_mode const depthMode = (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD) ? HUF_depth_optimal : HUF_depth_fast;
         typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode);
         huf_compress_f huf_compress;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
@@ -146,10 +166,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
         }
     }
 
-    if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
-        ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    {   size_t const minGain = ZSTD_minGain(srcSize, strategy);
+        if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+    }   }
     if (cLitSize==1) {
         ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
         return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
@@ -164,16 +185,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     switch(lhSize)
     {
     case 3: /* 2 - 2 - 10 - 10 */
+        if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
             MEM_writeLE24(ostart, lhc);
             break;
         }
     case 4: /* 2 - 2 - 14 - 14 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
             MEM_writeLE32(ostart, lhc);
             break;
         }
     case 5: /* 2 - 2 - 18 - 18 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
             MEM_writeLE32(ostart, lhc);
             ostart[4] = (BYTE)(cLitSize >> 10);
diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h
index bb260db9b..9eb74729d 100644
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,14 +18,18 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
 
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
-/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
+/* ZSTD_compressLiterals():
+ * @entropyWorkspace: must be aligned on 4-bytes boundaries
+ * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
+ * @suspectUncompressible: sampling checks, to potentially skip huffman coding
+ */
+size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
                         const void* src, size_t srcSize,
                               void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible, HUF_depth_mode depthMode);
+                        const ZSTD_hufCTables_t* prevHuf,
+                              ZSTD_hufCTables_t* nextHuf,
+                              ZSTD_strategy strategy, int disableLiteralCompression,
+                              int suspectUncompressible,
+                              int bmi2);
 
 #endif /* ZSTD_COMPRESS_LITERALS_H */
diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c
index 2c1eee567..8872d4d35 100644
--- a/lib/compress/zstd_compress_sequences.c
+++ b/lib/compress/zstd_compress_sequences.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.h b/lib/compress/zstd_compress_sequences.h
index 7991364c2..4a3a05da9 100644
--- a/lib/compress/zstd_compress_sequences.h
+++ b/lib/compress/zstd_compress_sequences.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index eed58e7cf..1971ec646 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.h b/lib/compress/zstd_compress_superblock.h
index 176f9b106..8e494f0d5 100644
--- a/lib/compress/zstd_compress_superblock.h
+++ b/lib/compress/zstd_compress_superblock.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 47afe3dc7..70a359a27 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index c2dbd54c1..0ad88ffc7 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h
index 6d8ee8c65..6f0047c4b 100644
--- a/lib/compress/zstd_double_fast.h
+++ b/lib/compress/zstd_double_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 291173449..5f2c6a2ed 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h
index 3bfeb2c5f..9e4236b47 100644
--- a/lib/compress/zstd_fast.h
+++ b/lib/compress/zstd_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index d5a7b5cbf..810bf011c 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index 150f7b390..c24f1c794 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index c14c62454..3d74ff19e 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 4e68dbf52..f147021d2 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm_geartab.h b/lib/compress/zstd_ldm_geartab.h
index 647f865be..ef34bc5c9 100644
--- a/lib/compress/zstd_ldm_geartab.h
+++ b/lib/compress/zstd_ldm_geartab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 800f87e9e..68d24aa7b 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,7 +16,7 @@
 #define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
 #define ZSTD_MAX_PRICE     (1<<30)
 
-#define ZSTD_PREDEF_THRESHOLD 1024   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
+#define ZSTD_PREDEF_THRESHOLD 8   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
 
 
 /*-*************************************
@@ -26,27 +26,35 @@
 #if 0    /* approximation at bit level (for tests) */
 #  define BITCOST_ACCURACY 0
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat))
+#  define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat))
 #elif 0  /* fractional bit accuracy (for tests) */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat))
+#  define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat))
 #else    /* opt==approx, ultra==accurate */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
+#  define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
 #endif
 
+/* ZSTD_bitWeight() :
+ * provide estimated "cost" of a stat in full bits only */
 MEM_STATIC U32 ZSTD_bitWeight(U32 stat)
 {
     return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER);
 }
 
+/* ZSTD_fracWeight() :
+ * provide fractional-bit "cost" of a stat,
+ * using linear interpolation approximation */
 MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
 {
     U32 const stat = rawStat + 1;
     U32 const hb = ZSTD_highbit32(stat);
     U32 const BWeight = hb * BITCOST_MULTIPLIER;
+    /* Fweight was meant for "Fractional weight"
+     * but it's effectively a value between 1 and 2
+     * using fixed point arithmetic */
     U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
     U32 const weight = BWeight + FWeight;
     assert(hb + BITCOST_ACCURACY < 31);
@@ -88,20 +96,26 @@ static U32 sum_u32(const unsigned table[], size_t nbElts)
     return total;
 }
 
-static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift)
+typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e;
+
+static U32
+ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1)
 {
     U32 s, sum=0;
-    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift);
+    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
+            (unsigned)lastEltIndex+1, (unsigned)shift );
     assert(shift < 30);
     for (s=0; s<lastEltIndex+1; s++) {
-        table[s] = 1 + (table[s] >> shift);
-        sum += table[s];
+        unsigned const base = base1 ? 1 : (table[s]>0);
+        unsigned const newStat = base + (table[s] >> shift);
+        sum += newStat;
+        table[s] = newStat;
     }
     return sum;
 }
 
 /* ZSTD_scaleStats() :
- * reduce all elements in table is sum too large
+ * reduce all elt frequencies in table if sum too large
  * return the resulting sum of elements */
 static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
 {
@@ -110,7 +124,7 @@ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
     DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
     assert(logTarget < 30);
     if (factor <= 1) return prevsum;
-    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor));
+    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed);
 }
 
 /* ZSTD_rescaleFreqs() :
@@ -129,18 +143,22 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
     DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
     optPtr->priceType = zop_dynamic;
 
-    if (optPtr->litLengthSum == 0) {  /* first block : init */
-        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {  /* heuristic */
-            DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
+    if (optPtr->litLengthSum == 0) {  /* no literals stats collected -> first block assumed -> init */
+
+        /* heuristic: use pre-defined stats for too small inputs */
+        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {
+            DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD);
             optPtr->priceType = zop_predef;
         }
 
         assert(optPtr->symbolCosts != NULL);
         if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
-            /* huffman table presumed generated by dictionary */
+
+            /* huffman stats covering the full value set : table presumed generated by dictionary */
             optPtr->priceType = zop_dynamic;
 
             if (compressedLiterals) {
+                /* generate literals statistics from huffman table */
                 unsigned lit;
                 assert(optPtr->litFreq != NULL);
                 optPtr->litSum = 0;
@@ -188,13 +206,14 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                     optPtr->offCodeSum += optPtr->offCodeFreq[of];
             }   }
 
-        } else {  /* not a dictionary */
+        } else {  /* first block, no dictionary */
 
             assert(optPtr->litFreq != NULL);
             if (compressedLiterals) {
+                /* base initial cost of literals on direct frequency within src */
                 unsigned lit = MaxLit;
                 HIST_count_simple(optPtr->litFreq, &lit, src, srcSize);   /* use raw first block to init statistics */
-                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8);
+                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
             }
 
             {   unsigned const baseLLfreqs[MaxLL+1] = {
@@ -224,10 +243,9 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                 optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
             }
 
-
         }
 
-    } else {   /* new block : re-use previous statistics, scaled down */
+    } else {   /* new block : scale down accumulated statistics */
 
         if (compressedLiterals)
             optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
@@ -275,10 +293,11 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
     assert(litLength <= ZSTD_BLOCKSIZE_MAX);
     if (optPtr->priceType == zop_predef)
         return WEIGHT(litLength, optLevel);
-    /* We can't compute the litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
-     * because it isn't representable in the zstd format. So instead just
-     * call it 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. In this case the block
-     * would be all literals.
+
+    /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
+     * because it isn't representable in the zstd format.
+     * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1.
+     * In such a case, the block would be all literals.
      */
     if (litLength == ZSTD_BLOCKSIZE_MAX)
         return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel);
@@ -292,7 +311,7 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
 }
 
 /* ZSTD_getMatchPrice() :
- * Provides the cost of the match part (offset + matchLength) of a sequence
+ * Provides the cost of the match part (offset + matchLength) of a sequence.
  * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
  * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
  * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
@@ -308,8 +327,9 @@ ZSTD_getMatchPrice(U32 const offBase,
     U32 const mlBase = matchLength - MINMATCH;
     assert(matchLength >= MINMATCH);
 
-    if (optPtr->priceType == zop_predef)  /* fixed scheme, do not use statistics */
-        return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER);
+    if (optPtr->priceType == zop_predef)  /* fixed scheme, does not use statistics */
+        return WEIGHT(mlBase, optLevel)
+             + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */
 
     /* dynamic statistics */
     price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
@@ -347,7 +367,7 @@ static void ZSTD_updateStats(optState_t* const optPtr,
         optPtr->litLengthSum++;
     }
 
-    /* offset code : expected to follow storeSeq() numeric representation */
+    /* offset code : follows storeSeq() numeric representation */
     {   U32 const offCode = ZSTD_highbit32(offBase);
         assert(offCode <= MaxOff);
         optPtr->offCodeFreq[offCode]++;
@@ -1352,7 +1372,7 @@ size_t ZSTD_compressBlock_btopt(
 /* ZSTD_initStats_ultra():
  * make a first compression pass, just to seed stats with more accurate starting values.
  * only works on first block, with no dictionary and no ldm.
- * this function cannot error, hence its contract must be respected.
+ * this function cannot error out, its narrow contract must be respected.
  */
 static void
 ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
@@ -1371,7 +1391,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
 
     ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict);   /* generate stats into ms->opt*/
 
-    /* invalidate first scan from history */
+    /* invalidate first scan from history, only keep entropy stats */
     ZSTD_resetSeqStore(seqStore);
     ms->window.base -= srcSize;
     ms->window.dictLimit += (U32)srcSize;
@@ -1395,20 +1415,20 @@ size_t ZSTD_compressBlock_btultra2(
     U32 const curr = (U32)((const BYTE*)src - ms->window.base);
     DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
 
-    /* 2-pass strategy:
+    /* 2-passes strategy:
      * this strategy makes a first pass over first block to collect statistics
-     * and seed next round's statistics with it.
-     * After 1st pass, function forgets everything, and starts a new block.
+     * in order to seed next round's statistics with it.
+     * After 1st pass, function forgets history, and starts a new block.
      * Consequently, this can only work if no data has been previously loaded in tables,
      * aka, no dictionary, no prefix, no ldm preprocessing.
      * The compression ratio gain is generally small (~0.5% on first block),
-     * the cost is 2x cpu time on first block. */
+    ** the cost is 2x cpu time on first block. */
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
     if ( (ms->opt.litLengthSum==0)   /* first block */
       && (seqStore->sequences == seqStore->sequencesStart)  /* no ldm */
       && (ms->window.dictLimit == ms->window.lowLimit)   /* no dictionary */
-      && (curr == ms->window.dictLimit)   /* start of frame, nothing already loaded nor skipped */
-      && (srcSize > ZSTD_PREDEF_THRESHOLD)
+      && (curr == ms->window.dictLimit)    /* start of frame, nothing already loaded nor skipped */
+      && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */
       ) {
         ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
     }
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 627255f53..342e5a311 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 0c10eb603..7a2c71720 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 271eb1ac7..ed4dc0e99 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index c6fd92860..e5c35dae9 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -289,10 +289,11 @@ typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1;   /* single-symbol decodi
 static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
     U64 D4;
     if (MEM_isLittleEndian()) {
-        D4 = (symbol << 8) + nbBits;
+        D4 = (U64)((symbol << 8) + nbBits);
     } else {
-        D4 = symbol + (nbBits << 8);
+        D4 = (U64)(symbol + (nbBits << 8));
     }
+    assert(D4 < (1U << 16));
     D4 *= 0x0001000100010001ULL;
     return D4;
 }
@@ -383,9 +384,8 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * rankStart[0] is not filled because there are no entries in the table for
      * weight 0.
      */
-    {
-        int n;
-        int nextRankStart = 0;
+    {   int n;
+        U32 nextRankStart = 0;
         int const unroll = 4;
         int const nLimit = (int)nbSymbols - unroll + 1;
         for (n=0; n<(int)tableLog+1; n++) {
@@ -412,10 +412,9 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * We can switch based on the length to a different inner loop which is
      * optimized for that particular case.
      */
-    {
-        U32 w;
-        int symbol=wksp->rankVal[0];
-        int rankStart=0;
+    {   U32 w;
+        int symbol = wksp->rankVal[0];
+        int rankStart = 0;
         for (w=1; w<tableLog+1; ++w) {
             int const symbolCount = wksp->rankVal[w];
             int const length = (1 << w) >> 1;
@@ -525,7 +524,7 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons
     while (p < pEnd)
         HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
 
-    return pEnd-pStart;
+    return (size_t)(pEnd-pStart);
 }
 
 FORCE_INLINE_TEMPLATE size_t
@@ -551,6 +550,10 @@ HUF_decompress1X1_usingDTable_internal_body(
     return dstSize;
 }
 
+/* HUF_decompress4X1_usingDTable_internal_body():
+ * Conditions :
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X1_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -594,6 +597,7 @@ HUF_decompress4X1_usingDTable_internal_body(
 
         if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
         if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
@@ -679,8 +683,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
     HUF_DecompressAsmArgs args;
-    {
-        size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+    {   size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
         FORWARD_IF_ERROR(ret, "Failed to init asm args");
         if (ret != 0)
             return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
@@ -700,8 +703,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     (void)iend;
 
     /* finish bit streams one by one. */
-    {
-        size_t const segmentSize = (dstSize+3) / 4;
+    {   size_t const segmentSize = (dstSize+3) / 4;
         BYTE* segmentEnd = (BYTE*)dst;
         int i;
         for (i = 0; i < 4; ++i) {
@@ -1246,6 +1248,11 @@ HUF_decompress1X2_usingDTable_internal_body(
     /* decoded size */
     return dstSize;
 }
+
+/* HUF_decompress4X2_usingDTable_internal_body():
+ * Conditions:
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X2_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -1286,8 +1293,9 @@ HUF_decompress4X2_usingDTable_internal_body(
         DTableDesc const dtd = HUF_getDTableDesc(DTable);
         U32 const dtLog = dtd.tableLog;
 
-        if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
-        if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (length4 > cSrcSize) return ERROR(corruption_detected);  /* overflow */
+        if (opStart4 > oend) return ERROR(corruption_detected);     /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
diff --git a/lib/decompress/huf_decompress_amd64.S b/lib/decompress/huf_decompress_amd64.S
index 3f0e5c26c..8d7c7a644 100644
--- a/lib/decompress/huf_decompress_amd64.S
+++ b/lib/decompress/huf_decompress_amd64.S
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index 6ffa35f6e..3647ad0a7 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h
index bd03268b5..c4ca8877a 100644
--- a/lib/decompress/zstd_ddict.h
+++ b/lib/decompress/zstd_ddict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e95b8822f..f00ef3a67 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -785,7 +785,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
         frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
                                         ? zfh.frameContentSize
-                                        : nbBlocks * zfh.blockSizeMax;
+                                        : (unsigned long long)nbBlocks * zfh.blockSizeMax;
         return frameSizeInfo;
     }
 }
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index e1ff21582..853ddeae1 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -166,6 +166,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 }
                 RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
                 RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+                if (!singleStream)
+                    RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
+                        "Not enough literals (%zu) for the 4-streams mode (min %u)",
+                        litSize, MIN_LITERALS_FOR_4_STREAMS);
                 RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
                 RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
                 ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
@@ -181,6 +185,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
                             dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
                     } else {
+                        assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
                         hufSuccess = HUF_decompress4X_usingDTable_bmi2(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
                             dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
@@ -509,7 +514,8 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
                 for (i = 8; i < n; i += 8) {
                     MEM_write64(spread + pos + i, sv);
                 }
-                pos += n;
+                assert(n>=0);
+                pos += (size_t)n;
             }
         }
         /* Now we spread those positions across the table.
@@ -2010,12 +2016,20 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
      * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
      * We don't expect that to be the case in 64-bit mode.
      * In block mode, window size is not known, so we have to be conservative.
-     * (note: but it could be evaluated from current-lowLimit)
+     * (note: it could possibly be evaluated from current-lowLimit)
      */
     ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
     DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
 
-    RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
+    /* Note : the wording of the specification
+     * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX.
+     * This generally does not happen, as it makes little sense,
+     * since an uncompressed block would feature same size and have no decompression cost.
+     * Also, note that decoder from reference libzstd before < v1.5.4
+     * would consider this edge case as an error.
+     * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX
+     * for broader compatibility with the deployed ecosystem of zstd decoders */
+    RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
 
     /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
diff --git a/lib/decompress/zstd_decompress_block.h b/lib/decompress/zstd_decompress_block.h
index c61a9d0c4..67791dbc3 100644
--- a/lib/decompress/zstd_decompress_block.h
+++ b/lib/decompress/zstd_decompress_block.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 91e9dceb5..32685f2d3 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index b83ea0fed..a968245b3 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_common.c b/lib/deprecated/zbuff_common.c
index e7d01a081..5a2f2db35 100644
--- a/lib/deprecated/zbuff_common.c
+++ b/lib/deprecated/zbuff_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_compress.c b/lib/deprecated/zbuff_compress.c
index 51cf158c4..1d8682150 100644
--- a/lib/deprecated/zbuff_compress.c
+++ b/lib/deprecated/zbuff_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
index d73c0f35f..51159ef53 100644
--- a/lib/deprecated/zbuff_decompress.c
+++ b/lib/deprecated/zbuff_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 724675d30..18c8ec538 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -647,7 +647,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
 
 void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
 {
-  const double ratio = (double)nbDmers / maxDictSize;
+  const double ratio = (double)nbDmers / (double)maxDictSize;
   if (ratio >= 10) {
       return;
   }
@@ -1040,7 +1040,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
       return COVER_dictSelectionError(totalCompressedSize);
     }
 
-    if (totalCompressedSize <= largestCompressed * regressionTolerance) {
+    if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) {
       COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize };
       free(largestDictbuffer);
       return selection;
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index 1aacdddd6..252624bde 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 63a2cee72..46bba0120 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index a932276ff..f41d505f8 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -373,7 +373,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
             elt = table[u];
             /* sort : improve rank */
             while ((u>1) && (table[u-1].savings < elt.savings))
-            table[u] = table[u-1], u--;
+                table[u] = table[u-1], u--;
             table[u] = elt;
             return u;
     }   }
@@ -524,7 +524,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
             if (solution.length==0) { cursor++; continue; }
             ZDICT_insertDictItem(dictList, dictListSize, solution, buffer);
             cursor += solution.length;
-            DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100);
+            DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0);
     }   }
 
 _cleanup:
diff --git a/lib/dll/example/Makefile b/lib/dll/example/Makefile
index 03b034dd5..86cf6906e 100644
--- a/lib/dll/example/Makefile
+++ b/lib/dll/example/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index a6f1174b8..9f53d4cbd 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index c9e4f632f..8c504c792 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index f777eb6e4..6ac876954 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index d80481784..dfaed7bb8 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 1b371953b..dab0260ee 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 20ca5681a..5b1fd7175 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 7a00d4304..9bf3cce64 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 4f7eb9277..23735443a 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 66b97ab8e..640240d62 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 9e73a5daa..5d2ebb2b5 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index bd423bfc1..2dcffc923 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 3839a5f63..53aaec468 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index 9e32b76e0..633891010 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index efe6fbefe..c2b88eb3e 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index bc35cfa6a..1ff39041f 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/libzstd.mk b/lib/libzstd.mk
index c97db5628..5e11d5d29 100644
--- a/lib/libzstd.mk
+++ b/lib/libzstd.mk
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -15,17 +15,34 @@
 # Zstd lib directory
 LIBZSTD ?= ./
 
+# ZSTD_LIB_MINIFY is a helper variable that
+# configures a bunch of other variables to space-optimized defaults.
+ZSTD_LIB_MINIFY ?= 0
+
 # Legacy support
-ZSTD_LEGACY_SUPPORT ?= 5
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  ZSTD_LEGACY_SUPPORT ?= 0
+else
+  ZSTD_LEGACY_SUPPORT ?= 5
+endif
 ZSTD_LEGACY_MULTITHREADED_API ?= 0
 
 # Build size optimizations
-HUF_FORCE_DECOMPRESS_X1 ?= 0
-HUF_FORCE_DECOMPRESS_X2 ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
-ZSTD_NO_INLINE ?= 0
-ZSTD_STRIP_ERROR_STRINGS ?= 0
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  HUF_FORCE_DECOMPRESS_X1 ?= 1
+  HUF_FORCE_DECOMPRESS_X2 ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
+  ZSTD_NO_INLINE ?= 1
+  ZSTD_STRIP_ERROR_STRINGS ?= 1
+else
+  HUF_FORCE_DECOMPRESS_X1 ?= 0
+  HUF_FORCE_DECOMPRESS_X2 ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
+  ZSTD_NO_INLINE ?= 0
+  ZSTD_STRIP_ERROR_STRINGS ?= 0
+endif
 
 # Assembly support
 ZSTD_NO_ASM ?= 0
@@ -61,17 +78,8 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
 CCVER := $(shell $(CC) --version)
 ZSTD_VERSION?= $(LIBVER)
 
-# ZSTD_LIB_MINIFY is a helper variable that
-# configures a bunch of other variables to space-optimized defaults.
-ZSTD_LIB_MINIFY ?= 0
 ifneq ($(ZSTD_LIB_MINIFY), 0)
   HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
-  ZSTD_LEGACY_SUPPORT ?= 0
-  ZSTD_LIB_DEPRECATED ?= 0
-  HUF_FORCE_DECOMPRESS_X1 ?= 1
-  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
-  ZSTD_NO_INLINE ?= 1
-  ZSTD_STRIP_ERROR_STRINGS ?= 1
 ifneq ($(HAVE_CC_OZ), 0)
     # Some compilers (clang) support an even more space-optimized setting.
     CFLAGS += -Oz
diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in
index 33184297c..d5cc0270c 100644
--- a/lib/libzstd.pc.in
+++ b/lib/libzstd.pc.in
@@ -1,5 +1,5 @@
 #   ZSTD - standard compression algorithm
-#   Copyright (C) 2014-2016, Yann Collet, Facebook
+#   Copyright (c) Meta Platforms, Inc. and affiliates.
 #   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
 prefix=@PREFIX@
diff --git a/lib/module.modulemap b/lib/module.modulemap
index bbb939782..eff98dfac 100644
--- a/lib/module.modulemap
+++ b/lib/module.modulemap
@@ -1,17 +1,27 @@
 module libzstd [extern_c] {
     header "zstd.h"
     export *
-    config_macros [exhaustive] /* zstd.h */ \
+    config_macros [exhaustive] \
+        /* zstd.h */ \
         ZSTD_STATIC_LINKING_ONLY, \
+        ZSTDLIB_VISIBILITY, \
         ZSTDLIB_VISIBLE, \
+        ZSTDLIB_HIDDEN, \
         ZSTD_DLL_EXPORT, \
         ZSTDLIB_STATIC_API, \
         ZSTD_DISABLE_DEPRECATE_WARNINGS, \
         ZSTD_CLEVEL_DEFAULT, \
-        /* zdict.h */ ZDICT_STATIC_LINKING_ONLY, \
+        /* zdict.h */ \
+        ZDICT_STATIC_LINKING_ONLY, \
+        ZDICTLIB_VISIBLE, \
+        ZDICTLIB_HIDDEN, \
         ZDICTLIB_VISIBILITY, \
+        ZDICTLIB_STATIC_API, \
         ZDICT_DISABLE_DEPRECATE_WARNINGS, \
-        /* zstd_errors.h */ ZSTDERRORLIB_VISIBILITY
+        /* zstd_errors.h */ \
+        ZSTDERRORLIB_VISIBLE, \
+        ZSTDERRORLIB_HIDDEN, \
+        ZSTDERRORLIB_VISIBILITY
 
     module dictbuilder [extern_c] {
         header "zdict.h"
diff --git a/lib/zdict.h b/lib/zdict.h
index 8e21ba0f7..2268f948a 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -8,32 +8,43 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-#ifndef DICTBUILDER_H_001
-#define DICTBUILDER_H_001
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef ZSTD_ZDICT_H
+#define ZSTD_ZDICT_H
 
 /*======  Dependencies  ======*/
 #include <stddef.h>  /* size_t */
 
 
 /* =====   ZDICTLIB_API : control library symbols visibility   ===== */
-#ifndef ZDICTLIB_VISIBILITY
-#  if defined(__GNUC__) && (__GNUC__ >= 4)
-#    define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#ifndef ZDICTLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZDICTLIB_VISIBILITY
+#    define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
 #  else
-#    define ZDICTLIB_VISIBILITY
+#    define ZDICTLIB_VISIBLE
 #  endif
 #endif
+
+#ifndef ZDICTLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZDICTLIB_HIDDEN
+#  endif
+#endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define ZDICTLIB_API ZDICTLIB_VISIBILITY
+#  define ZDICTLIB_API ZDICTLIB_VISIBLE
 #endif
 
 /*******************************************************************************
@@ -201,9 +212,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
                                     const size_t* samplesSizes, unsigned nbSamples);
 
 typedef struct {
-    int      compressionLevel;   /*< optimize for a specific zstd compression level; 0 means default */
-    unsigned notificationLevel;  /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
-    unsigned dictID;             /*< force dictID value; 0 means auto mode (32-bits random value)
+    int      compressionLevel;   /**< optimize for a specific zstd compression level; 0 means default */
+    unsigned notificationLevel;  /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+    unsigned dictID;             /**< force dictID value; 0 means auto mode (32-bits random value)
                                   *   NOTE: The zstd format reserves some dictionary IDs for future use.
                                   *         You may use them in private settings, but be warned that they
                                   *         may be used by zstd in a public dictionary registry in the future.
@@ -260,9 +271,21 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS
 ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
 ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
 
+#endif   /* ZSTD_ZDICT_H */
 
+#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
+#define ZSTD_ZDICT_H_STATIC
 
-#ifdef ZDICT_STATIC_LINKING_ONLY
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZDICTLIB_STATIC_API
+#  if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#    define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
+#  elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#    define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
+#  else
+#    define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
+#  endif
+#endif
 
 /* ====================================================================================
  * The definitions in this section are considered experimental.
@@ -318,7 +341,7 @@ typedef struct {
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
           void *dictBuffer, size_t dictBufferCapacity,
     const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
           ZDICT_cover_params_t parameters);
@@ -340,7 +363,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
  *          See ZDICT_trainFromBuffer() for details on failure modes.
  * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
  */
-ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
           void* dictBuffer, size_t dictBufferCapacity,
     const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
           ZDICT_cover_params_t* parameters);
@@ -361,7 +384,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
                     size_t dictBufferCapacity, const void *samplesBuffer,
                     const size_t *samplesSizes, unsigned nbSamples,
                     ZDICT_fastCover_params_t parameters);
@@ -384,7 +407,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
  *          See ZDICT_trainFromBuffer() for details on failure modes.
  * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
  */
-ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
                     size_t dictBufferCapacity, const void* samplesBuffer,
                     const size_t* samplesSizes, unsigned nbSamples,
                     ZDICT_fastCover_params_t* parameters);
@@ -409,7 +432,7 @@ typedef struct {
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  *  Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
     void* dictBuffer, size_t dictBufferCapacity,
     const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
     ZDICT_legacy_params_t parameters);
@@ -421,32 +444,31 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
    or _CRT_SECURE_NO_WARNINGS in Visual.
    Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
 #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
-#  define ZDICT_DEPRECATED(message) ZDICTLIB_API   /* disable deprecation warnings */
+#  define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
 #else
 #  define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
 #  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
-#    define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
+#    define ZDICT_DEPRECATED(message) [[deprecated(message)]]
 #  elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
+#    define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (ZDICT_GCC_VERSION >= 301)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
+#    define ZDICT_DEPRECATED(message) __attribute__((deprecated))
 #  elif defined(_MSC_VER)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
+#    define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
 #  else
 #    pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API
+#    define ZDICT_DEPRECATED(message)
 #  endif
 #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
 
 ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
+ZDICTLIB_STATIC_API
 size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
                                   const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
 
-#endif   /* ZDICT_STATIC_LINKING_ONLY */
+#endif   /* ZSTD_ZDICT_H_STATIC */
 
 #if defined (__cplusplus)
 }
 #endif
-
-#endif   /* DICTBUILDER_H_001 */
diff --git a/lib/zstd.h b/lib/zstd.h
index 1dff31b4e..c2dfda7bb 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -21,14 +21,24 @@ extern "C" {
 
 /* =====   ZSTDLIB_API : control library symbols visibility   ===== */
 #ifndef ZSTDLIB_VISIBLE
-#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDLIB_VISIBILITY
+#    define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
 #    define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
-#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
 #  else
 #    define ZSTDLIB_VISIBLE
+#  endif
+#endif
+
+#ifndef ZSTDLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
 #    define ZSTDLIB_HIDDEN
 #  endif
 #endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
 #  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
@@ -201,8 +211,30 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
 
 
 /*======  Helper functions  ======*/
-#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
 ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
 ZSTDLIB_API int         ZSTD_minCLevel(void);               /*!< minimum negative compression level allowed, requires v1.4.0+ */
@@ -446,6 +478,8 @@ typedef enum {
      * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableMatchFinderFallback
+     * ZSTD_c_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -465,7 +499,10 @@ typedef enum {
      ZSTD_c_experimentalParam13=1010,
      ZSTD_c_experimentalParam14=1011,
      ZSTD_c_experimentalParam15=1012,
-     ZSTD_c_experimentalParam16=1013
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015,
+
 } ZSTD_cParameter;
 
 typedef struct {
@@ -528,7 +565,7 @@ typedef enum {
  *                  They will be used to compress next frame.
  *                  Resetting session never fails.
  *  - The parameters : changes all parameters back to "default".
- *                  This removes any reference to any dictionary too.
+ *                  This also removes any reference to any dictionary or external matchfinder.
  *                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
  *                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
  *  - Both : similar to resetting the session, followed by resetting parameters.
@@ -990,8 +1027,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
  *           meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
- *           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+ *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ *           until parameters are reset, a new dictionary is loaded, or the dictionary
+ *           is explicitly invalidated by loading a NULL dictionary.
  *  Note 2 : Loading a dictionary involves building tables.
  *           It's also a CPU consuming operation, with non-negligible impact on latency.
  *           Tables are dependent on compression parameters, and for this reason,
@@ -1004,7 +1042,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
 /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
- *  Reference a prepared dictionary, to be used for all next compressed frames.
+ *  Reference a prepared dictionary, to be used for all future compressed frames.
  *  Note that compression parameters are enforced from within CDict,
  *  and supersede any compression parameter previously set within CCtx.
  *  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@@ -1039,9 +1077,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                  const void* prefix, size_t prefixSize);
 
 /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
- *  Create an internal DDict from dict buffer,
- *  to be used to decompress next frames.
- *  The dictionary remains valid for all future frames, until explicitly invalidated.
+ *  Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ *  The dictionary remains valid for all future frames, until explicitly invalidated, or
+ *  a new dictionary is loaded.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
  *            meaning "return to no-dictionary mode".
@@ -1065,9 +1103,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
  *  The memory for the table is allocated on the first call to refDDict, and can be
  *  freed with ZSTD_freeDCtx().
  *
+ *  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ *  will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Note 1 : Currently, only one dictionary can be managed.
- *           Referencing a new dictionary effectively "discards" any previous one.
  *  Special: referencing a NULL DDict means "return to no-dictionary mode".
  *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
  */
@@ -1706,6 +1745,13 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
  *  This function never fails (wide contract) */
 ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
+/*! ZSTD_CCtx_setCParams() :
+ *  Set all parameters provided within @cparams into the working @cctx.
+ *  Note : if modifying parameters during compression (MT mode only),
+ *         note that changes to the .windowLog parameter will be ignored.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
 /*! ZSTD_compress_advanced() :
  *  Note : this function is now DEPRECATED.
  *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
@@ -1713,10 +1759,10 @@ ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressio
 ZSTD_DEPRECATED("use ZSTD_compress2")
 ZSTDLIB_STATIC_API
 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
-                                          void* dst, size_t dstCapacity,
-                                    const void* src, size_t srcSize,
-                                    const void* dict,size_t dictSize,
-                                          ZSTD_parameters params);
+                              void* dst, size_t dstCapacity,
+                        const void* src, size_t srcSize,
+                        const void* dict,size_t dictSize,
+                              ZSTD_parameters params);
 
 /*! ZSTD_compress_usingCDict_advanced() :
  *  Note : this function is now DEPRECATED.
@@ -2010,6 +2056,27 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  */
 #define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
 
+/* ZSTD_c_enableMatchFinderFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal matchfinder if an
+ * external matchfinder is registered and returns an error code. This fallback is
+ * block-by-block: the internal matchfinder will only be called for blocks where
+ * the external matchfinder returns an error code. Fallback compression will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full external matchfinder API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
+
+/*  ZSTD_c_maxBlockSize
+ *
+ *  Default is ZSTD_BLOCKSIZE_MAX.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
 /*! ZSTD_CCtx_getParameter() :
  *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  *  and store it into int* value.
@@ -2476,8 +2543,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLev
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
 
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 
 ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -2501,8 +2568,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
   Data fragment must be large enough to ensure successful decoding.
  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
-  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
-           >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
   It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@@ -2521,7 +2588,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
 
   The most memory efficient way is to use a round buffer of sufficient size.
   Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
-  which can @return an error code if required value is too large for current system (in 32-bits mode).
+  which can return an error code if required value is too large for current system (in 32-bits mode).
   In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
   up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
   which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@@ -2541,7 +2608,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
 
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
   It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
   It can also be an error code, which can be tested with ZSTD_isError().
 
@@ -2642,6 +2709,142 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_
 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
 
 
+/* ********************** EXTERNAL MATCHFINDER API **********************
+ *
+ * *** OVERVIEW ***
+ * This API allows users to replace the zstd internal block-level matchfinder
+ * with an external matchfinder function. Potential applications of the API
+ * include hardware-accelerated matchfinders and matchfinders specialized to
+ * particular types of data.
+ *
+ * See contrib/externalMatchfinder for an example program employing the
+ * external matchfinder API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_externalMatchFinder_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ *   - externalMatchState: a pointer to a user-managed state for the external
+ *     matchfinder.
+ *
+ *   - outSeqs, outSeqsCapacity: an output buffer for sequences produced by the
+ *     external matchfinder. outSeqsCapacity is guaranteed >=
+ *     ZSTD_sequenceBound(srcSize). The memory backing outSeqs is managed by
+ *     the CCtx.
+ *
+ *   - src, srcSize: an input buffer which the external matchfinder must parse
+ *     into sequences. srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ *   - dict, dictSize: a history buffer, which may be empty, which the external
+ *     matchfinder may reference as it produces sequences for the src buffer.
+ *     Currently, zstd will always pass dictSize == 0 into external matchfinders,
+ *     but this will change in the future.
+ *
+ *   - compressionLevel: a signed integer representing the zstd compression level
+ *     set by the user for the current operation. The external matchfinder may
+ *     choose to use this information to change its compression strategy and
+ *     speed/ratio tradeoff. Note: The compression level does not reflect zstd
+ *     parameters set through the advanced API.
+ *
+ *   - windowSize: a size_t representing the maximum allowed offset for external
+ *     sequences. Note that sequence offsets are sometimes allowed to exceed the
+ *     windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ *     for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_EXTERNAL_MATCHFINDER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ *   - The sum of matchLengths and literalLengths is equal to srcSize.
+ *   - All sequences in the parse have matchLength != 0, except for the final
+ *     sequence. matchLength is not constrained for the final sequence.
+ *   - All offsets respect the windowSize parameter as specified in
+ *     doc/zstd_compression_format.md.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal matchfinder or fail the compression operation. The user can
+ * choose between the two behaviors by setting the
+ * ZSTD_c_enableMatchFinderFallback cParam. Fallback compression will follow any
+ * other cParam settings, such as compression level, the same as in a normal
+ * compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_externalMatchFinder_F
+ * function by calling ZSTD_registerExternalMatchFinder(cctx, externalMatchState,
+ * externalMatchFinder). This setting will persist until the next parameter reset
+ * of the CCtx.
+ *
+ * The externalMatchState must be initialized by the user before calling
+ * ZSTD_registerExternalMatchFinder. The user is responsible for destroying the
+ * externalMatchState.
+ *
+ * *** LIMITATIONS ***
+ * External matchfinders are compatible with all zstd compression APIs. There are
+ * only two limitations.
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
+ * external matchfinder.
+ *   - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in
+ *     some cases (see its documentation for details). Users must explicitly set
+ *     ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an
+ *     external matchfinder is registered.
+ *   - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ *     whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ *     check the docs on ZSTD_c_enableLongDistanceMatching whenever the external
+ *     matchfinder API is used in conjunction with advanced settings (like windowLog).
+ *
+ * Second, history buffers are not supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external matchfinder (for now). This has two implications:
+ *   - Dictionaries are not supported. Compression will *not* fail if the user
+ *     references a dictionary, but the dictionary won't have any effect.
+ *   - Stream history is not supported. All compression APIs, including streaming
+ *     APIs, work with the external matchfinder, but the external matchfinder won't
+ *     receive any history from the previous block. Each block is an independent chunk.
+ *
+ * Long-term, we plan to overcome both limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_EXTERNAL_MATCHFINDER_ERROR ((size_t)(-1))
+
+typedef size_t ZSTD_externalMatchFinder_F (
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+/*! ZSTD_registerExternalMatchFinder() :
+ * Instruct zstd to use an external matchfinder function.
+ *
+ * The externalMatchState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * The user is strongly encouraged to read the full API documentation (above)
+ * before calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerExternalMatchFinder(
+  ZSTD_CCtx* cctx,
+  void* externalMatchState,
+  ZSTD_externalMatchFinder_F* externalMatchFinder
+);
+
 #endif   /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
 
 #if defined (__cplusplus)
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 2ec0b0ab1..bd6dbee5f 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -20,19 +20,31 @@ extern "C" {
 
 
 /* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
-#ifndef ZSTDERRORLIB_VISIBILITY
-#  if defined(__GNUC__) && (__GNUC__ >= 4)
-#    define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#ifndef ZSTDERRORLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
 #  else
-#    define ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE
 #  endif
 #endif
+
+#ifndef ZSTDERRORLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDERRORLIB_HIDDEN
+#  endif
+#endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
 #endif
 
 /*-*********************************************
@@ -58,10 +70,12 @@ typedef enum {
   ZSTD_error_frameParameter_windowTooLarge = 16,
   ZSTD_error_corruption_detected = 20,
   ZSTD_error_checksum_wrong      = 22,
+  ZSTD_error_literals_headerWrong = 24,
   ZSTD_error_dictionary_corrupted      = 30,
   ZSTD_error_dictionary_wrong          = 32,
   ZSTD_error_dictionaryCreation_failed = 34,
   ZSTD_error_parameter_unsupported   = 40,
+  ZSTD_error_parameter_combination_unsupported = 41,
   ZSTD_error_parameter_outOfBound    = 42,
   ZSTD_error_tableLog_tooLarge       = 44,
   ZSTD_error_maxSymbolValue_tooLarge = 46,
@@ -79,6 +93,7 @@ typedef enum {
   ZSTD_error_seekableIO          = 102,
   ZSTD_error_dstBuffer_wrong     = 104,
   ZSTD_error_srcBuffer_wrong     = 105,
+  ZSTD_error_externalMatchFinder_failed = 106,
   ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;
 
diff --git a/programs/Makefile b/programs/Makefile
index 1cdec4a64..b79b60b4f 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.c b/programs/benchfn.c
index 1aadbdd91..8e6726f8d 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -229,9 +229,9 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
             cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
 
             /* estimate nbLoops for next run to last approximately 1 second */
-            if (loopDuration_ns > (runBudget_ns / 50)) {
+            if (loopDuration_ns > ((double)runBudget_ns / 50)) {
                 double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
-                cont->nbLoops = (unsigned)(runBudget_ns / fastestRun_ns) + 1;
+                cont->nbLoops = (unsigned)((double)runBudget_ns / fastestRun_ns) + 1;
             } else {
                 /* previous run was too short : blindly increase workload by x multiplier */
                 const unsigned multiplier = 10;
@@ -239,7 +239,7 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
                 cont->nbLoops *= multiplier;
             }
 
-            if(loopDuration_ns < runTimeMin_ns) {
+            if(loopDuration_ns < (double)runTimeMin_ns) {
                 /* don't report results for which benchmark run time was too small : increased risks of rounding errors */
                 assert(completed == 0);
                 continue;
diff --git a/programs/benchfn.h b/programs/benchfn.h
index 99d13ac47..1bd93d135 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 6ceca020c..285e401ef 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 11ac85da7..aa683dfc2 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.c b/programs/datagen.c
index 3b4f9e5c7..ddc690bb1 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.h b/programs/datagen.h
index b76ae2a22..ca7270006 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.c b/programs/dibio.c
index 8643bc378..b21338cd8 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.h b/programs/dibio.h
index 666c1e661..a96104c36 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.c b/programs/fileio.c
index 313a530ce..6bfa559e9 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -1059,12 +1059,12 @@ FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but no
             DISPLAYUPDATE_PROGRESS(
                     "\rRead : %u MB ==> %.2f%% ",
                     (unsigned)(inFileSize>>20),
-                    (double)outFileSize/inFileSize*100)
+                    (double)outFileSize/(double)inFileSize*100)
         } else {
             DISPLAYUPDATE_PROGRESS(
                     "\rRead : %u / %u MB ==> %.2f%% ",
                     (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                    (double)outFileSize/inFileSize*100);
+                    (double)outFileSize/(double)inFileSize*100);
     }   }
 
     while (1) {
@@ -1157,11 +1157,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
         if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
             DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20),
-                            (double)outFileSize/inFileSize*100)
+                            (double)outFileSize/(double)inFileSize*100)
         else
             DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                            (double)outFileSize/inFileSize*100);
+                            (double)outFileSize/(double)inFileSize*100);
         if (ret == LZMA_STREAM_END) break;
     }
 
@@ -1241,11 +1241,11 @@ FIO_compressLz4Frame(cRess_t* ress,
             if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
                 DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20),
-                                (double)outFileSize/inFileSize*100)
+                                (double)outFileSize/(double)inFileSize*100)
             } else {
                 DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                                (double)outFileSize/inFileSize*100);
+                                (double)outFileSize/(double)inFileSize*100);
             }
 
             /* Write Block */
diff --git a/programs/fileio.h b/programs/fileio.h
index b848934bc..e37398ded 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_asyncio.c b/programs/fileio_asyncio.c
index 92c9a5b1d..8f12fe1f9 100644
--- a/programs/fileio_asyncio.c
+++ b/programs/fileio_asyncio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_asyncio.h b/programs/fileio_asyncio.h
index 30db44b6e..34dad6f4d 100644
--- a/programs/fileio_asyncio.h
+++ b/programs/fileio_asyncio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_common.h b/programs/fileio_common.h
index aec2e8d56..827a5a06b 100644
--- a/programs/fileio_common.h
+++ b/programs/fileio_common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio_types.h b/programs/fileio_types.h
index a1fac2ca7..c1f42f1ad 100644
--- a/programs/fileio_types.h
+++ b/programs/fileio_types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/platform.h b/programs/platform.h
index 7006f97f2..18a3587bf 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.c b/programs/timefn.c
index 64577b0e9..7005b2696 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.h b/programs/timefn.h
index 8ba8ed787..438a8de07 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/util.c b/programs/util.c
index bdb651074..21d7d3b46 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -66,6 +66,27 @@ extern "C" {
 #define UTIL_DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
 
+static int g_traceDepth = 0;
+int g_traceFileStat = 0;
+
+#define UTIL_TRACE_CALL(...)                                         \
+    {                                                                \
+        if (g_traceFileStat) {                                       \
+            UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \
+            UTIL_DISPLAY(__VA_ARGS__);                               \
+            UTIL_DISPLAY("\n");                                      \
+            ++g_traceDepth;                                          \
+        }                                                            \
+    }
+
+#define UTIL_TRACE_RET(ret)                                                     \
+    {                                                                           \
+        if (g_traceFileStat) {                                                  \
+            --g_traceDepth;                                                     \
+            UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \
+        }                                                                      \
+    }
+
 /* A modified version of realloc().
  * If UTIL_realloc() fails the original block is freed.
  */
@@ -121,21 +142,34 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
 *  Functions
 ***************************************/
 
+void UTIL_traceFileStat(void)
+{
+    g_traceFileStat = 1;
+}
+
 int UTIL_stat(const char* filename, stat_t* statbuf)
 {
+    int ret;
+    UTIL_TRACE_CALL("UTIL_stat(%s)", filename);
 #if defined(_MSC_VER)
-    return !_stat64(filename, statbuf);
+    ret = !_stat64(filename, statbuf);
 #elif defined(__MINGW32__) && defined (__MSVCRT__)
-    return !_stati64(filename, statbuf);
+    ret = !_stati64(filename, statbuf);
 #else
-    return !stat(filename, statbuf);
+    ret = !stat(filename, statbuf);
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isRegularFile(const char* infilename)
 {
     stat_t statbuf;
-    return UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename);
+    ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isRegularFileStat(const stat_t* statbuf)
@@ -151,44 +185,66 @@ int UTIL_isRegularFileStat(const stat_t* statbuf)
 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
 {
     stat_t localStatBuf;
+    UTIL_TRACE_CALL("UTIL_chmod(%s, %u)", filename, (unsigned)permissions);
     if (statbuf == NULL) {
-        if (!UTIL_stat(filename, &localStatBuf)) return 0;
+        if (!UTIL_stat(filename, &localStatBuf)) {
+            UTIL_TRACE_RET(0);
+            return 0;
+        }
         statbuf = &localStatBuf;
     }
-    if (!UTIL_isRegularFileStat(statbuf)) return 0; /* pretend success, but don't change anything */
-    return chmod(filename, permissions);
+    if (!UTIL_isRegularFileStat(statbuf)) {
+        UTIL_TRACE_RET(0);
+        return 0; /* pretend success, but don't change anything */
+    }
+    UTIL_TRACE_CALL("chmod");
+    {
+        int const ret = chmod(filename, permissions);
+        UTIL_TRACE_RET(ret);
+        UTIL_TRACE_RET(ret);
+        return ret;
+    }
 }
 
 /* set access and modification times */
 int UTIL_utime(const char* filename, const stat_t *statbuf)
 {
     int ret;
+    UTIL_TRACE_CALL("UTIL_utime(%s)", filename);
     /* We check that st_mtime is a macro here in order to give us confidence
      * that struct stat has a struct timespec st_mtim member. We need this
      * check because there are some platforms that claim to be POSIX 2008
      * compliant but which do not have st_mtim... */
 #if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime)
-    /* (atime, mtime) */
-    struct timespec timebuf[2] = { {0, UTIME_NOW} };
-    timebuf[1] = statbuf->st_mtim;
-    ret = utimensat(AT_FDCWD, filename, timebuf, 0);
+    {
+        /* (atime, mtime) */
+        struct timespec timebuf[2] = { {0, UTIME_NOW} };
+        timebuf[1] = statbuf->st_mtim;
+        ret = utimensat(AT_FDCWD, filename, timebuf, 0);
+    }
 #else
-    struct utimbuf timebuf;
-    timebuf.actime = time(NULL);
-    timebuf.modtime = statbuf->st_mtime;
-    ret = utime(filename, &timebuf);
+    {
+        struct utimbuf timebuf;
+        timebuf.actime = time(NULL);
+        timebuf.modtime = statbuf->st_mtime;
+        ret = utime(filename, &timebuf);
+    }
 #endif
     errno = 0;
+    UTIL_TRACE_RET(ret);
     return ret;
 }
 
 int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
 {
     int res = 0;
-
     stat_t curStatBuf;
-    if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf))
+    UTIL_TRACE_CALL("UTIL_setFileStat(%s)", filename);
+
+    if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
+        UTIL_TRACE_RET(-1);
         return -1;
+    }
 
     /* set access and modification times */
     res += UTIL_utime(filename, statbuf);
@@ -200,13 +256,18 @@ int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
     res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 07777);  /* Copy file permissions */
 
     errno = 0;
+    UTIL_TRACE_RET(-res);
     return -res; /* number of errors is returned */
 }
 
 int UTIL_isDirectory(const char* infilename)
 {
     stat_t statbuf;
-    return UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename);
+    ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isDirectoryStat(const stat_t* statbuf)
@@ -224,33 +285,44 @@ int UTIL_compareStr(const void *p1, const void *p2) {
 
 int UTIL_isSameFile(const char* fName1, const char* fName2)
 {
+    int ret;
     assert(fName1 != NULL); assert(fName2 != NULL);
+    UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2);
 #if defined(_MSC_VER) || defined(_WIN32)
     /* note : Visual does not support file identification by inode.
      *        inode does not work on Windows, even with a posix layer, like msys2.
      *        The following work-around is limited to detecting exact name repetition only,
      *        aka `filename` is considered different from `subdir/../filename` */
-    return !strcmp(fName1, fName2);
+    ret = !strcmp(fName1, fName2);
 #else
     {   stat_t file1Stat;
         stat_t file2Stat;
-        return UTIL_stat(fName1, &file1Stat)
+        ret =  UTIL_stat(fName1, &file1Stat)
             && UTIL_stat(fName2, &file2Stat)
             && (file1Stat.st_dev == file2Stat.st_dev)
             && (file1Stat.st_ino == file2Stat.st_ino);
     }
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 /* UTIL_isFIFO : distinguish named pipes */
 int UTIL_isFIFO(const char* infilename)
 {
+    UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename);
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #if PLATFORM_POSIX_VERSION >= 200112L
-    stat_t statbuf;
-    if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) return 1;
+    {
+        stat_t statbuf;
+        if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) {
+            UTIL_TRACE_RET(1);
+            return 1;
+        }
+    }
 #endif
     (void)infilename;
+    UTIL_TRACE_RET(0);
     return 0;
 }
 
@@ -278,13 +350,20 @@ int UTIL_isBlockDevStat(const stat_t* statbuf)
 
 int UTIL_isLink(const char* infilename)
 {
+    UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename);
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #if PLATFORM_POSIX_VERSION >= 200112L
-    stat_t statbuf;
-    int const r = lstat(infilename, &statbuf);
-    if (!r && S_ISLNK(statbuf.st_mode)) return 1;
+    {
+        stat_t statbuf;
+        int const r = lstat(infilename, &statbuf);
+        if (!r && S_ISLNK(statbuf.st_mode)) {
+            UTIL_TRACE_RET(1);
+            return 1;
+        }
+    }
 #endif
     (void)infilename;
+    UTIL_TRACE_RET(0);
     return 0;
 }
 
@@ -294,13 +373,18 @@ static int g_fakeStdoutIsConsole = 0;
 
 int UTIL_isConsole(FILE* file)
 {
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file));
     if (file == stdin && g_fakeStdinIsConsole)
-        return 1;
-    if (file == stderr && g_fakeStderrIsConsole)
-        return 1;
-    if (file == stdout && g_fakeStdoutIsConsole)
-        return 1;
-    return IS_CONSOLE(file);
+        ret = 1;
+    else if (file == stderr && g_fakeStderrIsConsole)
+        ret = 1;
+    else if (file == stdout && g_fakeStdoutIsConsole)
+        ret = 1;
+    else
+        ret = IS_CONSOLE(file);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 void UTIL_fakeStdinIsConsole(void)
@@ -319,8 +403,16 @@ void UTIL_fakeStderrIsConsole(void)
 U64 UTIL_getFileSize(const char* infilename)
 {
     stat_t statbuf;
-    if (!UTIL_stat(infilename, &statbuf)) return UTIL_FILESIZE_UNKNOWN;
-    return UTIL_getFileSizeStat(&statbuf);
+    UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename);
+    if (!UTIL_stat(infilename, &statbuf)) {
+        UTIL_TRACE_RET(-1);
+        return UTIL_FILESIZE_UNKNOWN;
+    }
+    {
+        U64 const size = UTIL_getFileSizeStat(&statbuf);
+        UTIL_TRACE_RET((int)size);
+        return size;
+    }
 }
 
 U64 UTIL_getFileSizeStat(const stat_t* statbuf)
@@ -397,11 +489,16 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
 {
     U64 total = 0;
     unsigned n;
+    UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles);
     for (n=0; n<nbFiles; n++) {
         U64 const size = UTIL_getFileSize(fileNamesTable[n]);
-        if (size == UTIL_FILESIZE_UNKNOWN) return UTIL_FILESIZE_UNKNOWN;
+        if (size == UTIL_FILESIZE_UNKNOWN) {
+            UTIL_TRACE_RET(-1);
+            return UTIL_FILESIZE_UNKNOWN;
+        }
         total += size;
     }
+    UTIL_TRACE_RET((int)total);
     return total;
 }
 
diff --git a/programs/util.h b/programs/util.h
index cf1738772..88ee1bad1 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -189,6 +189,11 @@ void UTIL_fakeStdinIsConsole(void);
 void UTIL_fakeStdoutIsConsole(void);
 void UTIL_fakeStderrIsConsole(void);
 
+/**
+ * Emit traces for functions that read, or modify file metadata.
+ */
+void UTIL_traceFileStat(void);
+
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
 U64 UTIL_getFileSize(const char* infilename);
 U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
@@ -262,7 +267,6 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2);
 /*! UTIL_expandFNT() :
  *  read names from @fnt, and expand those corresponding to directories
  *  update @fnt, now containing only file names,
- * @return : 0 in case of success, 1 if error
  *  note : in case of error, @fnt[0] is NULL
  */
 void UTIL_expandFNT(FileNamesTable** fnt, int followLinks);
diff --git a/programs/windres/verrsrc.h b/programs/windres/verrsrc.h
index c1b60e90f..61b1f3ddc 100644
--- a/programs/windres/verrsrc.h
+++ b/programs/windres/verrsrc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc
index f5e404730..a2118c2df 100644
--- a/programs/windres/zstd.rc
+++ b/programs/windres/zstd.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "CompanyName", "Meta Platforms, Inc."
             VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+            VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/programs/zstd.1 b/programs/zstd.1
index 5cf3c6bb1..e81ac9615 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "August 2022" "zstd 1.5.3" "User Commands"
+.TH "ZSTD" "1" "December 2022" "zstd 1.5.3" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -17,10 +17,10 @@
 \fBzstdcat\fR is equivalent to \fBzstd \-dcf\fR
 .
 .SH "DESCRIPTION"
-\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip (1)\fR and \fBxz (1)\fR\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
+\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip\fR(1) and \fBxz\fR(1)\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
 .
 .P
-\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences :
+\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences:
 .
 .IP "\(bu" 4
 Source files are preserved by default\. It\'s possible to remove them automatically by using the \fB\-\-rm\fR command\.
@@ -34,10 +34,13 @@ When compressing a single file, \fBzstd\fR displays progress notifications and r
 .IP "\(bu" 4
 \fBzstd\fR does not accept input from console, though it does accept \fBstdin\fR when it\'s not the console\.
 .
+.IP "\(bu" 4
+\fBzstd\fR does not store the input\'s filename or attributes, only its contents\.
+.
 .IP "" 0
 .
 .P
-\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal : it will display an error message and skip the \fIfile\fR\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
+\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal: it will display an error message and skip the file\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
 .
 .P
 Unless \fB\-\-stdout\fR or \fB\-o\fR is specified, \fIfiles\fR are written to a new file whose name is derived from the source \fIfile\fR name:
@@ -50,12 +53,12 @@ When decompressing, the \fB\.zst\fR suffix is removed from the source filename t
 .
 .IP "" 0
 .
-.SS "Concatenation with \.zst files"
+.SS "Concatenation with \.zst Files"
 It is possible to concatenate multiple \fB\.zst\fR files\. \fBzstd\fR will decompress such agglomerated file as if it was a single \fB\.zst\fR file\.
 .
 .SH "OPTIONS"
 .
-.SS "Integer suffixes and special values"
+.SS "Integer Suffixes and Special Values"
 In most places where an integer argument is expected, an optional suffix is supported to easily indicate large integers\. There must be no space between the integer and the suffix\.
 .
 .TP
@@ -66,7 +69,7 @@ Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accep
 \fBMiB\fR
 Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
 .
-.SS "Operation mode"
+.SS "Operation Mode"
 If multiple operation mode options are given, the last one takes effect\.
 .
 .TP
@@ -83,20 +86,20 @@ Test the integrity of compressed \fIfiles\fR\. This option is equivalent to \fB\
 .
 .TP
 \fB\-b#\fR
-Benchmark file(s) using compression level #
+Benchmark file(s) using compression level \fI#\fR\. See \fIBENCHMARK\fR below for a description of this operation\.
 .
 .TP
-\fB\-\-train FILEs\fR
-Use FILEs as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\.
+\fB\-\-train FILES\fR
+Use \fIFILES\fR as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\. See \fIDICTIONARY BUILDER\fR below for a description of this operation\.
 .
 .TP
 \fB\-l\fR, \fB\-\-list\fR
 Display information related to a zstd compressed file, such as size, ratio, and checksum\. Some of these fields may not be available\. This command\'s output can be augmented with the \fB\-v\fR modifier\.
 .
-.SS "Operation modifiers"
+.SS "Operation Modifiers"
 .
 .IP "\(bu" 4
-\fB\-#\fR: \fB#\fR compression level [1\-19] (default: 3)
+\fB\-#\fR: selects \fB#\fR compression level [1\-19] (default: 3)
 .
 .IP "\(bu" 4
 \fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
@@ -108,13 +111,22 @@ Display information related to a zstd compressed file, such as size, ratio, and
 \fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to \fBZSTDMT_NBWORKERS_MAX\fR, which is either 64 in 32\-bit mode, or 256 for 64\-bit environments\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
 .
 .IP "\(bu" 4
-\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\. Note 1 : this mode is the only available one when multithread support is disabled\. Note 2 : this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
+\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\.
+.
+.IP
+Note 1: this mode is the only available one when multithread support is disabled\.
+.
+.IP
+Note 2: this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
 .
 .IP "\(bu" 4
 \fB\-\-auto\-threads={physical,logical} (default: physical)\fR: When using a default amount of threads via \fB\-T0\fR, choose the default based on the number of detected physical or logical cores\.
 .
 .IP "\(bu" 4
-\fB\-\-adapt[=min=#,max=#]\fR : \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
+\fB\-\-adapt[=min=#,max=#]\fR: \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MiB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\.
+.
+.IP
+\fINote\fR: at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
 .
 .IP "\(bu" 4
 \fB\-\-long[=#]\fR: enables long distance matching with \fB#\fR \fBwindowLog\fR, if \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
@@ -126,40 +138,49 @@ Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \
 \fB\-D DICT\fR: use \fBDICT\fR as Dictionary to compress or decompress FILE(s)
 .
 .IP "\(bu" 4
-\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize\.
+\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that \fIwindowSize\fR > \fIsrcSize\fR\.
 .
 .IP
-Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be automatically activated if chainLog < fileLog (fileLog being the windowLog required to cover the whole file)\. You can also manually force it\. Note: for all levels, you can use \-\-patch\-from in \-\-single\-thread mode to improve compression ratio at the cost of speed Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR
+Note: cannot use both this and \fB\-D\fR together\.
+.
+.IP
+Note: \fB\-\-long\fR mode will be automatically activated if \fIchainLog\fR < \fIfileLog\fR (\fIfileLog\fR being the \fIwindowLog\fR required to cover the whole file)\. You can also manually force it\.
+.
+.IP
+Note: for all levels, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread\fR mode to improve compression ratio at the cost of speed\.
+.
+.IP
+Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR\.
 .
 .IP "\(bu" 4
-\fB\-\-rsyncable\fR : \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
+\fB\-\-rsyncable\fR: \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
 .
 .IP "\(bu" 4
 \fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled)
 .
 .IP "\(bu" 4
-\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \-\-content\-size (meaning that the original size will be placed in the header)\.
+\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \fB\-\-content\-size\fR (meaning that the original size will be placed in the header)\.
 .
 .IP "\(bu" 4
 \fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
 .
 .IP "\(bu" 4
-\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
+\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MiB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
 .
 .IP
-This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\.
+This is also used during compression when using with \fB\-\-patch\-from=\fR\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MiB)\.
 .
 .IP
-Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GB\. zstd will load training samples up to the memory limit and ignore the rest\.
+Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GiB\. zstd will load training samples up to the memory limit and ignore the rest\.
 .
 .IP "\(bu" 4
-\fB\-\-stream\-size=#\fR : Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
+\fB\-\-stream\-size=#\fR: Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
 .
 .IP "\(bu" 4
 \fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
 .
 .IP "\(bu" 4
-\fB\-o FILE\fR: save result into \fBFILE\fR
+\fB\-o FILE\fR: save result into \fBFILE\fR\.
 .
 .IP "\(bu" 4
 \fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
@@ -171,10 +192,10 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
 \fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
 .
 .IP "\(bu" 4
-\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\-f) option is set\.
+\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\fB\-f\fR) option is set\.
 .
 .IP "\(bu" 4
-\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\.
+\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \fB\-o\fR, will trigger a confirmation prompt (which can be silenced with \fB\-f\fR), as this is a destructive operation\.
 .
 .IP "\(bu" 4
 \fB\-k\fR, \fB\-\-keep\fR: keep source file(s) after successful compression or decompression\. This is the default behavior\.
@@ -201,7 +222,7 @@ If input directory contains "\.\.", the files in this directory will be ignored\
 \fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit
 .
 .IP "\(bu" 4
-\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
+\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced: \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
 .
 .IP "\(bu" 4
 \fB\-v\fR, \fB\-\-verbose\fR: verbose mode, display more information
@@ -213,14 +234,14 @@ If input directory contains "\.\.", the files in this directory will be ignored\
 \fB\-\-no\-progress\fR: do not display the progress bar, but keep all other messages\.
 .
 .IP "\(bu" 4
-\fB\-\-show\-default\-cparams\fR: Shows the default compression parameters that will be used for a particular src file\. If the provided src file is not a regular file (e\.g\. named pipe), the cli will just output the default parameters\. That is, the parameters that are used when the src size is unknown\.
+\fB\-\-show\-default\-cparams\fR: shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size\. If the provided file is not a regular file (e\.g\. a pipe), this flag will output the parameters used for inputs of unknown size\.
 .
 .IP "\(bu" 4
 \fB\-\-\fR: All arguments after \fB\-\-\fR are treated as files
 .
 .IP "" 0
 .
-.SS "gzip Operation modifiers"
+.SS "gzip Operation Modifiers"
 When invoked via a \fBgzip\fR symlink, \fBzstd\fR will support further options that intend to mimic the \fBgzip\fR behavior:
 .
 .TP
@@ -231,7 +252,7 @@ do not store the original filename and timestamps when compressing a file\. This
 \fB\-\-best\fR
 alias to the option \fB\-9\fR\.
 .
-.SS "Interactions with Environment Variables"
+.SS "Environment Variables"
 Employing environment variables to set parameters has security implications\. Therefore, this avenue is intentionally limited\. Only \fBZSTD_CLEVEL\fR and \fBZSTD_NBTHREADS\fR are currently supported\. They set the compression level and number of threads to use during compression, respectively\.
 .
 .P
@@ -251,7 +272,7 @@ They can both be overridden by corresponding command line arguments: \fB\-#\fR f
 Use FILEs as training set to create a dictionary\. The training set should ideally contain a lot of samples (> 100), and weight typically 100x the target dictionary size (for example, ~10 MB for a 100 KB dictionary)\. \fB\-\-train\fR can be combined with \fB\-r\fR to indicate a directory rather than listing all the files, which can be useful to circumvent shell expansion limits\.
 .
 .IP
-Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KB of these samples will be used for training\.
+Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KiB of these samples will be used for training\.
 .
 .IP
 \fB\-\-train\fR supports multithreading if \fBzstd\fR is compiled with threading support (default)\. Additional advanced parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The slower cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Default \fB\-\-train\fR is equivalent to \fB\-\-train\-fastcover=d=8,steps=4\fR\.
@@ -281,7 +302,10 @@ In situations where the training set is larger than maximum memory, the CLI will
 .
 .TP
 \fB\-\-dictID=#\fR
-A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
+A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage: an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
+.
+.IP
+Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2^31, so they should not be used in public\.
 .
 .TP
 \fB\-\-train\-cover[=k#,d=#,steps=#,split=#,shrink[=#]]\fR
@@ -366,7 +390,7 @@ cut file(s) into independent chunks of size # (default: no chunking)
 set process priority to real\-time
 .
 .P
-\fBOutput Format:\fR CompressionLevel#Filename : InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+\fBOutput Format:\fR CompressionLevel#Filename: InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
 .
 .P
 \fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
@@ -377,14 +401,14 @@ set process priority to real\-time
 Specify the size of each compression job\. This parameter is only available when multi\-threading is enabled\. Each compression job is run in parallel, so this value indirectly impacts the nb of active threads\. Default job size varies depending on compression level (generally \fB4 * windowSize\fR)\. \fB\-B#\fR makes it possible to manually select a custom size\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 512 KB, or \fBoverlapSize\fR, whichever is largest\. Different job sizes will lead to non\-identical compressed frames\.
 .
 .SS "\-\-zstd[=options]:"
-\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
+\fBzstd\fR provides 22 predefined regular compression levels plus the fast levels\. This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor\. (You can see the result of this translation with \fB\-\-show\-default\-cparams\fR\.) These specific parameters can be overridden with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
 .
 .TP
 \fBstrategy\fR=\fIstrat\fR, \fBstrat\fR=\fIstrat\fR
 Specify a strategy used by a match finder\.
 .
 .IP
-There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
+There are 9 strategies numbered from 1 to 9, from fastest to strongest: 1=\fBZSTD_fast\fR, 2=\fBZSTD_dfast\fR, 3=\fBZSTD_greedy\fR, 4=\fBZSTD_lazy\fR, 5=\fBZSTD_lazy2\fR, 6=\fBZSTD_btlazy2\fR, 7=\fBZSTD_btopt\fR, 8=\fBZSTD_btultra\fR, 9=\fBZSTD_btultra2\fR\.
 .
 .TP
 \fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@@ -404,17 +428,17 @@ Specify the maximum number of bits for a hash table\.
 Bigger hash tables cause fewer collisions which usually makes compression faster, but requires more memory during compression\.
 .
 .IP
-The minimum \fIhlog\fR is 6 (64 B) and the maximum is 30 (1 GiB)\.
+The minimum \fIhlog\fR is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB)\.
 .
 .TP
 \fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR
-Specify the maximum number of bits for a hash chain or a binary tree\.
+Specify the maximum number of bits for the secondary search structure, whose form depends on the selected \fBstrategy\fR\.
 .
 .IP
-Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the ZSTD_fast strategy\.
+Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the \fBZSTD_fast\fR \fBstrategy\fR, which only has the primary hash table\.
 .
 .IP
-The minimum \fIclog\fR is 6 (64 B) and the maximum is 29 (524 Mib) on 32\-bit platforms and 30 (1 Gib) on 64\-bit platforms\.
+The minimum \fIclog\fR is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32\-bit platforms and 30 (1B entries / 4 GiB) on 64\-bit platforms\.
 .
 .TP
 \fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR
@@ -441,20 +465,23 @@ The minimum \fImml\fR is 3 and the maximum is 7\.
 The impact of this field vary depending on selected strategy\.
 .
 .IP
-For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\. t For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
+For \fBZSTD_btopt\fR, \fBZSTD_btultra\fR and \fBZSTD_btultra2\fR, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\.
+.
+.IP
+For \fBZSTD_fast\fR, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed: a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
 .
 .IP
 For all other strategies, this field has no impact\.
 .
 .IP
-The minimum \fItlen\fR is 0 and the maximum is 128 Kib\.
+The minimum \fItlen\fR is 0 and the maximum is 128 KiB\.
 .
 .TP
 \fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR
 Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
 .
 .IP
-The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default" : \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
+The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default": \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
 .
 .TP
 \fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
@@ -514,6 +541,12 @@ The following parameters sets advanced compression options to something similar
 .P
 \fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
 .
+.SH "SEE ALSO"
+\fBzstdgrep\fR(1), \fBzstdless\fR(1), \fBgzip\fR(1), \fBxz\fR(1)
+.
+.P
+The \fIzstandard\fR format is specified in Y\. Collet, "Zstandard Compression and the \'application/zstd\' Media Type", https://www\.ietf\.org/rfc/rfc8878\.txt, Internet RFC 8878 (February 2021)\.
+.
 .SH "BUGS"
 Report bugs at: https://github\.com/facebook/zstd/issues
 .
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 37c2ba187..45a88a347 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -4,7 +4,7 @@ zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
 SYNOPSIS
 --------
 
-`zstd` [*OPTIONS*] [-|_INPUT-FILE_] [-o _OUTPUT-FILE_]
+`zstd` [<OPTIONS>] [-|<INPUT-FILE>] [-o <OUTPUT-FILE>]
 
 `zstdmt` is equivalent to `zstd -T0`
 
@@ -16,7 +16,7 @@ SYNOPSIS
 DESCRIPTION
 -----------
 `zstd` is a fast lossless compression algorithm and data compression tool,
-with command line syntax similar to `gzip (1)` and `xz (1)`.
+with command line syntax similar to `gzip`(1) and `xz`(1).
 It is based on the **LZ77** family, with further FSE & huff0 entropy stages.
 `zstd` offers highly configurable compression speed,
 from fast modes at > 200 MB/s per core,
@@ -24,7 +24,7 @@ to strong modes with excellent compression ratios.
 It also features a very fast decoder, with speeds > 500 MB/s per core.
 
 `zstd` command line syntax is generally similar to gzip,
-but features the following differences :
+but features the following differences:
 
   - Source files are preserved by default.
     It's possible to remove them automatically by using the `--rm` command.
@@ -35,12 +35,13 @@ but features the following differences :
     Use `-q` to turn it off.
   - `zstd` does not accept input from console,
     though it does accept `stdin` when it's not the console.
+  - `zstd` does not store the input's filename or attributes, only its contents.
 
 `zstd` processes each _file_ according to the selected operation mode.
 If no _files_ are given or _file_ is `-`, `zstd` reads from standard input
 and writes the processed data to standard output.
 `zstd` will refuse to write compressed data to standard output
-if it is a terminal : it will display an error message and skip the _file_.
+if it is a terminal: it will display an error message and skip the file.
 Similarly, `zstd` will refuse to read compressed data from standard input
 if it is a terminal.
 
@@ -52,14 +53,15 @@ whose name is derived from the source _file_ name:
 * When decompressing, the `.zst` suffix is removed from the source filename to
   get the target filename
 
-### Concatenation with .zst files
+### Concatenation with .zst Files
 It is possible to concatenate multiple `.zst` files. `zstd` will decompress
 such agglomerated file as if it was a single `.zst` file.
 
 OPTIONS
 -------
 
-### Integer suffixes and special values
+### Integer Suffixes and Special Values
+
 In most places where an integer argument is expected,
 an optional suffix is supported to easily indicate large integers.
 There must be no space between the integer and the suffix.
@@ -71,7 +73,8 @@ There must be no space between the integer and the suffix.
     Multiply the integer by 1,048,576 (2\^20).
     `Mi`, `M`, and `MB` are accepted as synonyms for `MiB`.
 
-### Operation mode
+### Operation Mode
+
 If multiple operation mode options are given,
 the last one takes effect.
 
@@ -88,19 +91,21 @@ the last one takes effect.
     decompressed data is discarded and checksummed for errors.
     No files are created or removed.
 * `-b#`:
-    Benchmark file(s) using compression level #
-* `--train FILEs`:
-    Use FILEs as a training set to create a dictionary.
+    Benchmark file(s) using compression level _#_.
+    See _BENCHMARK_ below for a description of this operation.
+* `--train FILES`:
+    Use _FILES_ as a training set to create a dictionary.
     The training set should contain a lot of small files (> 100).
+    See _DICTIONARY BUILDER_ below for a description of this operation.
 * `-l`, `--list`:
     Display information related to a zstd compressed file, such as size, ratio, and checksum.
     Some of these fields may not be available.
     This command's output can be augmented with the `-v` modifier.
 
-### Operation modifiers
+### Operation Modifiers
 
 * `-#`:
-    `#` compression level \[1-19] (default: 3)
+    selects `#` compression level \[1-19\] (default: 3)
 * `--ultra`:
     unlocks high compression levels 20+ (maximum 22), using a lot more memory.
     Note that decompression will also require more memory when using these levels.
@@ -122,21 +127,24 @@ the last one takes effect.
     As compression is serialized with I/O, this can be slightly slower.
     Single-thread mode features significantly lower memory usage,
     which can be useful for systems with limited amount of memory, such as 32-bit systems.
-    Note 1 : this mode is the only available one when multithread support is disabled.
-    Note 2 : this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
+
+    Note 1: this mode is the only available one when multithread support is disabled.
+
+    Note 2: this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
     Final compressed result is also slightly different from `-T1`.
 * `--auto-threads={physical,logical} (default: physical)`:
     When using a default amount of threads via `-T0`, choose the default based on the number
     of detected physical or logical cores.
-* `--adapt[=min=#,max=#]` :
+* `--adapt[=min=#,max=#]`:
     `zstd` will dynamically adapt compression level to perceived I/O conditions.
     Compression level adaptation can be observed live by using command `-v`.
     Adaptation can be constrained between supplied `min` and `max` levels.
     The feature works when combined with multi-threading and `--long` mode.
     It does not work with `--single-thread`.
-    It sets window size to 8 MB by default (can be changed manually, see `wlog`).
+    It sets window size to 8 MiB by default (can be changed manually, see `wlog`).
     Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
-    _note_ : at the time of this writing, `--adapt` can remain stuck at low speed
+
+    _Note_: at the time of this writing, `--adapt` can remain stuck at low speed
     when combined with multiple worker threads (>=2).
 * `--long[=#]`:
     enables long distance matching with `#` `windowLog`, if `#` is not
@@ -153,18 +161,21 @@ the last one takes effect.
 * `--patch-from FILE`:
     Specify the file to be used as a reference point for zstd's diff engine.
     This is effectively dictionary compression with some convenient parameter
-    selection, namely that windowSize > srcSize.
+    selection, namely that _windowSize_ > _srcSize_.
 
-    Note: cannot use both this and -D together
-    Note: `--long` mode will be automatically activated if chainLog < fileLog
-        (fileLog being the windowLog required to cover the whole file). You
+    Note: cannot use both this and `-D` together.
+
+    Note: `--long` mode will be automatically activated if _chainLog_ < _fileLog_
+        (_fileLog_ being the _windowLog_ required to cover the whole file). You
         can also manually force it.
-    Note: for all levels, you can use --patch-from in --single-thread mode
-        to improve compression ratio at the cost of speed
+
+    Note: for all levels, you can use `--patch-from` in `--single-thread` mode
+        to improve compression ratio at the cost of speed.
+
     Note: for level 19, you can get increased compression ratio at the cost
         of speed by specifying `--zstd=targetLength=` to be something large
-        (i.e. 4096), and by setting a large `--zstd=chainLog=`
-* `--rsyncable` :
+        (i.e. 4096), and by setting a large `--zstd=chainLog=`.
+* `--rsyncable`:
     `zstd` will periodically synchronize the compression state to make the
     compressed file more rsync-friendly. There is a negligible impact to
     compression ratio, and the faster compression levels will see a small
@@ -177,24 +188,24 @@ the last one takes effect.
 * `--[no-]content-size`:
     enable / disable whether or not the original size of the file is placed in
     the header of the compressed file. The default option is
-    --content-size (meaning that the original size will be placed in the header).
+    `--content-size` (meaning that the original size will be placed in the header).
 * `--no-dictID`:
     do not store dictionary ID within frame header (dictionary compression).
     The decoder will have to rely on implicit knowledge about which dictionary to use,
     it won't be able to check if it's correct.
 * `-M#`, `--memory=#`:
-    Set a memory usage limit. By default, `zstd` uses 128 MB for decompression
+    Set a memory usage limit. By default, `zstd` uses 128 MiB for decompression
     as the maximum amount of memory the decompressor is allowed to use, but you can
     override this manually if need be in either direction (i.e. you can increase or
     decrease it).
 
-    This is also used during compression when using with --patch-from=. In this case,
-    this parameter overrides that maximum size allowed for a dictionary. (128 MB).
+    This is also used during compression when using with `--patch-from=`. In this case,
+    this parameter overrides that maximum size allowed for a dictionary. (128 MiB).
 
     Additionally, this can be used to limit memory for dictionary training. This parameter
-    overrides the default limit of 2 GB. zstd will load training samples up to the memory limit
+    overrides the default limit of 2 GiB. zstd will load training samples up to the memory limit
     and ignore the rest.
-* `--stream-size=#` :
+* `--stream-size=#`:
     Sets the pledged source size of input coming from a stream. This value must be exact, as it
     will be included in the produced frame header. Incorrect stream sizes will cause an error.
     This information will be used to better optimize compression parameters, resulting in
@@ -207,7 +218,7 @@ the last one takes effect.
     Exact guesses result in better compression ratios. Overestimates result in slightly
     degraded compression ratios, while underestimates may result in significant degradation.
 * `-o FILE`:
-    save result into `FILE`
+    save result into `FILE`.
 * `-f`, `--force`:
     disable input and output checks. Allows overwriting existing files, input
     from console, output to stdout, operating on links, block devices, etc.
@@ -227,11 +238,11 @@ the last one takes effect.
     enable / disable passing through uncompressed files as-is. During
     decompression when pass-through is enabled, unrecognized formats will be
     copied as-is from the input to the output. By default, pass-through will
-    occur when the output destination is stdout and the force (-f) option is
+    occur when the output destination is stdout and the force (`-f`) option is
     set.
 * `--rm`:
     remove source file(s) after successful compression or decompression. If used in combination with
-    -o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
+    `-o`, will trigger a confirmation prompt (which can be silenced with `-f`), as this is a destructive operation.
 * `-k`, `--keep`:
     keep source file(s) after successful compression or decompression.
     This is the default behavior.
@@ -270,7 +281,7 @@ the last one takes effect.
     display help/long help and exit
 * `-V`, `--version`:
     display version number and exit.
-    Advanced : `-vV` also displays supported formats.
+    Advanced: `-vV` also displays supported formats.
     `-vvV` also displays POSIX support.
     `-q` will only display the version number, suitable for machine reading.
 * `-v`, `--verbose`:
@@ -281,15 +292,13 @@ the last one takes effect.
 * `--no-progress`:
     do not display the progress bar, but keep all other messages.
 * `--show-default-cparams`:
-    Shows the default compression parameters that will be used for a
-    particular src file. If the provided src file is not a regular file
-    (e.g. named pipe), the cli will just output the default parameters.
-    That is, the parameters that are used when the src size is unknown.
+    shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size.
+    If the provided file is not a regular file (e.g. a pipe), this flag will output the parameters used for inputs of unknown size.
 * `--`:
     All arguments after `--` are treated as files
 
 
-### gzip Operation modifiers
+### gzip Operation Modifiers
 When invoked via a `gzip` symlink, `zstd` will support further
 options that intend to mimic the `gzip` behavior:
 
@@ -300,7 +309,7 @@ options that intend to mimic the `gzip` behavior:
     alias to the option `-9`.
 
 
-### Interactions with Environment Variables
+### Environment Variables
 
 Employing environment variables to set parameters has security implications.
 Therefore, this avenue is intentionally limited.
@@ -341,7 +350,7 @@ Compression of small files similar to the sample set will be greatly improved.
     Since dictionary compression is mostly effective for small files,
     the expectation is that the training set will only contain small files.
     In the case where some samples happen to be large,
-    only the first 128 KB of these samples will be used for training.
+    only the first 128 KiB of these samples will be used for training.
 
     `--train` supports multithreading if `zstd` is compiled with threading support (default).
     Additional advanced parameters can be specified with `--train-fastcover`.
@@ -389,11 +398,13 @@ Compression of small files similar to the sample set will be greatly improved.
     It's possible to provide an explicit number ID instead.
     It's up to the dictionary manager to not assign twice the same ID to
     2 different dictionaries.
-    Note that short numbers have an advantage :
+    Note that short numbers have an advantage:
     an ID < 256 will only need 1 byte in the compressed frame header,
     and an ID < 65536 will only need 2 bytes.
     This compares favorably to 4 bytes default.
 
+    Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2\^31, so they should not be used in public.
+
 * `--train-cover[=k#,d=#,steps=#,split=#,shrink[=#]]`:
     Select parameters for the default dictionary builder algorithm named cover.
     If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
@@ -482,7 +493,7 @@ BENCHMARK
 * `--priority=rt`:
     set process priority to real-time
 
-**Output Format:** CompressionLevel#Filename : InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+**Output Format:** CompressionLevel#Filename: InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
 
 **Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
 
@@ -499,9 +510,10 @@ This minimum is either 512 KB, or `overlapSize`, whichever is largest.
 Different job sizes will lead to non-identical compressed frames.
 
 ### --zstd[=options]:
-`zstd` provides 22 predefined compression levels.
-The selected or default predefined compression level can be changed with
-advanced compression options.
+`zstd` provides 22 predefined regular compression levels plus the fast levels.
+This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor.
+(You can see the result of this translation with `--show-default-cparams`.)
+These specific parameters can be overridden with advanced compression options.
 The _options_ are provided as a comma-separated list.
 You may specify only the options you want to change and the rest will be
 taken from the selected or default compression level.
@@ -510,10 +522,10 @@ The list of available _options_:
 - `strategy`=_strat_, `strat`=_strat_:
     Specify a strategy used by a match finder.
 
-    There are 9 strategies numbered from 1 to 9, from faster to stronger:
-    1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
-    4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
-    7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
+    There are 9 strategies numbered from 1 to 9, from fastest to strongest:
+    1=`ZSTD_fast`, 2=`ZSTD_dfast`, 3=`ZSTD_greedy`,
+    4=`ZSTD_lazy`, 5=`ZSTD_lazy2`, 6=`ZSTD_btlazy2`,
+    7=`ZSTD_btopt`, 8=`ZSTD_btultra`, 9=`ZSTD_btultra2`.
 
 - `windowLog`=_wlog_, `wlog`=_wlog_:
     Specify the maximum number of bits for a match distance.
@@ -533,19 +545,20 @@ The list of available _options_:
     Bigger hash tables cause fewer collisions which usually makes compression
     faster, but requires more memory during compression.
 
-    The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
+    The minimum _hlog_ is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB).
 
 - `chainLog`=_clog_, `clog`=_clog_:
-    Specify the maximum number of bits for a hash chain or a binary tree.
+    Specify the maximum number of bits for the secondary search structure,
+    whose form depends on the selected `strategy`.
 
     Higher numbers of bits increases the chance to find a match which usually
     improves compression ratio.
     It also slows down compression speed and increases memory requirements for
     compression.
-    This option is ignored for the ZSTD_fast strategy.
+    This option is ignored for the `ZSTD_fast` `strategy`, which only has the primary hash table.
 
-    The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
-    and 30 (1 Gib) on 64-bit platforms.
+    The minimum _clog_ is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32-bit platforms
+    and 30 (1B entries / 4 GiB) on 64-bit platforms.
 
 - `searchLog`=_slog_, `slog`=_slog_:
     Specify the maximum number of searches in a hash chain or a binary tree
@@ -567,19 +580,19 @@ The list of available _options_:
 - `targetLength`=_tlen_, `tlen`=_tlen_:
     The impact of this field vary depending on selected strategy.
 
-    For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
+    For `ZSTD_btopt`, `ZSTD_btultra` and `ZSTD_btultra2`, it specifies
     the minimum match length that causes match finder to stop searching.
     A larger `targetLength` usually improves compression ratio
     but decreases compression speed.
-t
-    For ZSTD\_fast, it triggers ultra-fast mode when > 0.
+
+    For `ZSTD_fast`, it triggers ultra-fast mode when > 0.
     The value represents the amount of data skipped between match sampling.
-    Impact is reversed : a larger `targetLength` increases compression speed
+    Impact is reversed: a larger `targetLength` increases compression speed
     but decreases compression ratio.
 
     For all other strategies, this field has no impact.
 
-    The minimum _tlen_ is 0 and the maximum is 128 Kib.
+    The minimum _tlen_ is 0 and the maximum is 128 KiB.
 
 - `overlapLog`=_ovlog_,  `ovlog`=_ovlog_:
     Determine `overlapSize`, amount of data reloaded from previous job.
@@ -591,7 +604,7 @@ t
     9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
     Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
     For example, 8 means "windowSize/2", and 6 means "windowSize/8".
-    Value 0 is special and means "default" : _ovlog_ is automatically determined by `zstd`.
+    Value 0 is special and means "default": _ovlog_ is automatically determined by `zstd`.
     In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
 
 - `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
@@ -641,6 +654,11 @@ similar to predefined level 19 for files bigger than 256 KB:
 
 `--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
 
+SEE ALSO
+--------
+`zstdgrep`(1), `zstdless`(1), `gzip`(1), `xz`(1)
+
+The <zstandard> format is specified in Y. Collet, "Zstandard Compression and the 'application/zstd' Media Type", https://www.ietf.org/rfc/rfc8878.txt, Internet RFC 8878 (February 2021).
 
 BUGS
 ----
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index cc6bbb935..9937a9c97 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -52,12 +52,12 @@
 /*-************************************
 *  Constants
 **************************************/
-#define COMPRESSOR_NAME "zstd command line interface"
+#define COMPRESSOR_NAME "Zstandard CLI"
 #ifndef ZSTD_VERSION
 #  define ZSTD_VERSION "v" ZSTD_VERSION_STRING
 #endif
 #define AUTHOR "Yann Collet"
-#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
+#define WELCOME_MESSAGE "*** %s (%i-bit) %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
 
 #define ZSTD_ZSTDMT "zstdmt"
 #define ZSTD_UNZSTD "unzstd"
@@ -143,160 +143,174 @@ static int exeNameMatch(const char* exeName, const char* test)
  */
 static void usage(FILE* f, const char* programName)
 {
-    DISPLAY_F(f, "Usage: %s [OPTION]... [FILE]... [-o file]\n", programName);
-    DISPLAY_F(f, "Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).\n\n");
-    DISPLAY_F(f, "  -o file              result stored into `file` (only 1 output file)\n");
-#ifndef ZSTD_NOCOMPRESS
-    DISPLAY_F(f, "  -1 .. -%d            compression level (faster .. better; default: %d)\n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
-#endif
-#ifndef ZSTD_NODECOMPRESS
-    DISPLAY_F(f, "  -d, --decompress     decompression\n");
-#endif
-    DISPLAY_F(f, "  -f, --force          disable input and output checks. Allows overwriting existing files,\n");
-    DISPLAY_F(f, "                       input from console, output to stdout, operating on links,\n");
-    DISPLAY_F(f, "                       block devices, etc. During decompression and when the output\n");
-    DISPLAY_F(f, "                       destination is stdout, pass-through unrecognized formats as-is.\n");
-    DISPLAY_F(f, "      --rm             remove source file(s) after successful de/compression\n");
-    DISPLAY_F(f, "  -k, --keep           preserve source file(s) (default) \n");
+    DISPLAY_F(f, "Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.\n\n");
+    DISPLAY_F(f, "Usage: %s [OPTIONS...] [INPUT... | -] [-o OUTPUT]\n\n", programName);
+    DISPLAY_F(f, "Options:\n");
+    DISPLAY_F(f, "  -o OUTPUT                     Write output to a single file, OUTPUT.\n");
+    DISPLAY_F(f, "  -k, --keep                    Preserve INPUT file(s). [Default] \n");
+    DISPLAY_F(f, "  --rm                          Remove INPUT file(s) after successful (de)compression.\n");
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
-        DISPLAY_F(f, "  -n, --no-name        do not store original filename when compressing\n");
+        DISPLAY_F(f, "  -n, --no-name                 Do not store original filename when compressing.\n\n");
     }
 #endif
-    DISPLAY_F(f, "  -D DICT              use DICT as Dictionary for compression or decompression\n");
-    DISPLAY_F(f, "  -h                   display usage and exit\n");
-    DISPLAY_F(f, "  -H,--help            display long help and exit\n");
+    DISPLAY_F(f, "\n");
+#ifndef ZSTD_NOCOMPRESS
+    DISPLAY_F(f, "  -#                            Desired compression level, where `#` is a number between 1 and %d;\n", ZSTDCLI_CLEVEL_MAX);
+    DISPLAY_F(f, "                                lower numbers provide faster compression, higher numbers yield\n");
+    DISPLAY_F(f, "                                better compression ratios. [Default: %d]\n\n", ZSTDCLI_CLEVEL_DEFAULT);
+#endif
+#ifndef ZSTD_NODECOMPRESS
+    DISPLAY_F(f, "  -d, --decompress              Perform decompression.\n");
+#endif
+    DISPLAY_F(f, "  -D DICT                       Use DICT as the dictionary for compression or decompression.\n\n");
+    DISPLAY_F(f, "  -f, --force                   Disable input and output checks. Allows overwriting existing files,\n");
+    DISPLAY_F(f, "                                receiving input from the console, printing ouput to STDOUT, and\n");
+    DISPLAY_F(f, "                                operating on links, block devices, etc. Unrecognized formats will be\n");
+    DISPLAY_F(f, "                                passed-through through as-is.\n\n");
+
+    DISPLAY_F(f, "  -h                            Display short usage and exit.\n");
+    DISPLAY_F(f, "  -H, --help                    Display full help and exit.\n");
+    DISPLAY_F(f, "  -V, --version                 Display the program version and exit.\n");
     DISPLAY_F(f, "\n");
 }
 
 static void usage_advanced(const char* programName)
 {
     DISPLAYOUT(WELCOME_MESSAGE);
+    DISPLAYOUT("\n");
     usage(stdout, programName);
-    DISPLAYOUT("Advanced options :\n");
-    DISPLAYOUT("  -V, --version        display Version number and exit\n");
+    DISPLAYOUT("Advanced options:\n");
+    DISPLAYOUT("  -c, --stdout                  Write to STDOUT (even if it is a console) and keep the INPUT file(s).\n\n");
 
-    DISPLAYOUT("  -c, --stdout         write to standard output (even if it is the console), keep original file\n");
-
-    DISPLAYOUT("  -v, --verbose        verbose mode; specify multiple times to increase verbosity\n");
-    DISPLAYOUT("  -q, --quiet          suppress warnings; specify twice to suppress errors too\n");
-    DISPLAYOUT("      --[no-]progress  forcibly display, or never display the progress counter\n");
-    DISPLAYOUT("                       note: any (de)compressed output to terminal will mix with progress counter text\n");
+    DISPLAYOUT("  -v, --verbose                 Enable verbose output; pass multiple times to increase verbosity.\n");
+    DISPLAYOUT("  -q, --quiet                   Suppress warnings; pass twice to suppress errors.\n");
+#ifndef ZSTD_NOTRACE
+    DISPLAYOUT("  --trace LOG                   Log tracing information to LOG.\n");
+#endif
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --[no-]progress               Forcibly show/hide the progress counter. NOTE: Any (de)compressed\n");
+    DISPLAYOUT("                                output to terminal will mix with progress counter text.\n\n");
 
 #ifdef UTIL_HAS_CREATEFILELIST
-    DISPLAYOUT("  -r                   operate recursively on directories\n");
-    DISPLAYOUT("     --filelist FILE   read list of files to operate upon from FILE\n");
-    DISPLAYOUT("     --output-dir-flat DIR :  processed files are stored into DIR\n");
+    DISPLAYOUT("  -r                            Operate recursively on directories.\n");
+    DISPLAYOUT("  --filelist LIST               Read a list of files to operate on from LIST.\n");
+    DISPLAYOUT("  --output-dir-flat DIR         Store processed files in DIR.\n");
 #endif
 
 #ifdef UTIL_HAS_MIRRORFILELIST
-    DISPLAYOUT("     --output-dir-mirror DIR :  processed files are stored into DIR respecting original directory structure\n");
+    DISPLAYOUT("  --output-dir-mirror DIR       Store processed files in DIR, respecting original directory structure.\n");
 #endif
     if (AIO_supported())
-        DISPLAYOUT("     --[no-]asyncio    use asynchronous IO (default: enabled)\n");
+        DISPLAYOUT("  --[no-]asyncio                Use asynchronous IO. [Default: Enabled]\n");
 
+    DISPLAYOUT("\n");
 #ifndef ZSTD_NOCOMPRESS
-    DISPLAYOUT("     --[no-]check      during compression, add XXH64 integrity checksum to frame (default: enabled)\n");
+    DISPLAYOUT("  --[no-]check                  Add XXH64 integrity checksums during compression. [Default: Add, Validate]\n");
 #ifndef ZSTD_NODECOMPRESS
-    DISPLAYOUT("                       if specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate)\n");
+    DISPLAYOUT("                                If `-d` is present, ignore/validate checksums during decompression.\n");
 #endif
 #else
 #ifdef ZSTD_NOCOMPRESS
-    DISPLAYOUT("      --[no-]check     during decompression, ignore/validate checksums in compressed frame (default: validate)");
+    DISPLAYOUT("  --[no-]check                  Ignore/validate checksums during decompression. [Default: Validate]");
 #endif
-    DISPLAYOUT("\n");
 #endif /* ZSTD_NOCOMPRESS */
 
-#ifndef ZSTD_NOTRACE
-    DISPLAYOUT("     --trace FILE      log tracing information to FILE\n");
-#endif
-    DISPLAYOUT("     --                all arguments after \"--\" are treated as files\n");
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --                            Treat remaining arguments after `--` as files.\n");
 
 #ifndef ZSTD_NOCOMPRESS
     DISPLAYOUT("\n");
-    DISPLAYOUT("Advanced compression options :\n");
-    DISPLAYOUT("     --ultra           enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
-    DISPLAYOUT("     --fast[=#]        switch to very fast compression levels (default: %u)\n", 1);
+    DISPLAYOUT("Advanced compression options:\n");
+    DISPLAYOUT("  --ultra                       Enable levels beyond %i, up to %i; requires more memory.\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
+    DISPLAYOUT("  --fast[=#]                    Use to very fast compression levels. [Default: %u]\n", 1);
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
-        DISPLAYOUT("     --best            compatibility alias for -9 \n");
-        DISPLAYOUT("     --no-name         do not store original filename when compressing\n");
+        DISPLAYOUT("  --best                        Compatibility alias for `-9`.\n");
     }
 #endif
-    DISPLAYOUT("     --long[=#]        enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
-    DISPLAYOUT("     --patch-from=FILE :  specify the file to be used as a reference point for zstd's diff engine. \n");
-    DISPLAYOUT("     --adapt           dynamically adapt compression level to I/O conditions\n");
+    DISPLAYOUT("  --adapt                       Dynamically adapt compression level to I/O conditions.\n");
+    DISPLAYOUT("  --long[=#]                    Enable long distance matching with window log #. [Default: %u]\n", g_defaultMaxWindowLog);
+    DISPLAYOUT("  --patch-from=REF              Use REF as the reference point for Zstandard's diff engine. \n\n");
 # ifdef ZSTD_MULTITHREAD
-    DISPLAYOUT("  -T#                  spawn # compression threads (default: 1, 0==# cores) \n");
-    DISPLAYOUT("  -B#                  select size of each job (default: 0==automatic) \n");
-    DISPLAYOUT("     --single-thread   use a single thread for both I/O and compression (result slightly different than -T1) \n");
-    DISPLAYOUT("     --auto-threads={physical,logical} :  use either physical cores or logical cores as default when specifying -T0 (default: physical)\n");
-    DISPLAYOUT("     --rsyncable       compress using a rsync-friendly method (-B sets block size) \n");
+    DISPLAYOUT("  -T#                           Spawn # compression threads. [Default: 1; pass 0 for core count.]\n");
+    DISPLAYOUT("  --single-thread               Share a single thread for I/O and compression (slightly different than `-T1`).\n");
+    DISPLAYOUT("  --auto-threads={physical|logical}\n");
+    DISPLAYOUT("                                Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
+    DISPLAYOUT("  -B#                           Set job size to #. [Default: 0 (automatic)]\n");
+    DISPLAYOUT("  --rsyncable                   Compress using a rsync-friendly method (`-B` sets block size). \n");
+    DISPLAYOUT("\n");
 # endif
-    DISPLAYOUT("     --exclude-compressed :  only compress files that are not already compressed \n");
-    DISPLAYOUT("     --stream-size=#   specify size of streaming input from `stdin` \n");
-    DISPLAYOUT("     --size-hint=#     optimize compression parameters for streaming input of approximately this size \n");
-    DISPLAYOUT("     --target-compressed-block-size=# :  generate compressed block of approximately targeted size \n");
-    DISPLAYOUT("     --no-dictID       don't write dictID into header (dictionary compression only)\n");
-    DISPLAYOUT("     --[no-]compress-literals :  force (un)compressed literals\n");
-    DISPLAYOUT("     --[no-]row-match-finder :  force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
+    DISPLAYOUT("  --exclude-compressed          Only compress files that are not already compressed.\n\n");
 
-    DISPLAYOUT("     --format=zstd     compress files to the .zst format (default)\n");
+    DISPLAYOUT("  --stream-size=#               Specify size of streaming input from STDIN.\n");
+    DISPLAYOUT("  --size-hint=#                 Optimize compression parameters for streaming input of approximately size #.\n");
+    DISPLAYOUT("  --target-compressed-block-size=#\n");
+    DISPLAYOUT("                                Generate compressed blocks of approximately # size.\n\n");
+    DISPLAYOUT("  --no-dictID                   Don't write `dictID` into the header (dictionary compression only).\n");
+    DISPLAYOUT("  --[no-]compress-literals      Force (un)compressed literals.\n");
+    DISPLAYOUT("  --[no-]row-match-finder       Explicitly enable/disable the fast, row-based matchfinder for\n");
+    DISPLAYOUT("                                the 'greedy', 'lazy', and 'lazy2' strategies.\n");
+
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --format=zstd                 Compress files to the `.zst` format. [Default]\n");
 #ifdef ZSTD_GZCOMPRESS
-    DISPLAYOUT("     --format=gzip     compress files to the .gz format\n");
+    DISPLAYOUT("  --format=gzip                 Compress files to the `.gz` format.\n");
 #endif
 #ifdef ZSTD_LZMACOMPRESS
-    DISPLAYOUT("     --format=xz       compress files to the .xz format\n");
-    DISPLAYOUT("     --format=lzma     compress files to the .lzma format\n");
+    DISPLAYOUT("  --format=xz                   Compress files to the `.xz` format.\n");
+    DISPLAYOUT("  --format=lzma                 Compress files to the `.lzma` format.\n");
 #endif
 #ifdef ZSTD_LZ4COMPRESS
-    DISPLAYOUT( "     --format=lz4     compress files to the .lz4 format\n");
+    DISPLAYOUT( "  --format=lz4                 Compress files to the `.lz4` format.\n");
 #endif
 #endif  /* !ZSTD_NOCOMPRESS */
 
 #ifndef ZSTD_NODECOMPRESS
     DISPLAYOUT("\n");
-    DISPLAYOUT("Advanced decompression options :\n");
-    DISPLAYOUT("  -l                   print information about zstd compressed files\n");
-    DISPLAYOUT("     --test            test compressed file integrity\n");
-    DISPLAYOUT("  -M#                  Set a memory usage limit for decompression\n");
+    DISPLAYOUT("Advanced decompression options:\n");
+    DISPLAYOUT("  -l                            Print information about Zstandard-compressed files.\n");
+    DISPLAYOUT("  --test                        Test compressed file integrity.\n");
+    DISPLAYOUT("  -M#                           Set the memory usage limit to # megabytes.\n");
 # if ZSTD_SPARSE_DEFAULT
-    DISPLAYOUT("     --[no-]sparse     sparse mode (default: enabled on file, disabled on stdout)\n");
+    DISPLAYOUT("  --[no-]sparse                 Enable sparse mode. [Default: Enabled for files, disabled for STDOUT.]\n");
 # else
-    DISPLAYOUT("     --[no-]sparse     sparse mode (default: disabled)\n");
+    DISPLAYOUT("  --[no-]sparse                 Enable sparse mode. [Default: Disabled]\n");
 # endif
     {
-        char const* passThroughDefault = "disabled";
+        char const* passThroughDefault = "Disabled";
         if (exeNameMatch(programName, ZSTD_CAT) ||
             exeNameMatch(programName, ZSTD_ZCAT) ||
             exeNameMatch(programName, ZSTD_GZCAT)) {
-            passThroughDefault = "enabled";
+            passThroughDefault = "Enabled";
         }
-        DISPLAYOUT("     --[no-]pass-through : passes through uncompressed files as-is (default: %s)\n", passThroughDefault);
+        DISPLAYOUT("  --[no-]pass-through           Pass through uncompressed files as-is. [Default: %s]\n", passThroughDefault);
     }
 #endif  /* ZSTD_NODECOMPRESS */
 
 #ifndef ZSTD_NODICT
     DISPLAYOUT("\n");
-    DISPLAYOUT("Dictionary builder :\n");
-    DISPLAYOUT("     --train ##        create a dictionary from a training set of files\n");
-    DISPLAYOUT("     --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] :  use the cover algorithm with optional args\n");
-    DISPLAYOUT("     --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] :  use the fast cover algorithm with optional args\n");
-    DISPLAYOUT("     --train-legacy[=s=#] :  use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
-    DISPLAYOUT("  -o DICT              DICT is dictionary name (default: %s)\n", g_defaultDictName);
-    DISPLAYOUT("     --maxdict=#       limit dictionary to specified size (default: %u)\n", g_defaultMaxDictSize);
-    DISPLAYOUT("     --dictID=#        force dictionary ID to specified value (default: random)\n");
+    DISPLAYOUT("Dictionary builder:\n");
+    DISPLAYOUT("  --train                       Create a dictionary from a training set of files.\n\n");
+    DISPLAYOUT("  --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]\n");
+    DISPLAYOUT("                                Use the cover algorithm (with optional arguments).\n");
+    DISPLAYOUT("  --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]\n");
+    DISPLAYOUT("                                Use the fast cover algorithm (with optional arguments).\n\n");
+    DISPLAYOUT("  --train-legacy[=s=#]          Use the legacy algorithm with selectivity #. [Default: %u]\n", g_defaultSelectivityLevel);
+    DISPLAYOUT("  -o NAME                       Use NAME as dictionary name. [Default: %s]\n", g_defaultDictName);
+    DISPLAYOUT("  --maxdict=#                   Limit dictionary to specified size #. [Default: %u]\n", g_defaultMaxDictSize);
+    DISPLAYOUT("  --dictID=#                    Force dictionary ID to #. [Default: Random]\n");
 #endif
 
 #ifndef ZSTD_NOBENCH
     DISPLAYOUT("\n");
-    DISPLAYOUT("Benchmark options : \n");
-    DISPLAYOUT("  -b#                  benchmark file(s), using # compression level (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT);
-    DISPLAYOUT("  -e#                  test all compression levels successively from -b# to -e# (default: 1)\n");
-    DISPLAYOUT("  -i#                  minimum evaluation time in seconds (default: 3s)\n");
-    DISPLAYOUT("  -B#                  cut file into independent chunks of size # (default: no chunking)\n");
-    DISPLAYOUT("  -S                   output one benchmark result per input file (default: consolidated result)\n");
-    DISPLAYOUT("     --priority=rt     set process priority to real-time\n");
+    DISPLAYOUT("Benchmark options:\n");
+    DISPLAYOUT("  -b#                           Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
+    DISPLAYOUT("  -e#                           Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
+    DISPLAYOUT("  -i#                           Set the minimum evaluation to time # seconds. [Default: 3]\n");
+    DISPLAYOUT("  -B#                           Cut file into independent chunks of size #. [Default: No chunking]\n");
+    DISPLAYOUT("  -S                            Output one benchmark result per input file. [Default: Consolidated result]\n");
+    DISPLAYOUT("  --priority=rt                 Set process priority to real-time.\n");
 #endif
 
 }
@@ -990,6 +1004,7 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--fake-stdin-is-console")) { UTIL_fakeStdinIsConsole(); continue; }
                 if (!strcmp(argument, "--fake-stdout-is-console")) { UTIL_fakeStdoutIsConsole(); continue; }
                 if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }
+                if (!strcmp(argument, "--trace-file-stat")) { UTIL_traceFileStat(); continue; }
 
                 /* long commands with arguments */
 #ifndef ZSTD_NODICT
diff --git a/programs/zstdcli_trace.c b/programs/zstdcli_trace.c
index b3b977feb..35075a52c 100644
--- a/programs/zstdcli_trace.c
+++ b/programs/zstdcli_trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/zstdcli_trace.h b/programs/zstdcli_trace.h
index 38c27dc04..9c135d3ca 100644
--- a/programs/zstdcli_trace.h
+++ b/programs/zstdcli_trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/zstdgrep.1.md b/programs/zstdgrep.1.md
index 35186a4bf..6370a81c7 100644
--- a/programs/zstdgrep.1.md
+++ b/programs/zstdgrep.1.md
@@ -4,16 +4,16 @@ zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files
 SYNOPSIS
 --------
 
-`zstdgrep` [*grep-flags*] [--] _pattern_ [_files_ ...]
+`zstdgrep` [<grep-flags>] [--] <pattern> [<files> ...]
 
 
 DESCRIPTION
 -----------
-`zstdgrep` runs `grep (1)` on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat (1)`.
+`zstdgrep` runs `grep`(1) on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat`(1).
 
-The grep-flags and pattern arguments are passed on to `grep (1)`.  If an `-e` flag is found in the `grep-flags`, `zstdgrep` will not look for a pattern argument.
+The <grep-flags> and <pattern> arguments are passed on to `grep`(1).  If an `-e` flag is found in the <grep-flags>, `zstdgrep` will not look for a <pattern> argument.
 
-Note that modern `grep` alternatives such as `ripgrep` (`rg`) support `zstd`-compressed files out of the box,
+Note that modern `grep` alternatives such as `ripgrep` (`rg`(1)) support `zstd`-compressed files out of the box,
 and can prove better alternatives than `zstdgrep` notably for unsupported complex pattern searches.
 Note though that such alternatives may also feature some minor command line differences.
 
@@ -23,7 +23,7 @@ In case of missing arguments or missing pattern, 1 will be returned, otherwise 0
 
 SEE ALSO
 --------
-`zstd (1)`
+`zstd`(1)
 
 AUTHORS
 -------
diff --git a/programs/zstdless.1.md b/programs/zstdless.1.md
index d91d48abc..67c1c7676 100644
--- a/programs/zstdless.1.md
+++ b/programs/zstdless.1.md
@@ -4,13 +4,13 @@ zstdless(1) -- view zstandard-compressed files
 SYNOPSIS
 --------
 
-`zstdless` [*flags*] [_file_ ...]
+`zstdless` [<flags>] [<file> ...]
 
 
 DESCRIPTION
 -----------
-`zstdless` runs `less (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
+`zstdless` runs `less`(1) on files or stdin, if no <file> argument is given, after decompressing them with `zstdcat`(1).
 
 SEE ALSO
 --------
-`zstd (1)`
+`zstd`(1)
diff --git a/tests/DEPRECATED-test-zstd-speed.py b/tests/DEPRECATED-test-zstd-speed.py
index 665e0a7f9..71d75b8cc 100755
--- a/tests/DEPRECATED-test-zstd-speed.py
+++ b/tests/DEPRECATED-test-zstd-speed.py
@@ -2,7 +2,7 @@
 # THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
 
 # ################################################################
-# Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/Makefile b/tests/Makefile
index afea6475a..8226176cc 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,6 @@
 
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -169,7 +169,7 @@ fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PR
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
 CLEAN += zstreamtest zstreamtest32
-ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
+ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c external_matchfinder.c
 ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
 ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
 zstreamtest32 : CFLAGS += -m32
diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py
index c98391193..153e7db4c 100644
--- a/tests/automated_benchmarking.py
+++ b/tests/automated_benchmarking.py
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/bigdict.c b/tests/bigdict.c
index fb0892586..ff2bb2d70 100644
--- a/tests/bigdict.c
+++ b/tests/bigdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/checkTag.c b/tests/checkTag.c
index f6c5e9766..26871ed0f 100644
--- a/tests/checkTag.c
+++ b/tests/checkTag.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/check_size.py b/tests/check_size.py
new file mode 100755
index 000000000..028b0a9c0
--- /dev/null
+++ b/tests/check_size.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+import os
+import subprocess
+import sys
+
+if len(sys.argv) != 3:
+	print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT")
+	sys.exit(1)
+
+file = sys.argv[1]
+limit = int(sys.argv[2])
+
+if not os.path.exists(file):
+	print(f"{file} does not exist")
+	sys.exit(1)
+
+size = os.path.getsize(file)
+
+if size > limit:
+	print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes")
+	sys.exit(1)
diff --git a/tests/cli-tests/README.md b/tests/cli-tests/README.md
index cdf9b8e71..7ca07c3e9 100644
--- a/tests/cli-tests/README.md
+++ b/tests/cli-tests/README.md
@@ -45,6 +45,16 @@ Examples:
 ./run.py --preserve --verbose basic/help.sh
 ```
 
+### Updating exact output
+
+If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
+
+Example:
+```
+./run.py --set-exact-output
+./run.py basic/help.sh --set-exact-output
+```
+
 ## Writing a test
 
 Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
diff --git a/tests/cli-tests/basic/help.sh.stdout.glob b/tests/cli-tests/basic/help.sh.stdout.glob
index bfb0f7753..5580dc669 100644
--- a/tests/cli-tests/basic/help.sh.stdout.glob
+++ b/tests/cli-tests/basic/help.sh.stdout.glob
@@ -1,25 +1,34 @@
 + zstd -h
-Usage: zstd *OPTION*... *FILE*... *-o file*
-Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).
+Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
 
-  -o file              result stored into `file` (only 1 output file)
-  -1 .. -19            compression level (faster .. better; default: 3)
-  -d, --decompress     decompression
-  -f, --force          disable input and output checks. Allows overwriting existing files,
-                       input from console, output to stdout, operating on links,
-                       block devices, etc. During decompression and when the output
-                       destination is stdout, pass-through unrecognized formats as-is.
-      --rm             remove source file(s) after successful de/compression
-  -k, --keep           preserve source file(s) (default)
-  -D DICT              use DICT as Dictionary for compression or decompression
-  -h                   display usage and exit
-  -H,--help            display long help and exit
+Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
+
+Options:
+  -o OUTPUT                     Write output to a single file, OUTPUT.
+  -k, --keep                    Preserve INPUT file(s). *Default*
+  --rm                          Remove INPUT file(s) after successful (de)compression.
+
+  -#                            Desired compression level, where `#` is a number between 1 and 19;
+                                lower numbers provide faster compression, higher numbers yield
+                                better compression ratios. *Default: 3*
+
+  -d, --decompress              Perform decompression.
+  -D DICT                       Use DICT as the dictionary for compression or decompression.
+
+  -f, --force                   Disable input and output checks. Allows overwriting existing files,
+                                receiving input from the console, printing ouput to STDOUT, and
+                                operating on links, block devices, etc. Unrecognized formats will be
+                                passed-through through as-is.
+
+  -h                            Display short usage and exit.
+  -H, --help                    Display full help and exit.
+  -V, --version                 Display the program version and exit.
 
 + zstd -H
 ...
-Advanced options :
+Advanced options:
 ...
 + zstd --help
 ...
-Advanced options :
+Advanced options:
 ...
diff --git a/tests/cli-tests/basic/version.sh.stdout.glob b/tests/cli-tests/basic/version.sh.stdout.glob
index 54968fa41..4cc9fb9b2 100644
--- a/tests/cli-tests/basic/version.sh.stdout.glob
+++ b/tests/cli-tests/basic/version.sh.stdout.glob
@@ -1,2 +1,2 @@
-*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
-*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh
index 4837790ce..47ac2029f 100755
--- a/tests/cli-tests/compression/levels.sh
+++ b/tests/cli-tests/compression/levels.sh
@@ -10,11 +10,9 @@ zstd --fast=10 file -o file-f10.zst
 zstd --fast=1 file -o file-f1.zst
 zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
-zstd -22 --ultra file -o file-22.zst
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
 
-cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst
diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact
index cb00433e6..5664f854b 100644
--- a/tests/cli-tests/compression/levels.sh.stderr.exact
+++ b/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -6,11 +6,9 @@ zstd --fast=10 file -o file-f10.zst
 zstd --fast=1 file -o file-f1.zst
 zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
-zstd -22 --ultra file -o file-22.zst
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
 
-cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh b/tests/cli-tests/file-stat/compress-file-to-file.sh
new file mode 100755
index 000000000..949f34ff1
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
new file mode 100644
index 000000000..34bb704c7
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -0,0 +1,36 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_isDirectory(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file, file.zst)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_utime(file.zst)
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh b/tests/cli-tests/file-stat/compress-file-to-stdout.sh
new file mode 100755
index 000000000..99ebfc46f
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..e01922f84
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,22 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_isDirectory(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-file.sh b/tests/cli-tests/file-stat/compress-stdin-to-file.sh
new file mode 100755
index 000000000..837946198
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 000000000..dce9005f4
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,28 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
new file mode 100755
index 000000000..64f4b0362
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..d47bdcf09
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,20 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh b/tests/cli-tests/file-stat/decompress-file-to-file.sh
new file mode 100755
index 000000000..3e08c247b
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dq --trace-file-stat file.zst
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
new file mode 100644
index 000000000..a2d7ad282
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -0,0 +1,30 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file.zst, file)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_utime(file)
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
new file mode 100755
index 000000000..518c2a950
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat file.zst > file
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..87fc5e97c
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,14 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-file.sh b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
new file mode 100755
index 000000000..135d75524
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst -o file
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 000000000..749fd395a
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,20 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
new file mode 100755
index 000000000..495f07b17
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst > file
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..61487f61e
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,12 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/progress/no-progress.sh.stderr.glob b/tests/cli-tests/progress/no-progress.sh.stderr.glob
index f07ad3803..b4262517a 100644
--- a/tests/cli-tests/progress/no-progress.sh.stderr.glob
+++ b/tests/cli-tests/progress/no-progress.sh.stderr.glob
@@ -55,36 +55,36 @@ decompress 2 files
 
 args = --no-progress --fake-stderr-is-console -v
 compress file to file
-*zstd*
+*Zstandard CLI*
 hello*hello.zst*
 compress pipe to pipe
-*zstd*
+*Zstandard CLI*
 *stdin*stdout*
 compress pipe to file
-*zstd*
+*Zstandard CLI*
 *stdin*hello.zst*
 compress file to pipe
-*zstd*
+*Zstandard CLI*
 *hello*stdout*
 compress 2 files
-*zstd*
+*Zstandard CLI*
 *hello*hello.zst*
 *world*world.zst*
 2 files compressed*
 decompress file to file
-*zstd*
+*Zstandard CLI*
 hello.zst*
 decompress pipe to pipe
-*zstd*
+*Zstandard CLI*
 *stdin*
 decompress pipe to file
-*zstd*
+*Zstandard CLI*
 *stdin*
 decompress file to pipe
-*zstd*
+*Zstandard CLI*
 hello.zst*
 decompress 2 files
-*zstd*
+*Zstandard CLI*
 hello.zst*
 world.zst*
 2 files decompressed*
diff --git a/tests/cli-tests/run.py b/tests/cli-tests/run.py
index d726fba7f..8b4e21a53 100755
--- a/tests/cli-tests/run.py
+++ b/tests/cli-tests/run.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -209,6 +209,7 @@ class Options:
         preserve: bool,
         scratch_dir: str,
         test_dir: str,
+        set_exact_output: bool,
     ) -> None:
         self.env = env
         self.timeout = timeout
@@ -216,6 +217,7 @@ class Options:
         self.preserve = preserve
         self.scratch_dir = scratch_dir
         self.test_dir = test_dir
+        self.set_exact_output = set_exact_output
 
 
 class TestCase:
@@ -335,7 +337,7 @@ class TestCase:
             self._test_stdin.close()
             self._test_stdin = None
 
-    def _check_output_exact(self, out_name: str, expected: bytes) -> None:
+    def _check_output_exact(self, out_name: str, expected: bytes, exact_name: str) -> None:
         """
         Check the output named :out_name: for an exact match against the :expected: content.
         Saves the success and message.
@@ -349,6 +351,10 @@ class TestCase:
             self._success[check_name] = False
             self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{diff(expected, actual)}"
 
+            if self._opts.set_exact_output:
+                with open(exact_name, "wb") as f:
+                    f.write(actual)
+
     def _check_output_glob(self, out_name: str, expected: bytes) -> None:
         """
         Check the output named :out_name: for a glob match against the :expected: glob.
@@ -386,7 +392,7 @@ class TestCase:
         ignore_name = f"{self._test_file}.{out_name}.ignore"
 
         if os.path.exists(exact_name):
-            return self._check_output_exact(out_name, read_file(exact_name))
+            return self._check_output_exact(out_name, read_file(exact_name), exact_name)
         elif os.path.exists(glob_name):
             return self._check_output_glob(out_name, read_file(glob_name))
         elif os.path.exists(ignore_name):
@@ -394,7 +400,7 @@ class TestCase:
             self._success[check_name] = True
             self._message[check_name] = f"{out_name} ignored!"
         else:
-            return self._check_output_exact(out_name, bytes())
+            return self._check_output_exact(out_name, bytes(), exact_name)
 
     def _check_stderr(self) -> None:
         """Checks the stderr output against the expectation."""
@@ -678,6 +684,11 @@ if __name__ == "__main__":
             "Scratch directory located in TEST_DIR/scratch/."
         )
     )
+    parser.add_argument(
+        "--set-exact-output",
+        action="store_true",
+        help="Set stderr.exact and stdout.exact for all failing tests, unless .ignore or .glob already exists"
+    )
     parser.add_argument(
         "tests",
         nargs="*",
@@ -714,6 +725,7 @@ if __name__ == "__main__":
         preserve=args.preserve,
         test_dir=args.test_dir,
         scratch_dir=scratch_dir,
+        set_exact_output=args.set_exact_output,
     )
 
     if len(args.tests) == 0:
diff --git a/tests/datagencli.c b/tests/datagencli.c
index 7300fdb76..09ec5e9ae 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index d2b126f39..20ad5add0 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -136,7 +136,7 @@ static void RAND_genDist(U32* seed, BYTE* dist, double weight)
     BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */
 
     while (i < DISTSIZE) {
-        size_t states = ((size_t)(weight * statesLeft)) + 1;
+        size_t states = ((size_t)(weight * (double)statesLeft)) + 1;
         size_t j;
         for (j = 0; j < states && i < DISTSIZE; j++, i++) {
             dist[i] = symb;
@@ -662,11 +662,11 @@ generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
          * ensure nice numbers */
         U32 matchLen =
                 MIN_SEQ_LEN +
-                ROUND(RAND_exp(seed, excessMatch / (double)(numSequences - i)));
+                ROUND(RAND_exp(seed, (double)excessMatch / (double)(numSequences - i)));
         U32 literalLen =
                 (RAND(seed) & 7)
                         ? ROUND(RAND_exp(seed,
-                                         literalsSize /
+                                         (double)literalsSize /
                                                  (double)(numSequences - i)))
                         : 0;
         /* actual offset, code to send, and point to copy up to when shifting
diff --git a/tests/external_matchfinder.c b/tests/external_matchfinder.c
new file mode 100644
index 000000000..8ae76d519
--- /dev/null
+++ b/tests/external_matchfinder.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "external_matchfinder.h"
+#include <string.h>
+#include "zstd_compress_internal.h"
+
+#define HSIZE 1024
+static U32 const HLOG = 10;
+static U32 const MLS = 4;
+static U32 const BADIDX = 0xffffffff;
+
+static size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    size_t seqCount = 0;
+    U32 hashTable[HSIZE];
+
+    (void)externalMatchState;
+    (void)dict;
+    (void)dictSize;
+    (void)outSeqsCapacity;
+    (void)compressionLevel;
+
+    {   int i;
+        for (i=0; i < HSIZE; i++) {
+            hashTable[i] = BADIDX;
+    }   }
+
+    while (ip + MLS < iend) {
+        size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS);
+        U32 const matchIndex = hashTable[hash];
+        hashTable[hash] = (U32)(ip - istart);
+
+        if (matchIndex != BADIDX) {
+            const BYTE* const match = istart + matchIndex;
+            U32 const matchLen = (U32)ZSTD_count(ip, match, iend);
+            if (matchLen >= ZSTD_MINMATCH_MIN) {
+                U32 const litLen = (U32)(ip - anchor);
+                U32 const offset = (U32)(ip - match);
+                ZSTD_Sequence const seq = {
+                    offset, litLen, matchLen, 0
+                };
+
+                /* Note: it's crucial to stay within the window size! */
+                if (offset <= windowSize) {
+                    outSeqs[seqCount++] = seq;
+                    ip += matchLen;
+                    anchor = ip;
+                    continue;
+                }
+            }
+        }
+
+        ip++;
+    }
+
+    {   ZSTD_Sequence const finalSeq = {
+            0, (U32)(iend - anchor), 0, 0
+        };
+        outSeqs[seqCount++] = finalSeq;
+    }
+
+    return seqCount;
+}
+
+size_t zstreamExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    EMF_testCase const testCase = *((EMF_testCase*)externalMatchState);
+    memset(outSeqs, 0, outSeqsCapacity);
+
+    switch (testCase) {
+        case EMF_ZERO_SEQS:
+            return 0;
+        case EMF_ONE_BIG_SEQ:
+            outSeqs[0].offset = 0;
+            outSeqs[0].matchLength = 0;
+            outSeqs[0].litLength = (U32)(srcSize);
+            return 1;
+         case EMF_LOTS_OF_SEQS:
+            return simpleExternalMatchFinder(
+                externalMatchState,
+                outSeqs, outSeqsCapacity,
+                src, srcSize,
+                dict, dictSize,
+                compressionLevel,
+                windowSize
+            );
+        case EMF_SMALL_ERROR:
+            return outSeqsCapacity + 1;
+        case EMF_BIG_ERROR:
+        default:
+            return ZSTD_EXTERNAL_MATCHFINDER_ERROR;
+    }
+}
diff --git a/tests/external_matchfinder.h b/tests/external_matchfinder.h
new file mode 100644
index 000000000..041f73e4d
--- /dev/null
+++ b/tests/external_matchfinder.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef EXTERNAL_MATCHFINDER
+#define EXTERNAL_MATCHFINDER
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+/* See external_matchfinder.c for details on each test case */
+typedef enum {
+    EMF_ZERO_SEQS = 0,
+    EMF_ONE_BIG_SEQ = 1,
+    EMF_LOTS_OF_SEQS = 2,
+    EMF_BIG_ERROR = 3,
+    EMF_SMALL_ERROR = 4
+} EMF_testCase;
+
+size_t zstreamExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+#endif // EXTERNAL_MATCHFINDER
diff --git a/tests/fullbench.c b/tests/fullbench.c
index b55ff767f..25c6d2b40 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 912348c38..35a19dd89 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c
index bdbf76916..e4767b3f5 100644
--- a/tests/fuzz/block_decompress.c
+++ b/tests/fuzz/block_decompress.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c
index 46a84c7b9..54012a1da 100644
--- a/tests/fuzz/block_round_trip.c
+++ b/tests/fuzz/block_round_trip.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/decompress_dstSize_tooSmall.c b/tests/fuzz/decompress_dstSize_tooSmall.c
index 3f7607b29..8ad3ea1c9 100644
--- a/tests/fuzz/decompress_dstSize_tooSmall.c
+++ b/tests/fuzz/decompress_dstSize_tooSmall.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c
index 33c58c8bd..8dea3e560 100644
--- a/tests/fuzz/dictionary_decompress.c
+++ b/tests/fuzz/dictionary_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_loader.c b/tests/fuzz/dictionary_loader.c
index 5b60bc484..1ac22740f 100644
--- a/tests/fuzz/dictionary_loader.c
+++ b/tests/fuzz/dictionary_loader.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c
index 0b20e8d67..6dd78c3f3 100644
--- a/tests/fuzz/dictionary_round_trip.c
+++ b/tests/fuzz/dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_stream_round_trip.c b/tests/fuzz/dictionary_stream_round_trip.c
index 9af712f0e..3ebbd843b 100644
--- a/tests/fuzz/dictionary_stream_round_trip.c
+++ b/tests/fuzz/dictionary_stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fse_read_ncount.c b/tests/fuzz/fse_read_ncount.c
index c32386099..29e19442a 100644
--- a/tests/fuzz/fse_read_ncount.c
+++ b/tests/fuzz/fse_read_ncount.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h
index 6f3fb2994..d1e439fef 100644
--- a/tests/fuzz/fuzz.h
+++ b/tests/fuzz/fuzz.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 17eac4f6f..03ffeeefd 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c
index eae8ee4b6..a93e8ba95 100644
--- a/tests/fuzz/fuzz_data_producer.c
+++ b/tests/fuzz/fuzz_data_producer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h
index 62771a9f8..8ca501f9b 100644
--- a/tests/fuzz/fuzz_data_producer.h
+++ b/tests/fuzz/fuzz_data_producer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_helpers.c b/tests/fuzz/fuzz_helpers.c
index 61c0debe5..1b6ad9736 100644
--- a/tests/fuzz/fuzz_helpers.c
+++ b/tests/fuzz/fuzz_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h
index c180478e9..aaf4c1df4 100644
--- a/tests/fuzz/fuzz_helpers.h
+++ b/tests/fuzz/fuzz_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/huf_decompress.c b/tests/fuzz/huf_decompress.c
index fea09fc93..13f040a17 100644
--- a/tests/fuzz/huf_decompress.c
+++ b/tests/fuzz/huf_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/huf_round_trip.c b/tests/fuzz/huf_round_trip.c
index 32b08d6d5..0993ecd14 100644
--- a/tests/fuzz/huf_round_trip.c
+++ b/tests/fuzz/huf_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/raw_dictionary_round_trip.c b/tests/fuzz/raw_dictionary_round_trip.c
index 0e6517654..7ceab2be6 100644
--- a/tests/fuzz/raw_dictionary_round_trip.c
+++ b/tests/fuzz/raw_dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c
index e6d2dec64..550c65d86 100644
--- a/tests/fuzz/regression_driver.c
+++ b/tests/fuzz/regression_driver.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/seekable_roundtrip.c b/tests/fuzz/seekable_roundtrip.c
index dcdcaae11..6f0aa288f 100644
--- a/tests/fuzz/seekable_roundtrip.c
+++ b/tests/fuzz/seekable_roundtrip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c
index cc872a081..ec8fef488 100644
--- a/tests/fuzz/sequence_compression_api.c
+++ b/tests/fuzz/sequence_compression_api.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c
index 8e6980b35..c9fea22d3 100644
--- a/tests/fuzz/simple_compress.c
+++ b/tests/fuzz/simple_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c
index dfff11cbc..ce5f9f098 100644
--- a/tests/fuzz/simple_decompress.c
+++ b/tests/fuzz/simple_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 9da986bc9..23a805af2 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c
index 86a39b8c9..0254d06ec 100644
--- a/tests/fuzz/stream_decompress.c
+++ b/tests/fuzz/stream_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index 719eac48d..8a28907b6 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c
index 9ce645d15..95dbdd49a 100644
--- a/tests/fuzz/zstd_frame_info.c
+++ b/tests/fuzz/zstd_frame_info.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 08ce70dd7..d5210627e 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h
index 9fbefdc72..a4cfe321b 100644
--- a/tests/fuzz/zstd_helpers.h
+++ b/tests/fuzz/zstd_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 879e537bc..3ad8ced5e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -82,8 +82,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 void FUZ_bug976(void);
 void FUZ_bug976(void)
 {   /* these constants shall not depend on MIN() macro */
-    assert(ZSTD_HASHLOG_MAX < 31);
-    assert(ZSTD_CHAINLOG_MAX < 31);
+    DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
+    DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
 }
 
 
@@ -118,23 +118,24 @@ static U32 FUZ_highbit32(U32 v32)
 /*=============================================
 *   Test macros
 =============================================*/
-#define CHECK_Z(f) {                               \
-    size_t const err = f;                          \
-    if (ZSTD_isError(err)) {                       \
-        DISPLAY("Error => %s : %s ",               \
-                #f, ZSTD_getErrorName(err));       \
-        exit(1);                                   \
+#define CHECK(fn)  { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
+
+#define CHECK_Z(f) {                          \
+    size_t const err = f;                     \
+    if (ZSTD_isError(err)) {                  \
+        DISPLAY("Error => %s : %s ",          \
+                #f, ZSTD_getErrorName(err));  \
+        exit(1);                              \
 }   }
 
-#define CHECK_VAR(var, fn)  var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
+#define CHECK_VAR(var, fn)  var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
 #define CHECK_NEWV(var, fn)  size_t const CHECK_VAR(var, fn)
-#define CHECK(fn)  { CHECK_NEWV(__err, fn); }
 #define CHECKPLUS(var, fn, more)  { CHECK_NEWV(var, fn); more; }
 
 #define CHECK_OP(op, lhs, rhs) {                                  \
     if (!((lhs) op (rhs))) {                                      \
         DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs);  \
-        goto _output_error;                                       \
+         exit(1);                                                 \
     }                                                             \
 }
 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
@@ -338,6 +339,7 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
 }
 
 #ifdef ZSTD_MULTITHREAD
+
 typedef struct {
     ZSTD_CCtx* cctx;
     ZSTD_threadPool* pool;
@@ -429,8 +431,8 @@ static int threadPoolTests(void) {
 
         ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
         ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
-        ZSTD_pthread_join(t1, NULL);
-        ZSTD_pthread_join(t2, NULL);
+        ZSTD_pthread_join(t1);
+        ZSTD_pthread_join(t2);
 
         assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
         free(decodedBuffer2);
@@ -461,6 +463,101 @@ _output_error:
 *   Unit tests
 =============================================*/
 
+static void test_compressBound(unsigned tnb)
+{
+    DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
+
+    /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
+     * for a large range of known valid values */
+    DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
+    {   int s;
+        for (s=0; s<30; s++) {
+            size_t const w = (size_t)1 << s;
+            CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
+    }   }
+
+    // Ensure error if srcSize too big
+    {   size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
+        CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
+        CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
+    }
+
+    DISPLAYLEVEL(3, "OK \n");
+}
+
+static void test_decompressBound(unsigned tnb)
+{
+    DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
+
+    // Simple compression, with size : should provide size;
+    {   const char example[] = "abcd";
+        char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
+        size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
+        CHECK_Z(cSize);
+        CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
+    }
+
+    // Simple small compression without size : should provide 1 block size
+    {   char cBuffer[ZSTD_COMPRESSBOUND(0)];
+        ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
+        ZSTD_inBuffer in = { NULL, 0, 0 };
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        assert(cctx);
+        CHECK_Z( ZSTD_initCStream(cctx, 0) );
+        CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
+        CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
+        CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
+        ZSTD_freeCCtx(cctx);
+    }
+
+    // Attempt to overflow 32-bit intermediate multiplication result
+    // This requires dBound >= 4 GB, aka 2^32.
+    // This requires 2^32 / 2^17 = 2^15 blocks
+    // => create 2^15 blocks (can be empty, or just 1 byte).
+    {   const char input[] = "a";
+        size_t const nbBlocks = (1 << 15) + 1;
+        size_t blockNb;
+        size_t const outCapacity = 1 << 18; // large margin
+        char* const outBuffer = malloc (outCapacity);
+        ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        assert(cctx);
+        assert(outBuffer);
+        CHECK_Z( ZSTD_initCStream(cctx, 0) );
+        for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+            ZSTD_inBuffer in = { input, sizeof(input), 0 };
+            CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
+            CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
+        }
+        CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
+        CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000LLU /* 4 GB */ );
+        ZSTD_freeCCtx(cctx);
+        free(outBuffer);
+    }
+
+    DISPLAYLEVEL(3, "OK \n");
+}
+
+static void test_setCParams(unsigned tnb)
+{
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    ZSTD_compressionParameters cparams;
+    assert(cctx);
+
+    DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
+
+    /* valid cparams */
+    cparams = ZSTD_getCParams(1, 0, 0);
+    CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
+
+    /* invalid cparams (must fail) */
+    cparams.windowLog = 99;
+    CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
+
+    free(cctx);
+    DISPLAYLEVEL(3, "OK \n");
+}
+
 static int basicUnitTests(U32 const seed, double compressibility)
 {
     size_t const CNBuffSize = 5 MB;
@@ -507,6 +604,12 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "%u (OK) \n", vn);
     }
 
+    test_compressBound(testNb++);
+
+    test_decompressBound(testNb++);
+
+    test_setCParams(testNb++);
+
     DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
     {
         ZSTD_compressionParameters params;
@@ -1579,7 +1682,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             DISPLAYLEVEL(3, "OK \n");
 
             DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
-            CHECK( ZSTD_compressBegin(staticCCtx, 1) );
+            CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
             DISPLAYLEVEL(3, "OK \n");
 
             DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
@@ -1647,8 +1750,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
             testResult = 1;
             goto _end;
         }
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
@@ -1678,9 +1781,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
-        CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
         CHECK_VAR(cSize, ZSTD_compress2(mtctx,
                                 compressedBuffer, compressedBufferSize,
                                 CNBuffer, CNBuffSize) );
@@ -1699,11 +1802,11 @@ static int basicUnitTests(U32 const seed, double compressibility)
         ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         char out[32];
         if (cctx == NULL || dctx == NULL) goto _output_error;
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
         CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
         DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
 
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
         {   char const* outPtr = out;
             ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
             ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
@@ -1718,9 +1821,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
     {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
         cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
-        CHECK(cSize);
+        CHECK_Z(cSize);
         ZSTD_freeCCtx(cctx);
     }
     DISPLAYLEVEL(3, "OK \n");
@@ -1728,13 +1831,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
     DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
     {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
         size_t cSize1, cSize2;
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
         cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
-        CHECK(cSize1);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
+        CHECK_Z(cSize1);
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
         cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
-        CHECK(cSize2);
+        CHECK_Z(cSize2);
         CHECK_LT(cSize1, cSize2);
         ZSTD_freeCCtx(cctx);
     }
@@ -1745,10 +1848,10 @@ static int basicUnitTests(U32 const seed, double compressibility)
         /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
          * ZSTDMT is forced to not take the shortcut.
          */
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
-        CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
+        CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
         ZSTD_freeCCtx(cctx);
     }
     DISPLAYLEVEL(3, "OK \n");
@@ -1758,22 +1861,22 @@ static int basicUnitTests(U32 const seed, double compressibility)
         int const jobSize = 512 KB;
         int value;
         /* Check that the overlap log and job size are unset. */
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 0);
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, 0);
         /* Set and check the overlap log and job size. */
-        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
-        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
+        CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 5);
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, jobSize);
         /* Set the number of workers and check the overlap log and job size. */
-        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 5);
-        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, jobSize);
         ZSTD_freeCCtxParams(params);
     }
@@ -1884,8 +1987,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
-        CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
-        CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
+        CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
+        CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
@@ -1945,8 +2048,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
         {   size_t const testSize = CNBuffSize / 3;
-            CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
-            CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
+            CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
+            CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
 
             CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
                                           (const char*)CNBuffer + dictSize, testSize) );
@@ -2780,7 +2883,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const wrongSrcSize = (srcSize + 1000);
         ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
         params.fParams.contentSizeFlag = 1;
-        CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
+        CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
         {   size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
             if (!ZSTD_isError(result)) goto _output_error;
             if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
@@ -2800,7 +2903,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
                                                      CNBuffer, srcSize, compressionLevel);
             if (ZSTD_isError(cSize_1pass)) goto _output_error;
 
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
             {   size_t const compressionResult = ZSTD_compress2(cctx,
                                     compressedBuffer, compressedBufferSize,
                                     CNBuffer, srcSize);
@@ -2819,13 +2922,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
         {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
             DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
             assert(cctx != NULL);
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
             {   size_t const compressedSize = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
                                 CNBuffer, inputSize);
-                CHECK(compressedSize);
+                CHECK_Z(compressedSize);
                 cSize = compressedSize;
                 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
             }
@@ -2835,13 +2938,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
             DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
             {   size_t const result = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
                                 CNBuffer, inputSize);
-                CHECK(result);
+                CHECK_Z(result);
                 if (result != cSize) goto _output_error;   /* must result in same compressed result, hence same size */
                 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error;  /* must result in exactly same content, hence same hash */
                 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
@@ -2856,7 +2959,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
         {   ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
-            CHECK(bounds.error);
+            CHECK_Z(bounds.error);
         }
         DISPLAYLEVEL(3, "OK \n");
 
@@ -2890,7 +2993,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
             ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
             size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2908,7 +3011,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_frameHeader zfh;
             size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
             if (zfhrt != 0) goto _output_error;
@@ -2930,7 +3033,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_inBuffer in = { CNBuffer, 0, 0 };
             ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
             size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2942,7 +3045,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
         /* one shot */
         {   size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
             if (result != 0) goto _output_error;
@@ -2977,13 +3080,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
             int check;
             if (ZSTD_isError(bounds.error))
                 continue;
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
             value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
-            CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2));
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
+            CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
             if (check != value2) goto _output_error;
-            CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
+            CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
             if (check != value1) goto _output_error;
         }
         ZSTD_freeDCtx(dctx);
@@ -3000,21 +3103,21 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin(cctx, 5) );
-        CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
+        CHECK_Z( ZSTD_compressBegin(cctx, 5) );
+        CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
         CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
-        CHECK( ZSTD_decompressBegin(dctx) );
+        CHECK_Z( ZSTD_decompressBegin(dctx) );
         { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
           if (r != blockSize) goto _output_error; }
         DISPLAYLEVEL(3, "OK \n");
 
         /* very long stream of block compression */
         DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin(cctx, -199) );  /* we just want to quickly overflow internal U32 index */
-        CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
+        CHECK_Z( ZSTD_compressBegin(cctx, -199) );  /* we just want to quickly overflow internal U32 index */
+        CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
         {   U64 const toCompress = 5000000000ULL;   /* > 4 GB */
             U64 compressed = 0;
             while (compressed < toCompress) {
@@ -3027,7 +3130,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* dictionary block compression */
         DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
+        CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
         CHECK_VAR(cSize,  ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
         RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed);  /* create a non-compressible second block */
         { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) );  /* for cctx history consistency */
@@ -3038,7 +3141,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
-        CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
+        CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
         {   CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
             if (r != blockSize) {
                 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
@@ -3057,8 +3160,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
         {   ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
             if (cdict==NULL) goto _output_error;
-            CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) );
-            CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
+            CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
             ZSTD_freeCDict(cdict);
         }
         DISPLAYLEVEL(3, "OK \n");
@@ -3227,7 +3330,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
         size_t nbSeq = 1;
         while (nbSeq <= maxNbSeq) {
-          CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
+          CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
           /* Check every sequence for the first 100, then skip more rapidly. */
           if (nbSeq < 100) {
             ++nbSeq;
@@ -3256,7 +3359,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const bound = ZSTD_compressBound(CNBuffSize);
         size_t size = 1;
         while (size <= CNBuffSize) {
-          CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
+          CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
           /* Check every size for the first 100, then skip more rapidly. */
           if (size < 100) {
             ++size;
@@ -3291,7 +3394,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             void* const outputBuffer = malloc(outputSize);
             ZSTD_CCtx* const cctx = ZSTD_createCCtx();
             if (!outputBuffer || !cctx) goto _output_error;
-            CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
+            CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
             free(outputBuffer);
             ZSTD_freeCCtx(cctx);
         }
@@ -3628,7 +3731,7 @@ static int longUnitTests(U32 const seed, double compressibility)
             while (approxIndex <= (maxIndex / 4) * 3) {
                 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
                 approxIndex += in.pos;
-                CHECK(in.pos == in.size);
+                CHECK_Z(in.pos == in.size);
                 in.pos = 0;
                 out.pos = 0;
             }
@@ -3654,7 +3757,7 @@ static int longUnitTests(U32 const seed, double compressibility)
             while (approxIndex <= maxIndex) {
                 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
                 approxIndex += in.pos;
-                CHECK(in.pos == in.size);
+                CHECK_Z(in.pos == in.size);
                 in.pos = 0;
                 out.pos = 0;
             }
@@ -3737,7 +3840,7 @@ static int longUnitTests(U32 const seed, double compressibility)
         RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
         RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
 
-        CHECK(cctx_params != NULL);
+        CHECK_Z(cctx_params != NULL);
 
         for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
             DISPLAYLEVEL(3, "\n    Testing with dictSize %u ", (U32)dictSize);
@@ -3780,11 +3883,6 @@ _end:
     free(compressedBuffer);
     free(decodedBuffer);
     return testResult;
-
-_output_error:
-    testResult = 1;
-    DISPLAY("Error detected in Unit tests ! \n");
-    goto _end;
 }
 
 
diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
index a50350f8d..cca310967 100644
--- a/tests/gzip/Makefile
+++ b/tests/gzip/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c
index b71f7419f..66caa9e99 100644
--- a/tests/invalidDictionaries.c
+++ b/tests/invalidDictionaries.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/legacy.c b/tests/legacy.c
index ac4938fe5..3be386495 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/longmatch.c b/tests/longmatch.c
index a171c0e4a..547b26190 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -36,27 +36,27 @@ compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size)
 int main(int argc, const char** argv)
 {
   ZSTD_CStream* ctx;
-  ZSTD_parameters params;
-  size_t rc;
-  unsigned windowLog;
+  unsigned windowLog = 18;
   (void)argc;
   (void)argv;
   /* Create stream */
-  ctx = ZSTD_createCStream();
+  ctx = ZSTD_createCCtx();
   if (!ctx) { return 1; }
   /* Set parameters */
-  memset(&params, 0, sizeof(params));
-  params.cParams.windowLog = 18;
-  params.cParams.chainLog = 13;
-  params.cParams.hashLog = 14;
-  params.cParams.searchLog = 1;
-  params.cParams.minMatch = 7;
-  params.cParams.targetLength = 16;
-  params.cParams.strategy = ZSTD_fast;
-  windowLog = params.cParams.windowLog;
-  /* Initialize stream */
-  rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0);
-  if (ZSTD_isError(rc)) { return 2; }
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, windowLog)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, 13)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, 14)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, 1)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, 7)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, 16)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, ZSTD_fast)))
+    return 2;
   {
     U64 compressed = 0;
     const U64 toCompress = ((U64)1) << 33;
@@ -97,5 +97,6 @@ int main(int argc, const char** argv)
     free(srcBuffer);
     free(dstBuffer);
   }
+  ZSTD_freeCCtx(ctx);
   return 0;
 }
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 756a43340..8971c65d6 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -566,10 +566,10 @@ resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_
     double cs = 0., ds = 0., rt, cm = 0.;
     const double r1 = 1, r2 = 0.1, rtr = 0.5;
     double ret;
-    if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; }
-    if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; }
-    if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; }
-    rt = ((double)srcSize / res.cSize);
+    if(target.cSpeed) { cs = (double)res.cSpeed / (double)target.cSpeed; }
+    if(target.dSpeed) { ds = (double)res.dSpeed / (double)target.dSpeed; }
+    if(target.cMem != (U32)-1) { cm = (double)target.cMem / (double)res.cMem; }
+    rt = ((double)srcSize / (double)res.cSize);
 
     ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr +
          (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
@@ -581,8 +581,8 @@ resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_
 static double
 resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes)
 {
-    double normalizedCSpeedGain1 = ((double)result1.cSpeed / lvlRes.cSpeed) - 1;
-    double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
+    double normalizedCSpeedGain1 = ((double)result1.cSpeed / (double)lvlRes.cSpeed) - 1;
+    double normalizedRatioGain1 = ((double)lvlRes.cSize / (double)result1.cSize) - 1;
     if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
@@ -854,7 +854,7 @@ BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize)
     }
 
     {   double const ratio = res.result.cSize ?
-                            (double)srcSize / res.result.cSize : 0;
+                            (double)srcSize / (double)res.result.cSize : 0;
         double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT;
         double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT;
 
@@ -937,7 +937,7 @@ BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, co
         }
         fprintf(f, "================================\n");
         fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
-            (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT);
+            (double)srcSize / (double)g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT);
 
 
         fprintf(f, "Overall Winner: \n");
@@ -977,7 +977,7 @@ BMK_print_cLevelEntry(FILE* f, const int cLevel,
     }
     /* print comment */
     {   double const ratio = result.cSize ?
-                            (double)srcSize / result.cSize : 0;
+                            (double)srcSize / (double)result.cSize : 0;
         double const cSpeedMBps = (double)result.cSpeed / MB_UNIT;
         double const dSpeedMBps = (double)result.dSpeed / MB_UNIT;
 
@@ -1726,19 +1726,19 @@ static int allBench(BMK_benchResult_t* resultPtr,
 
     /* calculate uncertainty in compression / decompression runs */
     if (benchres.cSpeed) {
-        U64 const loopDurationC = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
+        double const loopDurationC = (double)(((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
         uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
     }
 
     if (benchres.dSpeed) {
-        U64 const loopDurationD = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
+        double const loopDurationD = (double)(((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
         uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
     }
 
     /* optimistic assumption of benchres */
     {   BMK_benchResult_t resultMax = benchres;
-        resultMax.cSpeed = (unsigned long long)(resultMax.cSpeed * uncertaintyConstantC * VARIANCE);
-        resultMax.dSpeed = (unsigned long long)(resultMax.dSpeed * uncertaintyConstantD * VARIANCE);
+        resultMax.cSpeed = (unsigned long long)((double)resultMax.cSpeed * uncertaintyConstantC * VARIANCE);
+        resultMax.dSpeed = (unsigned long long)((double)resultMax.dSpeed * uncertaintyConstantD * VARIANCE);
 
         /* disregard infeasible results in feas mode */
         /* disregard if resultMax < winner in infeas mode */
@@ -1850,8 +1850,8 @@ static int BMK_seed(winnerInfo_t* winners,
 
         if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
             /* Validate solution is "good enough" */
-            double W_ratio = (double)buf.srcSize / testResult.cSize;
-            double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
+            double W_ratio = (double)buf.srcSize / (double)testResult.cSize;
+            double O_ratio = (double)buf.srcSize / (double)winners[cLevel].result.cSize;
             double W_ratioNote = log (W_ratio);
             double O_ratioNote = log (O_ratio);
             size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB);
@@ -1864,11 +1864,11 @@ static int BMK_seed(winnerInfo_t* winners,
             double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
             double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
 
-            double W_CSpeed_note = W_ratioNote * (double)( 30 + 10*cLevel) + log(testResult.cSpeed);
-            double O_CSpeed_note = O_ratioNote * (double)( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed);
+            double W_CSpeed_note = W_ratioNote * (double)( 30 + 10*cLevel) + log((double)testResult.cSpeed);
+            double O_CSpeed_note = O_ratioNote * (double)( 30 + 10*cLevel) + log((double)winners[cLevel].result.cSpeed);
 
-            double W_DSpeed_note = W_ratioNote * (double)( 20 + 2*cLevel) + log(testResult.dSpeed);
-            double O_DSpeed_note = O_ratioNote * (double)( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed);
+            double W_DSpeed_note = W_ratioNote * (double)( 20 + 2*cLevel) + log((double)testResult.dSpeed);
+            double O_DSpeed_note = O_ratioNote * (double)( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed);
 
             if (W_DMemUsed_note < O_DMemUsed_note) {
                 /* uses too much Decompression memory for too little benefit */
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 08f31c069..9e62722bf 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py
index 26293728e..3e65081e0 100755
--- a/tests/rateLimiter.py
+++ b/tests/rateLimiter.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/Makefile b/tests/regression/Makefile
index a440c6c94..ba8b43024 100644
--- a/tests/regression/Makefile
+++ b/tests/regression/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.c b/tests/regression/config.c
index 30d0ca5e2..1d86fff23 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.h b/tests/regression/config.h
index dd8893743..a4b542a90 100644
--- a/tests/regression/config.h
+++ b/tests/regression/config.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.c b/tests/regression/data.c
index 341b02d3b..43f085f2c 100644
--- a/tests/regression/data.c
+++ b/tests/regression/data.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.h b/tests/regression/data.h
index e54e6a1dc..a4ee92072 100644
--- a/tests/regression/data.h
+++ b/tests/regression/data.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/levels.h b/tests/regression/levels.h
index e98209d80..d15b12046 100644
--- a/tests/regression/levels.h
+++ b/tests/regression/levels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.c b/tests/regression/method.c
index 55b1154eb..f84a15ef3 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.h b/tests/regression/method.h
index 1a36a9399..8efdd33a0 100644
--- a/tests/regression/method.h
+++ b/tests/regression/method.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.c b/tests/regression/result.c
index 1f879c140..8ccb8751e 100644
--- a/tests/regression/result.c
+++ b/tests/regression/result.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.h b/tests/regression/result.h
index 197fa905d..8a761ea4d 100644
--- a/tests/regression/result.h
+++ b/tests/regression/result.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index dee7d5721..188eea900 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -11,10 +11,10 @@ silesia.tar,                        level 6,                            compress
 silesia.tar,                        level 7,                            compress simple,                    4576661
 silesia.tar,                        level 9,                            compress simple,                    4552899
 silesia.tar,                        level 13,                           compress simple,                    4502956
-silesia.tar,                        level 16,                           compress simple,                    4360527
-silesia.tar,                        level 19,                           compress simple,                    4266970
+silesia.tar,                        level 16,                           compress simple,                    4360546
+silesia.tar,                        level 19,                           compress simple,                    4265911
 silesia.tar,                        uncompressed literals,              compress simple,                    4854086
-silesia.tar,                        uncompressed literals optimal,      compress simple,                    4266970
+silesia.tar,                        uncompressed literals optimal,      compress simple,                    4265911
 silesia.tar,                        huffman literals,                   compress simple,                    6179047
 github.tar,                         level -5,                           compress simple,                    52115
 github.tar,                         level -3,                           compress simple,                    45678
@@ -28,10 +28,10 @@ github.tar,                         level 6,                            compress
 github.tar,                         level 7,                            compress simple,                    38110
 github.tar,                         level 9,                            compress simple,                    36760
 github.tar,                         level 13,                           compress simple,                    35501
-github.tar,                         level 16,                           compress simple,                    40471
-github.tar,                         level 19,                           compress simple,                    32149
+github.tar,                         level 16,                           compress simple,                    40466
+github.tar,                         level 19,                           compress simple,                    32276
 github.tar,                         uncompressed literals,              compress simple,                    38831
-github.tar,                         uncompressed literals optimal,      compress simple,                    32149
+github.tar,                         uncompressed literals optimal,      compress simple,                    32276
 github.tar,                         huffman literals,                   compress simple,                    42560
 silesia,                            level -5,                           compress cctx,                      6857372
 silesia,                            level -3,                           compress cctx,                      6503412
@@ -45,8 +45,8 @@ silesia,                            level 6,                            compress
 silesia,                            level 7,                            compress cctx,                      4566984
 silesia,                            level 9,                            compress cctx,                      4543018
 silesia,                            level 13,                           compress cctx,                      4493990
-silesia,                            level 16,                           compress cctx,                      4359864
-silesia,                            level 19,                           compress cctx,                      4296438
+silesia,                            level 16,                           compress cctx,                      4360041
+silesia,                            level 19,                           compress cctx,                      4296055
 silesia,                            long distance mode,                 compress cctx,                      4842075
 silesia,                            multithreaded,                      compress cctx,                      4842075
 silesia,                            multithreaded long distance mode,   compress cctx,                      4842075
@@ -55,7 +55,7 @@ silesia,                            small hash log,                     compress
 silesia,                            small chain log,                    compress cctx,                      4912197
 silesia,                            explicit params,                    compress cctx,                      4794052
 silesia,                            uncompressed literals,              compress cctx,                      4842075
-silesia,                            uncompressed literals optimal,      compress cctx,                      4296438
+silesia,                            uncompressed literals optimal,      compress cctx,                      4296055
 silesia,                            huffman literals,                   compress cctx,                      6172202
 silesia,                            multithreaded with advanced params, compress cctx,                      4842075
 github,                             level -5,                           compress cctx,                      204407
@@ -80,11 +80,11 @@ github,                             level 7,                            compress
 github,                             level 7 with dict,                  compress cctx,                      38755
 github,                             level 9,                            compress cctx,                      135122
 github,                             level 9 with dict,                  compress cctx,                      39398
-github,                             level 13,                           compress cctx,                      132729
+github,                             level 13,                           compress cctx,                      132878
 github,                             level 13 with dict,                 compress cctx,                      39948
-github,                             level 16,                           compress cctx,                      132729
+github,                             level 16,                           compress cctx,                      133209
 github,                             level 16 with dict,                 compress cctx,                      37568
-github,                             level 19,                           compress cctx,                      132729
+github,                             level 19,                           compress cctx,                      132879
 github,                             level 19 with dict,                 compress cctx,                      37567
 github,                             long distance mode,                 compress cctx,                      141069
 github,                             multithreaded,                      compress cctx,                      141069
@@ -94,7 +94,7 @@ github,                             small hash log,                     compress
 github,                             small chain log,                    compress cctx,                      139242
 github,                             explicit params,                    compress cctx,                      140932
 github,                             uncompressed literals,              compress cctx,                      136332
-github,                             uncompressed literals optimal,      compress cctx,                      132729
+github,                             uncompressed literals optimal,      compress cctx,                      132879
 github,                             huffman literals,                   compress cctx,                      175468
 github,                             multithreaded with advanced params, compress cctx,                      141069
 silesia,                            level -5,                           zstdcli,                            6857420
@@ -109,8 +109,8 @@ silesia,                            level 6,                            zstdcli,
 silesia,                            level 7,                            zstdcli,                            4567032
 silesia,                            level 9,                            zstdcli,                            4543066
 silesia,                            level 13,                           zstdcli,                            4494038
-silesia,                            level 16,                           zstdcli,                            4359912
-silesia,                            level 19,                           zstdcli,                            4296486
+silesia,                            level 16,                           zstdcli,                            4360089
+silesia,                            level 19,                           zstdcli,                            4296103
 silesia,                            long distance mode,                 zstdcli,                            4833785
 silesia,                            multithreaded,                      zstdcli,                            4842123
 silesia,                            multithreaded long distance mode,   zstdcli,                            4833785
@@ -134,14 +134,14 @@ silesia.tar,                        level 6,                            zstdcli,
 silesia.tar,                        level 7,                            zstdcli,                            4578719
 silesia.tar,                        level 9,                            zstdcli,                            4552903
 silesia.tar,                        level 13,                           zstdcli,                            4502960
-silesia.tar,                        level 16,                           zstdcli,                            4360531
-silesia.tar,                        level 19,                           zstdcli,                            4266974
+silesia.tar,                        level 16,                           zstdcli,                            4360550
+silesia.tar,                        level 19,                           zstdcli,                            4265915
 silesia.tar,                        no source size,                     zstdcli,                            4854160
 silesia.tar,                        long distance mode,                 zstdcli,                            4845745
 silesia.tar,                        multithreaded,                      zstdcli,                            4854164
 silesia.tar,                        multithreaded long distance mode,   zstdcli,                            4845745
 silesia.tar,                        small window log,                   zstdcli,                            7100701
-silesia.tar,                        small hash log,                     zstdcli,                            6529289
+silesia.tar,                        small hash log,                     zstdcli,                            6529264
 silesia.tar,                        small chain log,                    zstdcli,                            4917022
 silesia.tar,                        explicit params,                    zstdcli,                            4820713
 silesia.tar,                        uncompressed literals,              zstdcli,                            5122571
@@ -170,11 +170,11 @@ github,                             level 7,                            zstdcli,
 github,                             level 7 with dict,                  zstdcli,                            40745
 github,                             level 9,                            zstdcli,                            137122
 github,                             level 9 with dict,                  zstdcli,                            41393
-github,                             level 13,                           zstdcli,                            134729
+github,                             level 13,                           zstdcli,                            134878
 github,                             level 13 with dict,                 zstdcli,                            41900
-github,                             level 16,                           zstdcli,                            134729
+github,                             level 16,                           zstdcli,                            135209
 github,                             level 16 with dict,                 zstdcli,                            39577
-github,                             level 19,                           zstdcli,                            134729
+github,                             level 19,                           zstdcli,                            134879
 github,                             level 19 with dict,                 zstdcli,                            39576
 github,                             long distance mode,                 zstdcli,                            138332
 github,                             multithreaded,                      zstdcli,                            138332
@@ -184,7 +184,7 @@ github,                             small hash log,                     zstdcli,
 github,                             small chain log,                    zstdcli,                            138341
 github,                             explicit params,                    zstdcli,                            136197
 github,                             uncompressed literals,              zstdcli,                            167911
-github,                             uncompressed literals optimal,      zstdcli,                            159227
+github,                             uncompressed literals optimal,      zstdcli,                            154667
 github,                             huffman literals,                   zstdcli,                            144365
 github,                             multithreaded with advanced params, zstdcli,                            167911
 github.tar,                         level -5,                           zstdcli,                            52119
@@ -211,9 +211,9 @@ github.tar,                         level 9,                            zstdcli,
 github.tar,                         level 9 with dict,                  zstdcli,                            36632
 github.tar,                         level 13,                           zstdcli,                            35505
 github.tar,                         level 13 with dict,                 zstdcli,                            37134
-github.tar,                         level 16,                           zstdcli,                            40475
+github.tar,                         level 16,                           zstdcli,                            40470
 github.tar,                         level 16 with dict,                 zstdcli,                            33378
-github.tar,                         level 19,                           zstdcli,                            32153
+github.tar,                         level 19,                           zstdcli,                            32280
 github.tar,                         level 19 with dict,                 zstdcli,                            32716
 github.tar,                         no source size,                     zstdcli,                            38832
 github.tar,                         no source size with dict,           zstdcli,                            38004
@@ -248,8 +248,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced one pass,                  4505046
 silesia,                            level 12 row 2,                     advanced one pass,                  4503116
 silesia,                            level 13,                           advanced one pass,                  4493990
-silesia,                            level 16,                           advanced one pass,                  4359864
-silesia,                            level 19,                           advanced one pass,                  4296438
+silesia,                            level 16,                           advanced one pass,                  4360041
+silesia,                            level 19,                           advanced one pass,                  4296055
 silesia,                            no source size,                     advanced one pass,                  4842075
 silesia,                            long distance mode,                 advanced one pass,                  4833710
 silesia,                            multithreaded,                      advanced one pass,                  4842075
@@ -282,14 +282,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced one pass,                  4514049
 silesia.tar,                        level 12 row 2,                     advanced one pass,                  4513797
 silesia.tar,                        level 13,                           advanced one pass,                  4502956
-silesia.tar,                        level 16,                           advanced one pass,                  4360527
-silesia.tar,                        level 19,                           advanced one pass,                  4266970
+silesia.tar,                        level 16,                           advanced one pass,                  4360546
+silesia.tar,                        level 19,                           advanced one pass,                  4265911
 silesia.tar,                        no source size,                     advanced one pass,                  4854086
 silesia.tar,                        long distance mode,                 advanced one pass,                  4840452
 silesia.tar,                        multithreaded,                      advanced one pass,                  4854160
 silesia.tar,                        multithreaded long distance mode,   advanced one pass,                  4845741
 silesia.tar,                        small window log,                   advanced one pass,                  7100655
-silesia.tar,                        small hash log,                     advanced one pass,                  6529231
+silesia.tar,                        small hash log,                     advanced one pass,                  6529206
 silesia.tar,                        small chain log,                    advanced one pass,                  4917041
 silesia.tar,                        explicit params,                    advanced one pass,                  4806855
 silesia.tar,                        uncompressed literals,              advanced one pass,                  5122473
@@ -370,39 +370,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced one pass,                  39393
 github,                             level 9 with dict copy,             advanced one pass,                  39398
 github,                             level 9 with dict load,             advanced one pass,                  41710
-github,                             level 11 row 1,                     advanced one pass,                  135119
+github,                             level 11 row 1,                     advanced one pass,                  135367
 github,                             level 11 row 1 with dict dms,       advanced one pass,                  39671
 github,                             level 11 row 1 with dict dds,       advanced one pass,                  39671
 github,                             level 11 row 1 with dict copy,      advanced one pass,                  39651
 github,                             level 11 row 1 with dict load,      advanced one pass,                  41360
-github,                             level 11 row 2,                     advanced one pass,                  135119
+github,                             level 11 row 2,                     advanced one pass,                  135367
 github,                             level 11 row 2 with dict dms,       advanced one pass,                  39671
 github,                             level 11 row 2 with dict dds,       advanced one pass,                  39671
 github,                             level 11 row 2 with dict copy,      advanced one pass,                  39651
 github,                             level 11 row 2 with dict load,      advanced one pass,                  41360
-github,                             level 12 row 1,                     advanced one pass,                  134180
+github,                             level 12 row 1,                     advanced one pass,                  134402
 github,                             level 12 row 1 with dict dms,       advanced one pass,                  39677
 github,                             level 12 row 1 with dict dds,       advanced one pass,                  39677
 github,                             level 12 row 1 with dict copy,      advanced one pass,                  39677
 github,                             level 12 row 1 with dict load,      advanced one pass,                  41166
-github,                             level 12 row 2,                     advanced one pass,                  134180
+github,                             level 12 row 2,                     advanced one pass,                  134402
 github,                             level 12 row 2 with dict dms,       advanced one pass,                  39677
 github,                             level 12 row 2 with dict dds,       advanced one pass,                  39677
 github,                             level 12 row 2 with dict copy,      advanced one pass,                  39677
 github,                             level 12 row 2 with dict load,      advanced one pass,                  41166
-github,                             level 13,                           advanced one pass,                  132729
+github,                             level 13,                           advanced one pass,                  132878
 github,                             level 13 with dict,                 advanced one pass,                  39900
 github,                             level 13 with dict dms,             advanced one pass,                  39900
 github,                             level 13 with dict dds,             advanced one pass,                  39900
 github,                             level 13 with dict copy,            advanced one pass,                  39948
 github,                             level 13 with dict load,            advanced one pass,                  42624
-github,                             level 16,                           advanced one pass,                  132729
+github,                             level 16,                           advanced one pass,                  133209
 github,                             level 16 with dict,                 advanced one pass,                  37577
 github,                             level 16 with dict dms,             advanced one pass,                  37577
 github,                             level 16 with dict dds,             advanced one pass,                  37577
 github,                             level 16 with dict copy,            advanced one pass,                  37568
 github,                             level 16 with dict load,            advanced one pass,                  42338
-github,                             level 19,                           advanced one pass,                  132729
+github,                             level 19,                           advanced one pass,                  132879
 github,                             level 19 with dict,                 advanced one pass,                  37576
 github,                             level 19 with dict dms,             advanced one pass,                  37576
 github,                             level 19 with dict dds,             advanced one pass,                  37576
@@ -418,7 +418,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced one pass,                  136341
 github,                             explicit params,                    advanced one pass,                  137727
 github,                             uncompressed literals,              advanced one pass,                  165911
-github,                             uncompressed literals optimal,      advanced one pass,                  157227
+github,                             uncompressed literals optimal,      advanced one pass,                  152667
 github,                             huffman literals,                   advanced one pass,                  142365
 github,                             multithreaded with advanced params, advanced one pass,                  165911
 github.tar,                         level -5,                           advanced one pass,                  52115
@@ -521,13 +521,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced one pass,                  37220
 github.tar,                         level 13 with dict copy,            advanced one pass,                  37130
 github.tar,                         level 13 with dict load,            advanced one pass,                  36010
-github.tar,                         level 16,                           advanced one pass,                  40471
+github.tar,                         level 16,                           advanced one pass,                  40466
 github.tar,                         level 16 with dict,                 advanced one pass,                  33374
 github.tar,                         level 16 with dict dms,             advanced one pass,                  33206
 github.tar,                         level 16 with dict dds,             advanced one pass,                  33206
 github.tar,                         level 16 with dict copy,            advanced one pass,                  33374
 github.tar,                         level 16 with dict load,            advanced one pass,                  39081
-github.tar,                         level 19,                           advanced one pass,                  32149
+github.tar,                         level 19,                           advanced one pass,                  32276
 github.tar,                         level 19 with dict,                 advanced one pass,                  32712
 github.tar,                         level 19 with dict dms,             advanced one pass,                  32555
 github.tar,                         level 19 with dict dds,             advanced one pass,                  32555
@@ -566,8 +566,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced one pass small out,        4505046
 silesia,                            level 12 row 2,                     advanced one pass small out,        4503116
 silesia,                            level 13,                           advanced one pass small out,        4493990
-silesia,                            level 16,                           advanced one pass small out,        4359864
-silesia,                            level 19,                           advanced one pass small out,        4296438
+silesia,                            level 16,                           advanced one pass small out,        4360041
+silesia,                            level 19,                           advanced one pass small out,        4296055
 silesia,                            no source size,                     advanced one pass small out,        4842075
 silesia,                            long distance mode,                 advanced one pass small out,        4833710
 silesia,                            multithreaded,                      advanced one pass small out,        4842075
@@ -600,14 +600,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced one pass small out,        4514049
 silesia.tar,                        level 12 row 2,                     advanced one pass small out,        4513797
 silesia.tar,                        level 13,                           advanced one pass small out,        4502956
-silesia.tar,                        level 16,                           advanced one pass small out,        4360527
-silesia.tar,                        level 19,                           advanced one pass small out,        4266970
+silesia.tar,                        level 16,                           advanced one pass small out,        4360546
+silesia.tar,                        level 19,                           advanced one pass small out,        4265911
 silesia.tar,                        no source size,                     advanced one pass small out,        4854086
 silesia.tar,                        long distance mode,                 advanced one pass small out,        4840452
 silesia.tar,                        multithreaded,                      advanced one pass small out,        4854160
 silesia.tar,                        multithreaded long distance mode,   advanced one pass small out,        4845741
 silesia.tar,                        small window log,                   advanced one pass small out,        7100655
-silesia.tar,                        small hash log,                     advanced one pass small out,        6529231
+silesia.tar,                        small hash log,                     advanced one pass small out,        6529206
 silesia.tar,                        small chain log,                    advanced one pass small out,        4917041
 silesia.tar,                        explicit params,                    advanced one pass small out,        4806855
 silesia.tar,                        uncompressed literals,              advanced one pass small out,        5122473
@@ -688,39 +688,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced one pass small out,        39393
 github,                             level 9 with dict copy,             advanced one pass small out,        39398
 github,                             level 9 with dict load,             advanced one pass small out,        41710
-github,                             level 11 row 1,                     advanced one pass small out,        135119
+github,                             level 11 row 1,                     advanced one pass small out,        135367
 github,                             level 11 row 1 with dict dms,       advanced one pass small out,        39671
 github,                             level 11 row 1 with dict dds,       advanced one pass small out,        39671
 github,                             level 11 row 1 with dict copy,      advanced one pass small out,        39651
 github,                             level 11 row 1 with dict load,      advanced one pass small out,        41360
-github,                             level 11 row 2,                     advanced one pass small out,        135119
+github,                             level 11 row 2,                     advanced one pass small out,        135367
 github,                             level 11 row 2 with dict dms,       advanced one pass small out,        39671
 github,                             level 11 row 2 with dict dds,       advanced one pass small out,        39671
 github,                             level 11 row 2 with dict copy,      advanced one pass small out,        39651
 github,                             level 11 row 2 with dict load,      advanced one pass small out,        41360
-github,                             level 12 row 1,                     advanced one pass small out,        134180
+github,                             level 12 row 1,                     advanced one pass small out,        134402
 github,                             level 12 row 1 with dict dms,       advanced one pass small out,        39677
 github,                             level 12 row 1 with dict dds,       advanced one pass small out,        39677
 github,                             level 12 row 1 with dict copy,      advanced one pass small out,        39677
 github,                             level 12 row 1 with dict load,      advanced one pass small out,        41166
-github,                             level 12 row 2,                     advanced one pass small out,        134180
+github,                             level 12 row 2,                     advanced one pass small out,        134402
 github,                             level 12 row 2 with dict dms,       advanced one pass small out,        39677
 github,                             level 12 row 2 with dict dds,       advanced one pass small out,        39677
 github,                             level 12 row 2 with dict copy,      advanced one pass small out,        39677
 github,                             level 12 row 2 with dict load,      advanced one pass small out,        41166
-github,                             level 13,                           advanced one pass small out,        132729
+github,                             level 13,                           advanced one pass small out,        132878
 github,                             level 13 with dict,                 advanced one pass small out,        39900
 github,                             level 13 with dict dms,             advanced one pass small out,        39900
 github,                             level 13 with dict dds,             advanced one pass small out,        39900
 github,                             level 13 with dict copy,            advanced one pass small out,        39948
 github,                             level 13 with dict load,            advanced one pass small out,        42624
-github,                             level 16,                           advanced one pass small out,        132729
+github,                             level 16,                           advanced one pass small out,        133209
 github,                             level 16 with dict,                 advanced one pass small out,        37577
 github,                             level 16 with dict dms,             advanced one pass small out,        37577
 github,                             level 16 with dict dds,             advanced one pass small out,        37577
 github,                             level 16 with dict copy,            advanced one pass small out,        37568
 github,                             level 16 with dict load,            advanced one pass small out,        42338
-github,                             level 19,                           advanced one pass small out,        132729
+github,                             level 19,                           advanced one pass small out,        132879
 github,                             level 19 with dict,                 advanced one pass small out,        37576
 github,                             level 19 with dict dms,             advanced one pass small out,        37576
 github,                             level 19 with dict dds,             advanced one pass small out,        37576
@@ -736,7 +736,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced one pass small out,        136341
 github,                             explicit params,                    advanced one pass small out,        137727
 github,                             uncompressed literals,              advanced one pass small out,        165911
-github,                             uncompressed literals optimal,      advanced one pass small out,        157227
+github,                             uncompressed literals optimal,      advanced one pass small out,        152667
 github,                             huffman literals,                   advanced one pass small out,        142365
 github,                             multithreaded with advanced params, advanced one pass small out,        165911
 github.tar,                         level -5,                           advanced one pass small out,        52115
@@ -839,13 +839,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced one pass small out,        37220
 github.tar,                         level 13 with dict copy,            advanced one pass small out,        37130
 github.tar,                         level 13 with dict load,            advanced one pass small out,        36010
-github.tar,                         level 16,                           advanced one pass small out,        40471
+github.tar,                         level 16,                           advanced one pass small out,        40466
 github.tar,                         level 16 with dict,                 advanced one pass small out,        33374
 github.tar,                         level 16 with dict dms,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict dds,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict copy,            advanced one pass small out,        33374
 github.tar,                         level 16 with dict load,            advanced one pass small out,        39081
-github.tar,                         level 19,                           advanced one pass small out,        32149
+github.tar,                         level 19,                           advanced one pass small out,        32276
 github.tar,                         level 19 with dict,                 advanced one pass small out,        32712
 github.tar,                         level 19 with dict dms,             advanced one pass small out,        32555
 github.tar,                         level 19 with dict dds,             advanced one pass small out,        32555
@@ -884,8 +884,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced streaming,                 4505046
 silesia,                            level 12 row 2,                     advanced streaming,                 4503116
 silesia,                            level 13,                           advanced streaming,                 4493990
-silesia,                            level 16,                           advanced streaming,                 4359864
-silesia,                            level 19,                           advanced streaming,                 4296438
+silesia,                            level 16,                           advanced streaming,                 4360041
+silesia,                            level 19,                           advanced streaming,                 4296055
 silesia,                            no source size,                     advanced streaming,                 4842039
 silesia,                            long distance mode,                 advanced streaming,                 4833710
 silesia,                            multithreaded,                      advanced streaming,                 4842075
@@ -918,14 +918,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced streaming,                 4514049
 silesia.tar,                        level 12 row 2,                     advanced streaming,                 4513797
 silesia.tar,                        level 13,                           advanced streaming,                 4502956
-silesia.tar,                        level 16,                           advanced streaming,                 4360527
-silesia.tar,                        level 19,                           advanced streaming,                 4266970
+silesia.tar,                        level 16,                           advanced streaming,                 4360546
+silesia.tar,                        level 19,                           advanced streaming,                 4265911
 silesia.tar,                        no source size,                     advanced streaming,                 4859267
 silesia.tar,                        long distance mode,                 advanced streaming,                 4840452
 silesia.tar,                        multithreaded,                      advanced streaming,                 4854160
 silesia.tar,                        multithreaded long distance mode,   advanced streaming,                 4845741
 silesia.tar,                        small window log,                   advanced streaming,                 7117559
-silesia.tar,                        small hash log,                     advanced streaming,                 6529234
+silesia.tar,                        small hash log,                     advanced streaming,                 6529209
 silesia.tar,                        small chain log,                    advanced streaming,                 4917021
 silesia.tar,                        explicit params,                    advanced streaming,                 4806873
 silesia.tar,                        uncompressed literals,              advanced streaming,                 5127423
@@ -1006,39 +1006,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced streaming,                 39393
 github,                             level 9 with dict copy,             advanced streaming,                 39398
 github,                             level 9 with dict load,             advanced streaming,                 41710
-github,                             level 11 row 1,                     advanced streaming,                 135119
+github,                             level 11 row 1,                     advanced streaming,                 135367
 github,                             level 11 row 1 with dict dms,       advanced streaming,                 39671
 github,                             level 11 row 1 with dict dds,       advanced streaming,                 39671
 github,                             level 11 row 1 with dict copy,      advanced streaming,                 39651
 github,                             level 11 row 1 with dict load,      advanced streaming,                 41360
-github,                             level 11 row 2,                     advanced streaming,                 135119
+github,                             level 11 row 2,                     advanced streaming,                 135367
 github,                             level 11 row 2 with dict dms,       advanced streaming,                 39671
 github,                             level 11 row 2 with dict dds,       advanced streaming,                 39671
 github,                             level 11 row 2 with dict copy,      advanced streaming,                 39651
 github,                             level 11 row 2 with dict load,      advanced streaming,                 41360
-github,                             level 12 row 1,                     advanced streaming,                 134180
+github,                             level 12 row 1,                     advanced streaming,                 134402
 github,                             level 12 row 1 with dict dms,       advanced streaming,                 39677
 github,                             level 12 row 1 with dict dds,       advanced streaming,                 39677
 github,                             level 12 row 1 with dict copy,      advanced streaming,                 39677
 github,                             level 12 row 1 with dict load,      advanced streaming,                 41166
-github,                             level 12 row 2,                     advanced streaming,                 134180
+github,                             level 12 row 2,                     advanced streaming,                 134402
 github,                             level 12 row 2 with dict dms,       advanced streaming,                 39677
 github,                             level 12 row 2 with dict dds,       advanced streaming,                 39677
 github,                             level 12 row 2 with dict copy,      advanced streaming,                 39677
 github,                             level 12 row 2 with dict load,      advanced streaming,                 41166
-github,                             level 13,                           advanced streaming,                 132729
+github,                             level 13,                           advanced streaming,                 132878
 github,                             level 13 with dict,                 advanced streaming,                 39900
 github,                             level 13 with dict dms,             advanced streaming,                 39900
 github,                             level 13 with dict dds,             advanced streaming,                 39900
 github,                             level 13 with dict copy,            advanced streaming,                 39948
 github,                             level 13 with dict load,            advanced streaming,                 42624
-github,                             level 16,                           advanced streaming,                 132729
+github,                             level 16,                           advanced streaming,                 133209
 github,                             level 16 with dict,                 advanced streaming,                 37577
 github,                             level 16 with dict dms,             advanced streaming,                 37577
 github,                             level 16 with dict dds,             advanced streaming,                 37577
 github,                             level 16 with dict copy,            advanced streaming,                 37568
 github,                             level 16 with dict load,            advanced streaming,                 42338
-github,                             level 19,                           advanced streaming,                 132729
+github,                             level 19,                           advanced streaming,                 132879
 github,                             level 19 with dict,                 advanced streaming,                 37576
 github,                             level 19 with dict dms,             advanced streaming,                 37576
 github,                             level 19 with dict dds,             advanced streaming,                 37576
@@ -1054,7 +1054,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced streaming,                 136341
 github,                             explicit params,                    advanced streaming,                 137727
 github,                             uncompressed literals,              advanced streaming,                 165911
-github,                             uncompressed literals optimal,      advanced streaming,                 157227
+github,                             uncompressed literals optimal,      advanced streaming,                 152667
 github,                             huffman literals,                   advanced streaming,                 142365
 github,                             multithreaded with advanced params, advanced streaming,                 165911
 github.tar,                         level -5,                           advanced streaming,                 52152
@@ -1157,13 +1157,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced streaming,                 37220
 github.tar,                         level 13 with dict copy,            advanced streaming,                 37130
 github.tar,                         level 13 with dict load,            advanced streaming,                 36010
-github.tar,                         level 16,                           advanced streaming,                 40471
+github.tar,                         level 16,                           advanced streaming,                 40466
 github.tar,                         level 16 with dict,                 advanced streaming,                 33374
 github.tar,                         level 16 with dict dms,             advanced streaming,                 33206
 github.tar,                         level 16 with dict dds,             advanced streaming,                 33206
 github.tar,                         level 16 with dict copy,            advanced streaming,                 33374
 github.tar,                         level 16 with dict load,            advanced streaming,                 39081
-github.tar,                         level 19,                           advanced streaming,                 32149
+github.tar,                         level 19,                           advanced streaming,                 32276
 github.tar,                         level 19 with dict,                 advanced streaming,                 32712
 github.tar,                         level 19 with dict dms,             advanced streaming,                 32555
 github.tar,                         level 19 with dict dds,             advanced streaming,                 32555
@@ -1194,11 +1194,11 @@ silesia,                            level 6,                            old stre
 silesia,                            level 7,                            old streaming,                      4566984
 silesia,                            level 9,                            old streaming,                      4543018
 silesia,                            level 13,                           old streaming,                      4493990
-silesia,                            level 16,                           old streaming,                      4359864
-silesia,                            level 19,                           old streaming,                      4296438
+silesia,                            level 16,                           old streaming,                      4360041
+silesia,                            level 19,                           old streaming,                      4296055
 silesia,                            no source size,                     old streaming,                      4842039
 silesia,                            uncompressed literals,              old streaming,                      4842075
-silesia,                            uncompressed literals optimal,      old streaming,                      4296438
+silesia,                            uncompressed literals optimal,      old streaming,                      4296055
 silesia,                            huffman literals,                   old streaming,                      6172207
 silesia.tar,                        level -5,                           old streaming,                      6856523
 silesia.tar,                        level -3,                           old streaming,                      6505954
@@ -1212,11 +1212,11 @@ silesia.tar,                        level 6,                            old stre
 silesia.tar,                        level 7,                            old streaming,                      4576664
 silesia.tar,                        level 9,                            old streaming,                      4552900
 silesia.tar,                        level 13,                           old streaming,                      4502956
-silesia.tar,                        level 16,                           old streaming,                      4360527
-silesia.tar,                        level 19,                           old streaming,                      4266970
+silesia.tar,                        level 16,                           old streaming,                      4360546
+silesia.tar,                        level 19,                           old streaming,                      4265911
 silesia.tar,                        no source size,                     old streaming,                      4859267
 silesia.tar,                        uncompressed literals,              old streaming,                      4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming,                      4266970
+silesia.tar,                        uncompressed literals optimal,      old streaming,                      4265911
 silesia.tar,                        huffman literals,                   old streaming,                      6179056
 github,                             level -5,                           old streaming,                      204407
 github,                             level -5 with dict,                 old streaming,                      46718
@@ -1240,16 +1240,16 @@ github,                             level 7,                            old stre
 github,                             level 7 with dict,                  old streaming,                      38758
 github,                             level 9,                            old streaming,                      135122
 github,                             level 9 with dict,                  old streaming,                      39437
-github,                             level 13,                           old streaming,                      132729
+github,                             level 13,                           old streaming,                      132878
 github,                             level 13 with dict,                 old streaming,                      39900
-github,                             level 16,                           old streaming,                      132729
+github,                             level 16,                           old streaming,                      133209
 github,                             level 16 with dict,                 old streaming,                      37577
-github,                             level 19,                           old streaming,                      132729
+github,                             level 19,                           old streaming,                      132879
 github,                             level 19 with dict,                 old streaming,                      37576
 github,                             no source size,                     old streaming,                      140599
 github,                             no source size with dict,           old streaming,                      40654
 github,                             uncompressed literals,              old streaming,                      136332
-github,                             uncompressed literals optimal,      old streaming,                      132729
+github,                             uncompressed literals optimal,      old streaming,                      132879
 github,                             huffman literals,                   old streaming,                      175468
 github.tar,                         level -5,                           old streaming,                      52152
 github.tar,                         level -5 with dict,                 old streaming,                      51045
@@ -1275,14 +1275,14 @@ github.tar,                         level 9,                            old stre
 github.tar,                         level 9 with dict,                  old streaming,                      36484
 github.tar,                         level 13,                           old streaming,                      35501
 github.tar,                         level 13 with dict,                 old streaming,                      37130
-github.tar,                         level 16,                           old streaming,                      40471
+github.tar,                         level 16,                           old streaming,                      40466
 github.tar,                         level 16 with dict,                 old streaming,                      33374
-github.tar,                         level 19,                           old streaming,                      32149
+github.tar,                         level 19,                           old streaming,                      32276
 github.tar,                         level 19 with dict,                 old streaming,                      32712
 github.tar,                         no source size,                     old streaming,                      38828
 github.tar,                         no source size with dict,           old streaming,                      38000
 github.tar,                         uncompressed literals,              old streaming,                      38831
-github.tar,                         uncompressed literals optimal,      old streaming,                      32149
+github.tar,                         uncompressed literals optimal,      old streaming,                      32276
 github.tar,                         huffman literals,                   old streaming,                      42560
 silesia,                            level -5,                           old streaming advanced,             6854744
 silesia,                            level -3,                           old streaming advanced,             6503319
@@ -1296,8 +1296,8 @@ silesia,                            level 6,                            old stre
 silesia,                            level 7,                            old streaming advanced,             4566984
 silesia,                            level 9,                            old streaming advanced,             4543018
 silesia,                            level 13,                           old streaming advanced,             4493990
-silesia,                            level 16,                           old streaming advanced,             4359864
-silesia,                            level 19,                           old streaming advanced,             4296438
+silesia,                            level 16,                           old streaming advanced,             4360041
+silesia,                            level 19,                           old streaming advanced,             4296055
 silesia,                            no source size,                     old streaming advanced,             4842039
 silesia,                            long distance mode,                 old streaming advanced,             4842075
 silesia,                            multithreaded,                      old streaming advanced,             4842075
@@ -1307,7 +1307,7 @@ silesia,                            small hash log,                     old stre
 silesia,                            small chain log,                    old streaming advanced,             4912197
 silesia,                            explicit params,                    old streaming advanced,             4795452
 silesia,                            uncompressed literals,              old streaming advanced,             4842075
-silesia,                            uncompressed literals optimal,      old streaming advanced,             4296438
+silesia,                            uncompressed literals optimal,      old streaming advanced,             4296055
 silesia,                            huffman literals,                   old streaming advanced,             6172207
 silesia,                            multithreaded with advanced params, old streaming advanced,             4842075
 silesia.tar,                        level -5,                           old streaming advanced,             6856523
@@ -1322,18 +1322,18 @@ silesia.tar,                        level 6,                            old stre
 silesia.tar,                        level 7,                            old streaming advanced,             4576664
 silesia.tar,                        level 9,                            old streaming advanced,             4552900
 silesia.tar,                        level 13,                           old streaming advanced,             4502956
-silesia.tar,                        level 16,                           old streaming advanced,             4360527
-silesia.tar,                        level 19,                           old streaming advanced,             4266970
+silesia.tar,                        level 16,                           old streaming advanced,             4360546
+silesia.tar,                        level 19,                           old streaming advanced,             4265911
 silesia.tar,                        no source size,                     old streaming advanced,             4859267
 silesia.tar,                        long distance mode,                 old streaming advanced,             4859271
 silesia.tar,                        multithreaded,                      old streaming advanced,             4859271
 silesia.tar,                        multithreaded long distance mode,   old streaming advanced,             4859271
 silesia.tar,                        small window log,                   old streaming advanced,             7117562
-silesia.tar,                        small hash log,                     old streaming advanced,             6529234
+silesia.tar,                        small hash log,                     old streaming advanced,             6529209
 silesia.tar,                        small chain log,                    old streaming advanced,             4917021
 silesia.tar,                        explicit params,                    old streaming advanced,             4806873
 silesia.tar,                        uncompressed literals,              old streaming advanced,             4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4266970
+silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4265911
 silesia.tar,                        huffman literals,                   old streaming advanced,             6179056
 silesia.tar,                        multithreaded with advanced params, old streaming advanced,             4859271
 github,                             level -5,                           old streaming advanced,             213265
@@ -1360,9 +1360,9 @@ github,                             level 9,                            old stre
 github,                             level 9 with dict,                  old streaming advanced,             38981
 github,                             level 13,                           old streaming advanced,             138676
 github,                             level 13 with dict,                 old streaming advanced,             39725
-github,                             level 16,                           old streaming advanced,             138676
+github,                             level 16,                           old streaming advanced,             138575
 github,                             level 16 with dict,                 old streaming advanced,             40789
-github,                             level 19,                           old streaming advanced,             132729
+github,                             level 19,                           old streaming advanced,             132879
 github,                             level 19 with dict,                 old streaming advanced,             37576
 github,                             no source size,                     old streaming advanced,             140599
 github,                             no source size with dict,           old streaming advanced,             40608
@@ -1374,7 +1374,7 @@ github,                             small hash log,                     old stre
 github,                             small chain log,                    old streaming advanced,             139275
 github,                             explicit params,                    old streaming advanced,             140937
 github,                             uncompressed literals,              old streaming advanced,             141104
-github,                             uncompressed literals optimal,      old streaming advanced,             132729
+github,                             uncompressed literals optimal,      old streaming advanced,             132879
 github,                             huffman literals,                   old streaming advanced,             181107
 github,                             multithreaded with advanced params, old streaming advanced,             141104
 github.tar,                         level -5,                           old streaming advanced,             52152
@@ -1401,9 +1401,9 @@ github.tar,                         level 9,                            old stre
 github.tar,                         level 9 with dict,                  old streaming advanced,             36312
 github.tar,                         level 13,                           old streaming advanced,             35501
 github.tar,                         level 13 with dict,                 old streaming advanced,             35807
-github.tar,                         level 16,                           old streaming advanced,             40471
+github.tar,                         level 16,                           old streaming advanced,             40466
 github.tar,                         level 16 with dict,                 old streaming advanced,             38578
-github.tar,                         level 19,                           old streaming advanced,             32149
+github.tar,                         level 19,                           old streaming advanced,             32276
 github.tar,                         level 19 with dict,                 old streaming advanced,             32704
 github.tar,                         no source size,                     old streaming advanced,             38828
 github.tar,                         no source size with dict,           old streaming advanced,             38015
@@ -1415,7 +1415,7 @@ github.tar,                         small hash log,                     old stre
 github.tar,                         small chain log,                    old streaming advanced,             41669
 github.tar,                         explicit params,                    old streaming advanced,             41385
 github.tar,                         uncompressed literals,              old streaming advanced,             38831
-github.tar,                         uncompressed literals optimal,      old streaming advanced,             32149
+github.tar,                         uncompressed literals optimal,      old streaming advanced,             32276
 github.tar,                         huffman literals,                   old streaming advanced,             42560
 github.tar,                         multithreaded with advanced params, old streaming advanced,             38831
 github,                             level -5 with dict,                 old streaming cdict,                46718
diff --git a/tests/regression/test.c b/tests/regression/test.c
index 1de6be86c..07600be57 100644
--- a/tests/regression/test.c
+++ b/tests/regression/test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c
index 9aa208cff..77411cd49 100644
--- a/tests/roundTripCrash.c
+++ b/tests/roundTripCrash.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.c b/tests/seqgen.c
index 1e340c851..0d8a766c8 100644
--- a/tests/seqgen.c
+++ b/tests/seqgen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.h b/tests/seqgen.h
index cea3f552b..df17398ef 100644
--- a/tests/seqgen.h
+++ b/tests/seqgen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/test-license.py b/tests/test-license.py
index f27cb6774..d54c16419 100755
--- a/tests/test-license.py
+++ b/tests/test-license.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -83,8 +83,8 @@ def valid_copyright(lines):
             continue
         if "present" in line:
             return (False, f"Copyright line '{line}' contains 'present'!")
-        if "Facebook, Inc" not in line:
-            return (False, f"Copyright line '{line}' does not contain 'Facebook, Inc'")
+        if "Meta Platforms, Inc" not in line:
+            return (False, f"Copyright line '{line}' does not contain 'Meta Platforms, Inc'")
         year = YEAR_REGEX.search(line)
         if year is not None:
             return (False, f"Copyright line '{line}' contains {year.group(0)}; it should be yearless")
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index 94e3b8c5b..7117c1952 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -2,7 +2,7 @@
 """Test zstd interoperability between versions"""
 
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -23,7 +23,7 @@ from subprocess import Popen, PIPE
 repo_url = 'https://github.com/facebook/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
 make_cmd = 'make'
-make_args = ['-j','CFLAGS=-O1']
+make_args = ['-j','CFLAGS=-O0']
 git_cmd = 'git'
 test_dat_src = 'README.md'
 test_dat = 'test_dat'
@@ -86,41 +86,46 @@ def create_dict(tag, dict_source_path):
         if result == 0:
             print(dict_name + ' created')
         else:
-            print('ERROR: creating of ' + dict_name + ' failed')
+            raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
     else:
         print(dict_name + ' already exists')
 
 
+def zstd(tag, args, input_file, output_file):
+    """
+    Zstd compress input_file to output_file.
+    Need this helper because 0.5.0 is broken when stdout is not a TTY.
+    Throws an exception if the command returns non-zero.
+    """
+    with open(input_file, "rb") as i:
+        with open(output_file, "wb") as o:
+            cmd = ['./zstd.' + tag] + args
+            print("Running: '{}', input={}, output={}" .format(
+                ' '.join(cmd), input_file, output_file
+            ))
+            subprocess.check_call(cmd, stdin=i, stdout=o)
+
+
 def dict_compress_sample(tag, sample):
     dict_name = 'dict.' + tag
-    DEVNULL = open(os.devnull, 'wb')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-15'], sample, sample + '_15_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-18'], sample, sample + '_18_64_' + tag + '_dictio.zst')
     # zstdFiles = glob.glob("*.zst*")
     # print(zstdFiles)
     print(tag + " : dict compression completed")
 
 
 def compress_sample(tag, sample):
-    DEVNULL = open(os.devnull, 'wb')
-    if subprocess.call(['./zstd.' + tag, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-1'], sample, sample + '_01_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-3'], sample, sample + '_03_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-5'], sample, sample + '_05_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-9'], sample, sample + '_09_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-15'], sample, sample + '_15_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-18'], sample, sample + '_18_64_' + tag + '_nodict.zst')
     # zstdFiles = glob.glob("*.zst*")
     # print(zstdFiles)
     print(tag + " : compression completed")
@@ -150,23 +155,13 @@ def decompress_zst(tag):
     dec_error = 0
     list_zst = sorted(glob.glob('*_nodict.zst'))
     for file_zst in list_zst:
-        print(file_zst, end=' ')
-        print(tag, end=' ')
+        print(file_zst + ' ' + tag)
         file_dec = file_zst + '_d64_' + tag + '.dec'
-        if tag <= 'v0.5.0':
-            params = ['./zstd.' + tag, '-df', file_zst, file_dec]
+        zstd(tag, ['-d'], file_zst, file_dec)
+        if not filecmp.cmp(file_dec, test_dat):
+            raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
         else:
-            params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
-        if execute(params) == 0:
-            if not filecmp.cmp(file_dec, test_dat):
-                print('ERR !! ')
-                dec_error = 1
-            else:
-                print('OK     ')
-        else:
-            print('command does not work')
-            dec_error = 1
-    return dec_error
+            print('OK     ')
 
 
 def decompress_dict(tag):
@@ -181,22 +176,13 @@ def decompress_dict(tag):
         if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
             continue
         dict_name = 'dict.' + dict_tag
-        print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
+        print(file_zst + ' ' + tag + ' dict=' + dict_tag)
         file_dec = file_zst + '_d64_' + tag + '.dec'
-        if tag <= 'v0.5.0':
-            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
+        zstd(tag, ['-D', dict_name, '-d'], file_zst, file_dec)
+        if not filecmp.cmp(file_dec, test_dat):
+            raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
         else:
-            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
-        if execute(params) == 0:
-            if not filecmp.cmp(file_dec, test_dat):
-                print('ERR !! ')
-                dec_error = 1
-            else:
-                print('OK     ')
-        else:
-            print('command does not work')
-            dec_error = 1
-    return dec_error
+            print('OK     ')
 
 
 if __name__ == '__main__':
@@ -267,25 +253,19 @@ if __name__ == '__main__':
     print('Compress test.dat by all released zstd')
     print('-----------------------------------------------')
 
-    error_code = 0
     for tag in tags:
         print(tag)
         if tag >= 'v0.5.0':
             create_dict(tag, dict_source_path)
             dict_compress_sample(tag, test_dat)
             remove_duplicates()
-            error_code += decompress_dict(tag)
+            decompress_dict(tag)
         compress_sample(tag, test_dat)
         remove_duplicates()
-        error_code += decompress_zst(tag)
+        decompress_zst(tag)
 
     print('')
     print('Enumerate different compressed files')
     zstds = sorted(glob.glob('*.zst'))
     for zstd in zstds:
         print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
-
-    if error_code != 0:
-        print('======  ERROR !!!  =======')
-
-    sys.exit(error_code)
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 453c45409..664ff632a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -39,7 +39,7 @@
 #include "seqgen.h"
 #include "util.h"
 #include "timefn.h"       /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
-
+#include "external_matchfinder.h"   /* zstreamExternalMatchFinder, EMF_testCase */
 
 /*-************************************
  *  Constants
@@ -1834,6 +1834,97 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : External matchfinder API: ", testNb++);
+    {
+        size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
+        BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize));
+        size_t const checkBufSize = CNBufferSize;
+        BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
+        int enableFallback;
+        EMF_testCase externalMatchState;
+
+        CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
+
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+
+        /* Reference external matchfinder outside the test loop to
+         * check that the reference is preserved across compressions */
+        ZSTD_registerExternalMatchFinder(
+            zc,
+            &externalMatchState,
+            zstreamExternalMatchFinder
+        );
+
+        for (enableFallback = 0; enableFallback < 1; enableFallback++) {
+            size_t testCaseId;
+
+            EMF_testCase const EMF_successCases[] = {
+                EMF_ONE_BIG_SEQ,
+                EMF_LOTS_OF_SEQS,
+            };
+            size_t const EMF_numSuccessCases = 2;
+
+            EMF_testCase const EMF_failureCases[] = {
+                EMF_ZERO_SEQS,
+                EMF_BIG_ERROR,
+                EMF_SMALL_ERROR,
+            };
+            size_t const EMF_numFailureCases = 3;
+
+            /* Test external matchfinder success scenarios */
+            for (testCaseId = 0; testCaseId < EMF_numSuccessCases; testCaseId++) {
+                size_t res;
+                externalMatchState = EMF_successCases[testCaseId];
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+                CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
+                CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
+                CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
+            }
+
+            /* Test external matchfinder failure scenarios */
+            for (testCaseId = 0; testCaseId < EMF_numFailureCases; testCaseId++) {
+                size_t res;
+                externalMatchState = EMF_failureCases[testCaseId];
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+                if (enableFallback) {
+                    CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
+                    CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
+                } else {
+                    CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+                    CHECK(
+                        ZSTD_getErrorCode(res) != ZSTD_error_externalMatchFinder_failed,
+                        "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+                    );
+                }
+            }
+
+            /* Test compression with external matchfinder + empty src buffer */
+            {
+                size_t res;
+                externalMatchState = EMF_ZERO_SEQS;
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
+                CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
+                CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
+            }
+        }
+
+        /* Test that reset clears the external matchfinder */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+        free(dstBuf);
+        free(checkBuf);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 8673ca348..aef29be0b 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -388,7 +388,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
                     marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
-                    (double)srcSize / fastestC );
+                    (double)srcSize / (double)fastestC );
 
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
 #if 1
@@ -527,8 +527,8 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
                     marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
-                    (double)srcSize / fastestC,
-                    (double)srcSize / fastestD );
+                    (double)srcSize / (double)fastestC,
+                    (double)srcSize / (double)fastestD );
 
             /* CRC Checking */
             {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -558,8 +558,8 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
         }   /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
 
         if (g_displayLevel == 1) {
-            double cSpeed = (double)srcSize / fastestC;
-            double dSpeed = (double)srcSize / fastestD;
+            double cSpeed = (double)srcSize / (double)fastestC;
+            double dSpeed = (double)srcSize / (double)fastestD;
             if (g_additionalParam)
                 DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
             else
diff --git a/zlibWrapper/gzcompatibility.h b/zlibWrapper/gzcompatibility.h
index 3a46127b8..9d11b984f 100644
--- a/zlibWrapper/gzcompatibility.h
+++ b/zlibWrapper/gzcompatibility.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 386c0f0ac..f31cb59db 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index c39cf6379..230bf8411 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

From 71dbe8f9d40ee16c3bbefe52ccfae410ccfda934 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 4 Jan 2023 20:00:04 -0800
Subject: [PATCH 096/197] minor: fix conversion warnings

---
 lib/compress/zstd_compress_internal.h |  2 +-
 lib/compress/zstd_opt.c               | 25 +++++++++++++------------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index f755a1f79..bb5e25083 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -582,7 +582,7 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
 {
     U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6;
     ZSTD_STATIC_ASSERT(ZSTD_btultra == 8);
-    assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
+    assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat));
     return (srcSize >> minlog) + 2;
 }
 
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 68d24aa7b..fdd7f9d8b 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -65,7 +65,7 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
 /* debugging function,
  * @return price in bytes as fractional value
  * for debug messages only */
-MEM_STATIC double ZSTD_fCost(U32 price)
+MEM_STATIC double ZSTD_fCost(int price)
 {
     return (double)price / (BITCOST_MULTIPLIER*8);
 }
@@ -575,16 +575,17 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
     ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
 }
 
-FORCE_INLINE_TEMPLATE
-U32 ZSTD_insertBtAndGetAllMatches (
-                    ZSTD_match_t* matches,   /* store result (found matches) in this table (presumed large enough) */
-                    ZSTD_matchState_t* ms,
-                    U32* nextToUpdate3,
-                    const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
-                    const U32 rep[ZSTD_REP_NUM],
-                    U32 const ll0,   /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
-                    const U32 lengthToBeat,
-                    U32 const mls /* template */)
+FORCE_INLINE_TEMPLATE U32
+ZSTD_insertBtAndGetAllMatches (
+                ZSTD_match_t* matches,  /* store result (found matches) in this table (presumed large enough) */
+                ZSTD_matchState_t* ms,
+                U32* nextToUpdate3,
+                const BYTE* const ip, const BYTE* const iLimit,
+                const ZSTD_dictMode_e dictMode,
+                const U32 rep[ZSTD_REP_NUM],
+                const U32 ll0,  /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
+                const U32 lengthToBeat,
+                const U32 mls /* template */)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
@@ -1151,7 +1152,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                         U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
                         U32 const sequencePrice = literalsPrice + matchPrice;
                         DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
-                                    pos, ZSTD_fCost(sequencePrice));
+                                    pos, ZSTD_fCost((int)sequencePrice));
                         opt[pos].mlen = pos;
                         opt[pos].off = offBase;
                         opt[pos].litlen = litlen;

From 8b130009e3246098acbf659e8622034bd9e20efd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 6 Jan 2023 15:25:36 -0800
Subject: [PATCH 097/197] minor simplification refactoring for timefn

`UTIL_getSpanTimeMicro()` can be factored in a generic way,
reducing OS-dependent code.
---
 programs/fileio.c    |  1 +
 programs/timefn.c    | 45 ++++++--------------------------------------
 programs/timefn.h    | 32 ++++++++++++++++---------------
 tests/decodecorpus.c |  5 +++--
 tests/zstreamtest.c  |  1 +
 5 files changed, 28 insertions(+), 56 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 6bfa559e9..1e3486683 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -28,6 +28,7 @@
 #include <stdio.h>      /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */
 #include <stdlib.h>     /* malloc, free */
 #include <string.h>     /* strcmp, strlen */
+#include <time.h>       /* clock_t, to measure process time */
 #include <fcntl.h>      /* O_WRONLY */
 #include <assert.h>
 #include <errno.h>      /* errno */
diff --git a/programs/timefn.c b/programs/timefn.c
index 7005b2696..08aa1cfcb 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -25,20 +25,6 @@
 
 UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
 
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static LARGE_INTEGER ticksPerSecond;
-    static int init = 0;
-    if (!init) {
-        if (!QueryPerformanceFrequency(&ticksPerSecond)) {
-            perror("timefn::QueryPerformanceFrequency");
-            abort();
-        }
-        init = 1;
-    }
-    return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
-}
-
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 {
     static LARGE_INTEGER ticksPerSecond;
@@ -59,17 +45,6 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 
 UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
 
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static mach_timebase_info_data_t rate;
-    static int init = 0;
-    if (!init) {
-        mach_timebase_info(&rate);
-        init = 1;
-    }
-    return (((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom))/1000ULL;
-}
-
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 {
     static mach_timebase_info_data_t rate;
@@ -115,15 +90,6 @@ static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
     return diff;
 }
 
-PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    PTime micro = 0;
-    micro += 1000000ULL * diff.tv_sec;
-    micro += diff.tv_nsec / 1000ULL;
-    return micro;
-}
-
 PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
 {
     UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
@@ -134,25 +100,26 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
 }
 
 
-
-#else   /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
+#else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
 
 UTIL_time_t UTIL_getTime(void) { return clock(); }
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
 
 #endif
 
+/* ==== Common functions, valid for all time API ==== */
 
+PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
+{
+    return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
+}
 
-/* returns time span in microseconds */
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
 {
     UTIL_time_t const clockEnd = UTIL_getTime();
     return UTIL_getSpanTimeMicro(clockStart, clockEnd);
 }
 
-/* returns time span in microseconds */
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart )
 {
     UTIL_time_t const clockEnd = UTIL_getTime();
diff --git a/programs/timefn.h b/programs/timefn.h
index 438a8de07..3e4476a4e 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -16,12 +16,6 @@ extern "C" {
 #endif
 
 
-/*-****************************************
-*  Dependencies
-******************************************/
-#include <time.h>         /* clock_t, clock, CLOCKS_PER_SEC */
-
-
 
 /*-****************************************
 *  Local Types
@@ -31,7 +25,7 @@ extern "C" {
 # if defined(_AIX)
 #  include <inttypes.h>
 # else
-#  include <stdint.h> /* intptr_t */
+#  include <stdint.h> /* uint64_t */
 # endif
   typedef uint64_t           PTime;  /* Precise Time */
 #else
@@ -41,8 +35,10 @@ extern "C" {
 
 
 /*-****************************************
-*  Time functions
+*  Time types (note: OS dependent)
 ******************************************/
+#include <time.h>     /* TIME_UTC, then struct timespec and clock_t */
+
 #if defined(_WIN32)   /* Windows */
 
     #include <windows.h>   /* LARGE_INTEGER */
@@ -63,7 +59,7 @@ extern "C" {
     typedef struct timespec UTIL_time_t;
     #define UTIL_TIME_INITIALIZER { 0, 0 }
 
-#else   /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
+#else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
 
     #define UTIL_TIME_USES_C90_CLOCK
     typedef clock_t UTIL_time_t;
@@ -72,15 +68,21 @@ extern "C" {
 #endif
 
 
-UTIL_time_t UTIL_getTime(void);
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
+/*-****************************************
+*  Time functions
+******************************************/
 
-#define SEC_TO_MICRO ((PTime)1000000)
-PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
+UTIL_time_t UTIL_getTime(void);
+void UTIL_waitForNextTick(void);
+
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
 
-void UTIL_waitForNextTick(void);
+#define SEC_TO_MICRO ((PTime)1000000)
+PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
+PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
+
+
 
 
 #if defined (__cplusplus)
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 20ad5add0..15e89d51f 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -14,6 +14,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <time.h>  /* time(), for seed random initialization */
 
 #include "util.h"
 #include "timefn.h"   /* UTIL_clockSpanMicro, SEC_TO_MICRO, UTIL_TIME_INITIALIZER */
@@ -24,7 +25,7 @@
 #include "zdict.h"
 
 /* Direct access to internal compression functions is required */
-#include "zstd_compress.c"
+#include "zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
 
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
@@ -165,7 +166,7 @@ static double RAND_exp(U32* seed, double mean)
 /*-*******************************************************
 *  Constants and Structs
 *********************************************************/
-const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"};
+const char* BLOCK_TYPES[] = {"raw", "rle", "compressed"};
 
 #define MAX_DECOMPRESSED_SIZE_LOG 20
 #define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG)
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 664ff632a..10239d3f9 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -25,6 +25,7 @@
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
 #include <string.h>       /* strcmp */
+#include <time.h>         /* time_t, time(), to randomize seed */
 #include <assert.h>       /* assert */
 #include "timefn.h"       /* UTIL_time_t, UTIL_getTime */
 #include "mem.h"

From 6f17a5d8df75c639f30dde6ebd58f11b9f32242e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Jan 2023 05:10:46 +0000
Subject: [PATCH 098/197] Bump actions/checkout from 3.2.0 to 3.3.0

Bumps [actions/checkout](https://github.com/actions/checkout) from 3.2.0 to 3.3.0.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/755da8c3cf115ac066823e79a1e1788f8940201b...ac593985615ec2ede58e132d2e21d2b1cbd6127c)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/dev-long-tests.yml          | 44 ++++++-------
 .github/workflows/dev-short-tests.yml         | 64 +++++++++----------
 .../workflows/publish-release-artifacts.yml   |  2 +-
 .github/workflows/scorecards.yml              |  2 +-
 4 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index f75f61f8e..2cf3702f7 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -13,7 +13,7 @@ jobs:
   make-all:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make all
       run: make all
 
@@ -24,7 +24,7 @@ jobs:
       DEVNULLRIGHTS: 1
       READFROMBLOCKDEVICE: 1
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make test
       run: make test
 
@@ -32,28 +32,28 @@ jobs:
   make-test-osx:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: OS-X test
       run: make test # make -c lib all doesn't work because of the fact that it's not a tty
 
   no-intrinsics-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: no intrinsics fuzztest
       run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
 
   tsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
   ubsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: undefined behavior sanitizer zstreamtest
       run: CC=clang make uasan-test-zstream
 
@@ -61,7 +61,7 @@ jobs:
   tsan-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: thread sanitizer fuzztest
       run: CC=clang make tsan-fuzztest
 
@@ -69,7 +69,7 @@ jobs:
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-8 + ASan + UBSan + Test Zstd
       # See https://askubuntu.com/a/1428822
       run: |
@@ -81,14 +81,14 @@ jobs:
   clang-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + ASan + UBSan + Test Zstd
       run: CC=clang make -j uasan-test-zstd </dev/null V=1
 
   gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -102,7 +102,7 @@ jobs:
   gcc-8-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-8 + ASan + UBSan + Fuzz Test
       # See https://askubuntu.com/a/1428822
       run: |
@@ -114,14 +114,14 @@ jobs:
   clang-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + ASan + UBSan + Fuzz Test
       run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
@@ -131,7 +131,7 @@ jobs:
   clang-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
@@ -141,28 +141,28 @@ jobs:
   asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Regression Test
       run: make -j uasanregressiontest
 
   clang-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + ASan + UBSan + Regression Test
       run: CC=clang make -j uasanregressiontest
 
   msan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: MSan + Regression Test
       run: make -j msanregressiontest
 
   clang-msan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + MSan + Fuzz Test
       run: |
         sudo apt-get -qqq update
@@ -173,7 +173,7 @@ jobs:
   clang-msan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + MSan + Test Zstd
       run: |
         sudo apt-get update
@@ -183,7 +183,7 @@ jobs:
   armfuzz:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Qemu ARM emulation + Fuzz Test
         run: |
           sudo apt-get -qqq update
@@ -193,7 +193,7 @@ jobs:
   valgrind-fuzz-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: valgrind + fuzz test stack mode    # ~ 7mn
       shell: 'script -q -e -c "bash {0}"'
       run: |
@@ -213,7 +213,7 @@ jobs:
           { compiler: gcc, platform: x64, action: test, script: ""},
         ]
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Mingw long test
       run: |
         $env:PATH_ORIGINAL = $env:PATH
diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index b9998e024..0106ddbb1 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -14,21 +14,21 @@ jobs:
   linux-kernel:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: linux kernel, library + build + test
       run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
 
   benchmarking:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make benchmarking
       run: make benchmarking
 
   check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make check on 32-bit
       run: |
         sudo apt update
@@ -38,7 +38,7 @@ jobs:
   check-x32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
       env:
         CHECK_CONSTRAINED_MEM: true
@@ -50,7 +50,7 @@ jobs:
   gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-7 + libzstdmt compilation
       # See https://askubuntu.com/a/1428822
       run: |
@@ -67,7 +67,7 @@ jobs:
   cmake-build-and-test-check:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: cmake build and test check
       run: |
         FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
@@ -78,7 +78,7 @@ jobs:
   cpp-gnu90-c99-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: C++, gnu90 and c99 compatibility
       run: |
         make cxxtest
@@ -92,7 +92,7 @@ jobs:
   mingw-cross-compilation:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: mingw cross-compilation
       run: |
         # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@@ -103,7 +103,7 @@ jobs:
   armbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ARM Build Test
       run: |
         sudo apt-get -qqq update
@@ -113,7 +113,7 @@ jobs:
   bourne-shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Bourne shell compatibility (shellcheck)
       run: |
         wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
@@ -123,7 +123,7 @@ jobs:
   zlib-wrapper:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: zlib wrapper test
       run: |
         sudo apt-get -qqq update
@@ -134,7 +134,7 @@ jobs:
   lz4-threadpool-libs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: LZ4, thread pool, and libs build testslib wrapper test
       run: |
         make lz4install
@@ -148,7 +148,7 @@ jobs:
   gcc-make-tests-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Make all, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -158,7 +158,7 @@ jobs:
   gcc-8-make:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: gcc-8 build
         # See https://askubuntu.com/a/1428822
         run: |
@@ -170,7 +170,7 @@ jobs:
   implicit-fall-through:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: -Wimplicit-fallthrough build
         run: |
           make clean
@@ -181,7 +181,7 @@ jobs:
   meson-linux:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Install packages
         run: |
           sudo apt-get update
@@ -205,7 +205,7 @@ jobs:
   meson-windows:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Install packages
         run: pip install --pre meson
       - name: Initialize the MSVC dev command prompt
@@ -234,7 +234,7 @@ jobs:
             flags: "-A Win32"
           - generator: "MinGW Makefiles"
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
       uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build
@@ -253,7 +253,7 @@ jobs:
         platform: [x64, Win32]
         configuration: [Debug, Release]
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
       uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build
@@ -272,7 +272,7 @@ jobs:
 #        platform: [x64, Win32]
 #        configuration: [Debug, Release]
 #    steps:
-#    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+#    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
 #    - name: Add MSBuild to PATH
 #      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
 #    - name: Build
@@ -287,7 +287,7 @@ jobs:
   libzstd-size:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: libzstd size test
       run: |
         make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
@@ -298,7 +298,7 @@ jobs:
   minimal-decompressor-macros:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: minimal decompressor macros
       run: |
         make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -313,7 +313,7 @@ jobs:
   dynamic-bmi2:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: dynamic bmi2 tests
       run: |
         make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@@ -325,7 +325,7 @@ jobs:
   test-variants:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make all variants & validate
       run: |
         make -j -C programs allVariants MOREFLAGS=-O0
@@ -351,7 +351,7 @@ jobs:
       XCC: ${{ matrix.xcc }}
       XEMU: ${{ matrix.xemu }}
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: apt update & install
       run: |
         sudo apt-get update
@@ -404,7 +404,7 @@ jobs:
           { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
         ]
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Mingw short test
       run: |
         ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
@@ -437,7 +437,7 @@ jobs:
         platform: [x64, Win32]
         configuration: [Release]
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
       uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
     - name: Build and run tests
@@ -457,7 +457,7 @@ jobs:
   intel-cet-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Build Zstd
       run: |
         make -j zstd V=1
@@ -478,7 +478,7 @@ jobs:
     container:
       image: debian:testing
     steps:
-      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Install dependencies
         run: |
           apt -y update
@@ -493,7 +493,7 @@ jobs:
   versions-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Versions Compatibility Test
       run: |
         make -C tests versionsTest
@@ -501,7 +501,7 @@ jobs:
   clangbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make clangbuild
       run: |
         make clangbuild
@@ -525,7 +525,7 @@ jobs:
 #        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
 #        sudo apt-get update
 #        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+#    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
 #    - name: make check
 #      run: |
 #        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml
index 0903f0d3c..2c89a91a5 100644
--- a/.github/workflows/publish-release-artifacts.yml
+++ b/.github/workflows/publish-release-artifacts.yml
@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
 
       - name: Archive
         env:
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index afdf4d63f..df7b5cdf4 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -27,7 +27,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # tag=v3
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
         with:
           persist-credentials: false
 

From 59a536aa01f825bab50bc2aa2e9e12109912588e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Jan 2023 05:10:50 +0000
Subject: [PATCH 099/197] Bump actions/upload-artifact from 3.1.1 to 3.1.2

Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.1 to 3.1.2.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/83fd05a356d7e2593de66fc9913b3002723633cb...0b7f8abb1508181956e8e162db84b466c27e18ce)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/dev-long-tests.yml | 2 +-
 .github/workflows/scorecards.yml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index f75f61f8e..7726daafa 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -279,7 +279,7 @@ jobs:
         dry-run: false
         sanitizer: ${{ matrix.sanitizer }}
     - name: Upload Crash
-      uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2
       if: failure() && steps.build.outcome == 'success'
       with:
         name: ${{ matrix.sanitizer }}-artifacts
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index afdf4d63f..fbddeb1a6 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -51,7 +51,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2
         with:
           name: SARIF file
           path: results.sarif

From fe08137d9abbf72bcb01740c7b398179ac5cb85e Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 9 Jan 2023 07:53:53 -0800
Subject: [PATCH 100/197] resolve max block value in cctx and use when
 calculating the max block size

---
 lib/compress/zstd_compress.c | 38 +++++++++++++++++++++++++-----------
 lib/zstd.h                   |  2 +-
 tests/fuzz/zstd_helpers.c    |  1 +
 3 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e13a80779..e84ed9f27 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -288,6 +288,15 @@ static int ZSTD_resolveExternalSequenceValidation(int mode) {
 #endif
 }
 
+/* Sets the default maximum block size. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+    if (maxBlockSize == 0) {
+        return ZSTD_BLOCKSIZE_MAX;
+    } else {
+        return maxBlockSize;
+    }
+}
+
 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
  * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
@@ -312,6 +321,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
     cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+    cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -351,6 +361,7 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
     ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
     cctxParams->compressionLevel = compressionLevel;
     cctxParams->fParams.contentSizeFlag = 1;
+    // cctxParams->maxBlockSize = ZSTD_BLOCKSIZE_MAX;
     return 0;
 }
 
@@ -377,6 +388,7 @@ ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
     cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
     cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+    cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -675,7 +687,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
     case ZSTD_c_enableMatchFinderFallback:
-    case ZSTD_c_maxBlockSize
+    case ZSTD_c_maxBlockSize:
     default:
         return 0;
     }
@@ -974,8 +986,8 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
     case ZSTD_c_maxBlockSize:
         BOUNDCHECK(ZSTD_c_maxBlockSize, value);
         CCtxParams->maxBlockSize = value;
-        return CCtxParams->prefetchCDictTables;
-    
+        return CCtxParams->maxBlockSize;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -1561,10 +1573,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const size_t buffInSize,
         const size_t buffOutSize,
         const U64 pledgedSrcSize,
-        int useExternalMatchFinder)
+        int useExternalMatchFinder,
+        size_t maxBlockSize)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+    size_t const blockSize = MIN(maxBlockSize, windowSize);
     size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
@@ -1616,7 +1629,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
      * be needed. However, we still allocate two 0-sized buffers, which can
      * take space under ASAN. */
     return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
-        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
 }
 
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1666,7 +1679,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+        size_t const blockSize = MIN(params->maxBlockSize, (size_t)1 << cParams.windowLog);
         size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
                 ? ((size_t)1 << cParams.windowLog) + blockSize
                 : 0;
@@ -1677,7 +1690,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
-            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
     }
 }
 
@@ -1959,7 +1972,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     }
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+        size_t const blockSize = MIN(params->maxBlockSize, windowSize);
         size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
         size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
@@ -1977,7 +1990,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         size_t const neededSpace =
             ZSTD_estimateCCtxSize_usingCCtxParams_internal(
                 &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
-                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder);
+                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder, params->maxBlockSize);
         int resizeWorkspace;
 
         FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
@@ -2355,6 +2368,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
         params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
         params.ldmParams = srcCCtx->appliedParams.ldmParams;
         params.fParams = fParams;
+        params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
         ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
                                 /* loadedDictSize */ 0,
                                 ZSTDcrp_leaveDirty, zbuff);
@@ -4489,7 +4503,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
     ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
     assert(!ZSTD_checkCParams(cParams));
-    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
+    return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
 }
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -5927,6 +5941,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
     params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+    params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
 
 #ifdef ZSTD_MULTITHREAD
     if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
@@ -6118,6 +6133,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
         /* Reset to the original values. */
         cctx->requestedParams.inBufferMode = originalInBufferMode;
         cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
         FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
         if (result != 0) {  /* compression not completed, due to lack of output space */
             assert(oPos == dstCapacity);
diff --git a/lib/zstd.h b/lib/zstd.h
index b46700b54..4437a2461 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -501,7 +501,7 @@ typedef enum {
      ZSTD_c_experimentalParam15=1012,
      ZSTD_c_experimentalParam16=1013,
      ZSTD_c_experimentalParam17=1014,
-     ZSTD_c_experimentalParam18=1015,
+     ZSTD_c_experimentalParam18=1015
 } ZSTD_cParameter;
 
 typedef struct {
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index d5210627e..0f602d1db 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -104,6 +104,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
     setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
     setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
+    setRand(cctx, ZSTD_c_maxBlockSize, 1 << 12, ZSTD_BLOCKSIZE_MAX, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }

From 3ac0b913024af5b9adea8e626bd62f951080498e Mon Sep 17 00:00:00 2001
From: Daniel Kutenin <kutdanila@yandex.ru>
Date: Tue, 10 Jan 2023 15:33:50 +0000
Subject: [PATCH 101/197] Fix fuzzing with ZSTD_MULTITHREAD

At Google we fuzz zstd without ZSTD_MULTITHREAD but we want inputs to be as much as reproducible. It allows us to test new fuzzing methods for our fuzz team internally and have more horsepower to find bugs
---
 tests/fuzz/zstd_helpers.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index d5210627e..ef4f05774 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -89,12 +89,11 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
             ZSTD_LDM_HASHRATELOG_MAX, producer);
     /* Set misc parameters */
-#ifndef ZSTD_MULTITHREAD
-    setRand(cctx, ZSTD_c_nbWorkers, 0, 0, producer);
-    setRand(cctx, ZSTD_c_rsyncable, 0, 0, producer);
-#else
     setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer);
     setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer);
+#ifndef ZSTD_MULTITHREAD
+    set(cctx, ZSTD_c_nbWorkers, 0);
+    set(cctx, ZSTD_c_rsyncable, 0);
 #endif
     setRand(cctx, ZSTD_c_useRowMatchFinder, 0, 2, producer);
     setRand(cctx, ZSTD_c_enableDedicatedDictSearch, 0, 1, producer);

From ca2ff788df535bc339338b5ec4b198335a264490 Mon Sep 17 00:00:00 2001
From: Daniel Kutenin <kutdanila@yandex.ru>
Date: Tue, 10 Jan 2023 20:30:03 +0000
Subject: [PATCH 102/197] Make the producer use the same amount of entropy

---
 tests/fuzz/zstd_helpers.c | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index ef4f05774..6ad3d975e 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -26,9 +26,14 @@ static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
     FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
 }
 
+static unsigned produceParamValue(unsigned min, unsigned max,
+                                  FUZZ_dataProducer_t *producer) {
+    return FUZZ_dataProducer_uint32Range(producer, min, max);
+}
+
 static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
                     unsigned max, FUZZ_dataProducer_t *producer) {
-    unsigned const value = FUZZ_dataProducer_uint32Range(producer, min, max);
+    unsigned const value = produceParamValue(min, max, producer);
     set(cctx, param, value);
 }
 
@@ -89,11 +94,18 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
             ZSTD_LDM_HASHRATELOG_MAX, producer);
     /* Set misc parameters */
-    setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer);
-    setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer);
 #ifndef ZSTD_MULTITHREAD
+    // To reproduce with or without ZSTD_MULTITHREAD, we are going to use
+    // the same amount of entropy.
+    unsigned const nbWorkers_value = produceParamValue(0, 2, producer);
+    unsigned const rsyncable_value = produceParamValue(0, 1, producer);
+    (void)nbWorkers_value;
+    (void)rsyncable_value;
     set(cctx, ZSTD_c_nbWorkers, 0);
     set(cctx, ZSTD_c_rsyncable, 0);
+#else
+    setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer);
+    setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer);
 #endif
     setRand(cctx, ZSTD_c_useRowMatchFinder, 0, 2, producer);
     setRand(cctx, ZSTD_c_enableDedicatedDictSearch, 0, 1, producer);

From 1fffcfe01d56b69d8ab09b79e4dbe37c9a091dda Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Wed, 11 Jan 2023 11:09:57 -0800
Subject: [PATCH 103/197] update minimum threshold for max block size

---
 lib/compress/zstd_compress.c | 3 +--
 lib/zstd.h                   | 1 +
 tests/fuzz/zstd_helpers.c    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e84ed9f27..e27cbd613 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -361,7 +361,6 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
     ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
     cctxParams->compressionLevel = compressionLevel;
     cctxParams->fParams.contentSizeFlag = 1;
-    // cctxParams->maxBlockSize = ZSTD_BLOCKSIZE_MAX;
     return 0;
 }
 
@@ -617,7 +616,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         return bounds;
 
     case ZSTD_c_maxBlockSize:
-        bounds.lowerBound = 1;
+        bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
         bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
         return bounds;
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 4437a2461..228788685 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -141,6 +141,7 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
 
 #define ZSTD_BLOCKSIZELOG_MAX  17
 #define ZSTD_BLOCKSIZE_MAX     (1<<ZSTD_BLOCKSIZELOG_MAX)
+#define ZSTD_BLOCKSIZE_MAX_MIN 1 << 10 /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 /***************************************
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 0f602d1db..ab243f1ec 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -104,7 +104,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
     setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
     setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
-    setRand(cctx, ZSTD_c_maxBlockSize, 1 << 12, ZSTD_BLOCKSIZE_MAX, producer);
+    setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }

From 4f7183d887789d4d2bb2e5af850c427f1df725ff Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Wed, 11 Jan 2023 16:29:23 -0500
Subject: [PATCH 104/197] Completely overhaul Windows CI (#3410)

* Overhaul windows CI

* upgrade setup-msbuild from v1.1.3 to v1.3

* remove cmake 2019 test

* fix 32-bit gcc mingw test

* merge conflict
---
 .github/workflows/dev-long-tests.yml  |  70 ++++++-----------
 .github/workflows/dev-short-tests.yml | 105 ++++++++++++--------------
 2 files changed, 71 insertions(+), 104 deletions(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index 2cf3702f7..aca0b5da5 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -204,57 +204,31 @@ jobs:
         make -C tests test-fuzzer-stackmode
 
   mingw-long-test:
-    runs-on: windows-2019
-    strategy:
-      fail-fast: false
-      matrix:
-        include: [
-          { compiler: clang, platform: x64, action: build, script: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument -Wno-implicit-int-float-conversion' make -j allzstd V=1"},
-          { compiler: gcc, platform: x64, action: test, script: ""},
-        ]
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: msys2 {0}
     steps:
     - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
-    - name: Mingw long test
+    - uses: msys2/setup-msys2@v2
+      with:
+        msystem: MINGW64
+        install: make
+        update: true
+    # Based on https://ariya.io/2020/07/on-github-actions-with-msys2
+    - name: install mingw gcc
+      run: pacman --noconfirm -S gcc
+    - name: MINGW64 gcc fuzztest
       run: |
-        $env:PATH_ORIGINAL = $env:PATH
-        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
-        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
-        IF ("${{matrix.platform}}" -eq "x64")
-        {
-            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
-        }
-        ELSEIF ("${{matrix.platform}}" -eq "x86")
-        {
-            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
-        }
-        IF ("${{matrix.action}}" -eq "build")
-        {
-          make -v
-          sh -c "${{matrix.compiler}} -v"
-          ECHO "Building zlib to static link"
-          $env:CC = "${{matrix.compiler}}"
-          sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib"
-          sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
-          ECHO "Building zstd"
-          $env:CPPFLAGS = "-I../../zlib"
-          $env:LDFLAGS = "../../zlib/libz.a"
-          sh -c "${{matrix.script}}"
-        }
-        ELSEIF ("${{matrix.action}}" -eq "test")
-        {
-            ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
-            $env:CC = "gcc"
-            $env:CXX = "g++"
-            MKDIR build\cmake\build
-            CD build\cmake\build
-            $env:FUZZERTEST = "-T2mn"
-            $env:ZSTREAM_TESTTIME = "-T2mn"
-            cmake -G "Visual Studio 14 2015 Win64" ..
-            cd ..\..\..
-            make clean
-        }
+        export CC="gcc"
+        export CXX="g++"
+        export FUZZERTEST="-T2mn"
+        export ZSTREAM_TESTTIME="-T2mn"
+        echo "Testing $CC $CXX MINGW64"
+        make -v
+        $CC --version
+        $CXX --version
+        make -C tests fuzztest
 
   # lasts ~20mn
   oss-fuzz:
diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 0106ddbb1..d0dba3209 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -223,20 +223,20 @@ jobs:
         run: |
           meson install -C builddir --destdir staging/
 
-  cmake-visual-2019:
-    runs-on: windows-2019
+  cmake-visual-2022:
     strategy:
       matrix:
         include:
-          - generator: "Visual Studio 16 2019"
+          - generator: "Visual Studio 17 2022"
             flags: "-A x64"
-          - generator: "Visual Studio 16 2019"
+          - generator: "Visual Studio 17 2022"
             flags: "-A Win32"
           - generator: "MinGW Makefiles"
+    runs-on: windows-2022
     steps:
     - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
+      uses: microsoft/setup-msbuild@v1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
@@ -246,41 +246,29 @@ jobs:
         cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} ..
         cmake.exe --build .
 
-  visual-2019:
-    runs-on: windows-2019
+  msbuild-visual-studio:
     strategy:
       matrix:
-        platform: [x64, Win32]
-        configuration: [Debug, Release]
+        include: [
+          { name: "VS 2022 x64 Debug", platform: x64, configuration: Debug, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2022 Win32 Debug", platform: Win32, configuration: Debug, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2022 x64 Release", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2022 Win32 Release", platform: Win32, configuration: Release, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2019 x64 Release", platform: Win32, configuration: Release, toolset: v142, runner: "windows-2019"},
+          { name: "VS 2019 Win32 Release", platform: x64, configuration: Release, toolset: v142, runner: "windows-2019"},
+        ]
+    runs-on: ${{matrix.runner}}
     steps:
     - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
+      uses: microsoft/setup-msbuild@v1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
       run: >
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142
+        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=${{matrix.toolset}}
         /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
 
-# TODO: fix as part of https://github.com/facebook/zstd/issues/3064
-#  visual-2015:
-#    # only GH actions windows-2016 contains VS 2015
-#    runs-on: windows-2016
-#    strategy:
-#      matrix:
-#        platform: [x64, Win32]
-#        configuration: [Debug, Release]
-#    steps:
-#    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
-#    - name: Add MSBuild to PATH
-#      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
-#    - name: Build
-#      working-directory: ${{env.GITHUB_WORKSPACE}}
-#      run: >
-#        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140
-#        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
-
   # This tests that we don't accidently grow the size too much.
   # If the size grows intentionally, you can raise these numbers.
   # But we do need to think about binary size, since it is a concern.
@@ -394,44 +382,49 @@ jobs:
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
 
   mingw-short-test:
-    runs-on: windows-2019
+    runs-on: windows-latest
     strategy:
-      fail-fast: false
       matrix:
         include: [
-          { compiler: gcc, platform: x64, script: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"},
-          { compiler: gcc, platform: x86, script: "CFLAGS=-Werror make -j allzstd"},
-          { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
+          { compiler: gcc, msystem: MINGW32, cflags: "-Werror"},
+          { compiler: gcc, msystem: MINGW64, cflags: "-Werror"},
+          { compiler: clang, msystem: MINGW64, cflags: "--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument"},
         ]
+    defaults:
+      run:
+        shell: msys2 {0}
     steps:
     - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
-    - name: Mingw short test
+    - uses: msys2/setup-msys2@v2
+      with:
+        msystem: ${{ matrix.msystem }}
+        install: make diffutils
+        update: true
+    # Based on https://ariya.io/2020/07/on-github-actions-with-msys2
+    - name: install mingw gcc
+      if: ${{ (matrix.msystem == 'MINGW32') && (matrix.compiler == 'gcc') }}
+      run: pacman --noconfirm -S mingw-w64-i686-gcc
+    - name: install mingw gcc
+      if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'gcc') }}
+      run: pacman --noconfirm -S mingw-w64-x86_64-gcc
+    - name: install mingw clang x86_64
+      if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'clang') }}
+      run: pacman --noconfirm -S mingw-w64-x86_64-clang
+    - name: install mingw clang i686
+      if: ${{ (matrix.msystem == 'MINGW32') && (matrix.compiler == 'clang') }}
+      run: pacman --noconfirm -S mingw-w64-i686-clang
+    - name: run mingw tests
       run: |
-        ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
-        $env:PATH_ORIGINAL = $env:PATH
-        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
-        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
-        IF ("${{matrix.platform}}" -eq "x64")
-        {
-            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
-        }
-        ELSEIF ("${{matrix.platform}}" -eq "x86")
-        {
-            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
-        }
         make -v
-        sh -c "${{matrix.compiler}} -v"
-        $env:CC = "${{matrix.compiler}}"
-        sh -c "${{matrix.script}}"
-        ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
+        export CC=${{ matrix.compiler }}
+        $CC --version
+        CFLAGS="${{ matrix.cflags }}" make -j allzstd
+        echo "Testing $CC ${{ matrix.msystem }}"
         make clean
         make check
 
-
   visual-runtime-tests:
-    runs-on: windows-2019
+    runs-on: windows-latest
     strategy:
       matrix:
         platform: [x64, Win32]
@@ -439,7 +432,7 @@ jobs:
     steps:
     - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@34cfbaee7f672c76950673338facd8a73f637506 # tag=v1.1.3
+      uses: microsoft/setup-msbuild@v1.3
     - name: Build and run tests
       working-directory: ${{env.GITHUB_WORKSPACE}}
       env:

From 53eb5a758c4e618c106eebafa49abde5845f96ef Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Thu, 12 Jan 2023 08:55:39 -0800
Subject: [PATCH 105/197] add simple test for maxBlockSize expected
 functionality

---
 lib/compress/zstd_compress.c |  2 +-
 lib/zstd.h                   |  2 +-
 tests/zstreamtest.c          | 43 ++++++++++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e27cbd613..6872d3364 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -288,7 +288,7 @@ static int ZSTD_resolveExternalSequenceValidation(int mode) {
 #endif
 }
 
-/* Sets the default maximum block size. */
+/* Resolves maxBlockSize to the default if no value is present. */
 static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
     if (maxBlockSize == 0) {
         return ZSTD_BLOCKSIZE_MAX;
diff --git a/lib/zstd.h b/lib/zstd.h
index 228788685..0c0fc1afc 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -141,7 +141,7 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
 
 #define ZSTD_BLOCKSIZELOG_MAX  17
 #define ZSTD_BLOCKSIZE_MAX     (1<<ZSTD_BLOCKSIZELOG_MAX)
-#define ZSTD_BLOCKSIZE_MAX_MIN 1 << 10 /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 /***************************************
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 664ff632a..bd829fbb1 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1925,6 +1925,49 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+
+    /* Test maxBlockSize cctx param functionality */
+    DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        size_t const srcSize = 2 << 10;
+        void* const src = CNBuffer;
+        size_t const dstSize = ZSTD_compressBound(srcSize);
+        void* const dst1 = compressedBuffer;
+        void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+        size_t size1, size2;
+        void* const checkBuf = malloc(srcSize);
+
+        memset(src, 'x', srcSize);
+
+        /* Quick test to make sure maxBlockSize bounds are enforced */
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
+
+        /* maxBlockSize = 1KB */
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1 << 10));
+        size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+        if (ZSTD_isError(size1)) goto _output_error;
+        CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+        CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+        /* maxBlockSize = 3KB */
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3 << 10));
+        size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+        if (ZSTD_isError(size2)) goto _output_error;
+        CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+        CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+        /* Compressed output should not be equal */
+        assert(memcmp(dst1, dst2, dstSize) != 0);
+        assert(size1 - size2 == 4); /* We add another RLE block with header + character */
+        ZSTD_freeCCtx(cctx);
+        free(checkBuf);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);

From 06b096db473ef28fe58e34d75d30825970c403fe Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Thu, 12 Jan 2023 13:41:50 -0800
Subject: [PATCH 106/197] additional tests and documentation updates + allow
 maxBlockSize to be set to 0 (goes to default)

---
 lib/compress/zstd_compress.c |   3 +-
 lib/zstd.h                   |   7 ++-
 tests/zstreamtest.c          | 106 ++++++++++++++++++++++++++---------
 3 files changed, 89 insertions(+), 27 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 6872d3364..2ff1fa209 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -983,7 +983,8 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         return CCtxParams->enableMatchFinderFallback;
 
     case ZSTD_c_maxBlockSize:
-        BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+        if (value!=0)    /* 0 ==> default */
+            BOUNDCHECK(ZSTD_c_maxBlockSize, value);
         CCtxParams->maxBlockSize = value;
         return CCtxParams->maxBlockSize;
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 0c0fc1afc..9d1d507ff 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2071,8 +2071,13 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
 #define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
 
 /*  ZSTD_c_maxBlockSize
+ *  Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ *  The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
  *
- *  Default is ZSTD_BLOCKSIZE_MAX.
+ *  This parameter can be used to set an upper bound on the blocksize
+ *  that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ *  bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ *  compressBound() innacurate). Only currently meant to be used for testing.
  *
  */
 #define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index bd829fbb1..3b8512310 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1930,41 +1930,97 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
     DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
     {
         ZSTD_CCtx* cctx = ZSTD_createCCtx();
-        size_t const srcSize = 2 << 10;
-        void* const src = CNBuffer;
-        size_t const dstSize = ZSTD_compressBound(srcSize);
-        void* const dst1 = compressedBuffer;
-        void* const dst2 = (BYTE*)compressedBuffer + dstSize;
-        size_t size1, size2;
-        void* const checkBuf = malloc(srcSize);
-
-        memset(src, 'x', srcSize);
 
         /* Quick test to make sure maxBlockSize bounds are enforced */
         assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
         assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
 
-        /* maxBlockSize = 1KB */
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1 << 10));
-        size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+        /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
+        {
+            size_t srcSize = 2 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+            memset(src, 'x', srcSize);
 
-        if (ZSTD_isError(size1)) goto _output_error;
-        CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
-        CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+            /* maxBlockSize = 1KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
 
-        /* maxBlockSize = 3KB */
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3 << 10));
-        size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
 
-        if (ZSTD_isError(size2)) goto _output_error;
-        CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
-        CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+            /* maxBlockSize = 3KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
 
-        /* Compressed output should not be equal */
-        assert(memcmp(dst1, dst2, dstSize) != 0);
-        assert(size1 - size2 == 4); /* We add another RLE block with header + character */
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 - size2 == 4); /* We add another RLE block with header + character */
+            assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
+
+            /* maxBlockSize = 1KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = 3KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+
+            free(checkBuf);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+        /* Test maxBlockSize = 0 is valid */
+        {   size_t srcSize = 256 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+
+            /* maxBlockSize = 0 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+            free(checkBuf);
+        }
         ZSTD_freeCCtx(cctx);
-        free(checkBuf);
     }
     DISPLAYLEVEL(3, "OK \n");
 

From 796699c0bc12ae1efa405418b14cb6dd0101cc6f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 11 Jan 2023 15:11:51 -0800
Subject: [PATCH 107/197] fix root cause of #3416

A minor change in 5434de0 changed a `<=` into a `<`,
and as an indirect consequence allowed compression attempt of literals when there are only 6 literals to compress
(previous limit was effectively 7 literals).

This is not in itself a problem, as the threshold is merely an heuristic,
but it emerged a bug that has always been there, and was just never triggered so far due to the previous limit.
This bug would make the literal compressor believes that all literals are the same symbol,
but for the exact case where nbLiterals==6, plus a pretty wild combination of other limit conditions,
this outcome could be false, resulting in data corruption.

Replaced the blind heuristic by an actual test for all limit cases,
so that even if the threshold is changed again in the future,
the detection of RLE mode will remain reliable.
---
 lib/common/huf.h                      |  4 +--
 lib/compress/huf_compress.c           |  6 ++---
 lib/compress/zstd_compress_literals.c | 38 +++++++++++++++++++--------
 lib/compress/zstd_compress_literals.h |  4 +++
 lib/decompress/zstd_decompress.c      |  1 +
 5 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index f7316f9b2..9683f915a 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -221,7 +221,7 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,    /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
                        HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                       unsigned suspectUncompressible, HUF_depth_mode depthMode);
+                       int suspectUncompressible, HUF_depth_mode depthMode);
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@@ -328,7 +328,7 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,   /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
                        HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                       unsigned suspectUncompressible, HUF_depth_mode depthMode);
+                       int suspectUncompressible, HUF_depth_mode depthMode);
 
 size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 #ifndef HUF_FORCE_DECOMPRESS_X1
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index f7ca2d3bb..75200851b 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1318,7 +1318,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
                        HUF_nbStreams_e nbStreams,
                        void* workSpace, size_t wkspSize,
                        HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
-                 const int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
+                 const int bmi2, int suspectUncompressible, HUF_depth_mode depthMode)
 {
     HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
     BYTE* const ostart = (BYTE*)dst;
@@ -1439,7 +1439,7 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
-                      int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
+                      int bmi2, int suspectUncompressible, HUF_depth_mode depthMode)
 {
     DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
@@ -1472,7 +1472,7 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                      unsigned suspectUncompressible, HUF_depth_mode depthMode)
+                      int suspectUncompressible, HUF_depth_mode depthMode)
 {
     DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index 666e5315d..e84781873 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -24,9 +24,9 @@ static size_t showHexa(const void* src, size_t srcSize)
     const BYTE* const ip = (const BYTE*)src;
     size_t u;
     for (u=0; u<srcSize; u++) {
-        RAWLOG(6, " %02X", ip[u]); (void)ip;
+        RAWLOG(5, " %02X", ip[u]); (void)ip;
     }
-    RAWLOG(6, " \n");
+    RAWLOG(5, " \n");
     return srcSize;
 }
 
@@ -65,12 +65,26 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
     return srcSize + flSize;
 }
 
+static int allBytesIdentical(const void* src, size_t srcSize)
+{
+    assert(srcSize >= 1);
+    assert(src != NULL);
+    {   const BYTE b = ((const BYTE*)src)[0];
+        size_t p;
+        for (p=1; p<srcSize; p++) {
+            if (((const BYTE*)src)[p] != b) return 0;
+        }
+        return 1;
+    }
+}
+
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     BYTE* const ostart = (BYTE*)dst;
     U32   const flSize = 1 + (srcSize>31) + (srcSize>4095);
 
-    (void)dstCapacity;  /* dstCapacity already guaranteed to be >=4, hence large enough */
+    assert(dstCapacity >= 4); (void)dstCapacity;
+    assert(allBytesIdentical(src, srcSize));
 
     switch(flSize)
     {
@@ -88,7 +102,7 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
     }
 
     ostart[flSize] = *(const BYTE*)src;
-    DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
+    DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
     return flSize+1;
 }
 
@@ -105,8 +119,8 @@ ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
     /* btultra2 : min 8 bytes;
      * then 2x larger for each successive compression strategy
      * max threshold 64 bytes */
-    {   int const shift = MIN(9-strategy, 3);
-        size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : 8 << shift;
+    {   int const shift = MIN(9-(int)strategy, 3);
+        size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
         DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
         return mintc;
     }
@@ -148,7 +162,7 @@ size_t ZSTD_compressLiterals (
     {   HUF_repeat repeat = prevHuf->repeatMode;
         int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
         HUF_depth_mode const depthMode = (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD) ? HUF_depth_optimal : HUF_depth_fast;
-        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode);
+        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, int, HUF_depth_mode);
         huf_compress_f huf_compress;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
         huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
@@ -159,9 +173,10 @@ size_t ZSTD_compressLiterals (
                                 (HUF_CElt*)nextHuf->CTable,
                                 &repeat, preferRepeat,
                                 bmi2, suspectUncompressible, depthMode);
+        DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
         if (repeat != HUF_repeat_none) {
             /* reused the existing table */
-            DEBUGLOG(5, "Reusing previous huffman table");
+            DEBUGLOG(5, "reusing statistics from previous huffman block");
             hType = set_repeat;
         }
     }
@@ -172,9 +187,10 @@ size_t ZSTD_compressLiterals (
             return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }   }
     if (cLitSize==1) {
-        ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-        return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
-    }
+        if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+    }   }
 
     if (hType == set_compressed) {
         /* using a newly constructed table */
diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h
index 9eb74729d..b060c8ad2 100644
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@@ -16,6 +16,10 @@
 
 size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
+/* ZSTD_compressRleLiteralsBlock() :
+ * Conditions :
+ * - All bytes in @src are identical
+ * - dstCapacity >= 4 */
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
 /* ZSTD_compressLiterals():
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index f00ef3a67..84577b6e1 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -981,6 +981,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
     }
     ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
     /* Allow caller to get size read */
+    DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr);
     *srcPtr = ip;
     *srcSizePtr = remainingSrcSize;
     return (size_t)(op-ostart);

From ac45e078a5bc22e06d1be1375c25795b7ef3c1ff Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 12 Jan 2023 15:49:01 -0800
Subject: [PATCH 108/197] add explanation about new test

as requested by @terrelln
---
 lib/compress/zstd_compress_literals.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index e84781873..0dbc2c4d4 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -187,6 +187,11 @@ size_t ZSTD_compressLiterals (
             return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }   }
     if (cLitSize==1) {
+        /* A return value of 1 signals that the alphabet consists of a single symbol.
+         * However, in some rare circumstances, it could be the compressed size (a single byte).
+         * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
+         * (it's also necessary to not generate statistics).
+         * Therefore, in such a case, actively check that all bytes are identical. */
         if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
             ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
             return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);

From 5b266196a41e6a15e21bd4f0eeab43b938db1d90 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 11 Jan 2023 18:14:40 -0800
Subject: [PATCH 109/197] Add support for in-place decompression

* Add a function and macro ZSTD_decompressionMargin() that computes the
  decompression margin for in-place decompression. The function computes
  a tight margin that works in all cases, and the macro computes an upper
  bound that will only work if flush isn't used.
* When doing in-place decompression, make sure that our output buffer
  doesn't overlap with the input buffer. This ensures that we don't
  decide to use the portion of the output buffer that overlaps the input
  buffer for temporary memory, like for literals.
* Add a simple unit test.
* Add in-place decompression to the simple_round_trip and
  stream_round_trip fuzzers. This should help verify that our margin stays
  correct.
---
 lib/common/zstd_internal.h       |  1 +
 lib/decompress/zstd_decompress.c | 68 ++++++++++++++++++++++++++++++--
 lib/legacy/zstd_legacy.h         |  7 ++++
 lib/zstd.h                       | 45 +++++++++++++++++++++
 tests/fuzz/simple_round_trip.c   | 44 +++++++++++++++++----
 tests/fuzz/stream_round_trip.c   | 18 +++++++++
 tests/fuzzer.c                   | 54 +++++++++++++++++++++++++
 7 files changed, 227 insertions(+), 10 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 48558873d..12e1106a1 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -341,6 +341,7 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
  *          `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
  */
 typedef struct {
+    size_t nbBlocks;
     size_t compressedSize;
     unsigned long long decompressedBound;
 } ZSTD_frameSizeInfo;   /* decompress & legacy */
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index f00ef3a67..4559451d0 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -782,6 +782,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
             ip += 4;
         }
 
+        frameSizeInfo.nbBlocks = nbBlocks;
         frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
         frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
                                         ? zfh.frameContentSize
@@ -825,6 +826,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
     return bound;
 }
 
+size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
+{
+    size_t margin = 0;
+    unsigned maxBlockSize = 0;
+
+    /* Iterate over each frame */
+    while (srcSize > 0) {
+        ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+        size_t const compressedSize = frameSizeInfo.compressedSize;
+        unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+        ZSTD_frameHeader zfh;
+
+        FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
+        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+            return ERROR(corruption_detected);
+
+        if (zfh.frameType == ZSTD_frame) {
+            /* Add the frame header to our margin */
+            margin += zfh.headerSize;
+            /* Add the checksum to our margin */
+            margin += zfh.checksumFlag ? 4 : 0;
+            /* Add 3 bytes per block */
+            margin += 3 * frameSizeInfo.nbBlocks;
+
+            /* Compute the max block size */
+            maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax);
+        } else {
+            assert(zfh.frameType == ZSTD_skippableFrame);
+            /* Add the entire skippable frame size to our margin. */
+            margin += compressedSize;
+        }
+
+        assert(srcSize >= compressedSize);
+        src = (const BYTE*)src + compressedSize;
+        srcSize -= compressedSize;
+    }
+
+    /* Add the max block size back to the margin. */
+    margin += maxBlockSize;
+
+    return margin;
+}
 
 /*-*************************************************************
  *   Frame decoding
@@ -850,7 +893,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
         if (srcSize == 0) return 0;
         RETURN_ERROR(dstBuffer_null, "");
     }
-    ZSTD_memcpy(dst, src, srcSize);
+    ZSTD_memmove(dst, src, srcSize);
     return srcSize;
 }
 
@@ -928,6 +971,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
 
     /* Loop on each block */
     while (1) {
+        BYTE* oBlockEnd = oend;
         size_t decodedSize;
         blockProperties_t blockProperties;
         size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
@@ -937,16 +981,34 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         remainingSrcSize -= ZSTD_blockHeaderSize;
         RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
 
+        if (ip >= op && ip < oBlockEnd) {
+            /* We are decompressing in-place. Limit the output pointer so that we
+             * don't overwrite the block that we are currently reading. This will
+             * fail decompression if the input & output pointers aren't spaced
+             * far enough apart.
+             *
+             * This is important to set, even when the pointers are far enough
+             * apart, because ZSTD_decompressBlock_internal() can decide to store
+             * literals in the output buffer, after the block it is decompressing.
+             * Since we don't want anything to overwrite our input, we have to tell
+             * ZSTD_decompressBlock_internal to never write past ip.
+             *
+             * See ZSTD_allocateLiteralsBuffer() for reference.
+             */
+            oBlockEnd = op + (ip - op);
+        }
+
         switch(blockProperties.blockType)
         {
         case bt_compressed:
-            decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
+            decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming);
             break;
         case bt_raw :
+            /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */
             decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
             break;
         case bt_rle :
-            decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize);
+            decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize);
             break;
         case bt_reserved :
         default:
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 9f53d4cbd..dd173251d 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -242,6 +242,13 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
         frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
         frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
     }
+    /* In all cases, decompressedBound == nbBlocks * ZSTD_BLOCKSIZE_MAX.
+     * So we can compute nbBlocks without having to change every function.
+     */
+    if (frameSizeInfo.decompressedBound != ZSTD_CONTENTSIZE_ERROR) {
+        assert((frameSizeInfo.decompressedBound & (ZSTD_BLOCKSIZE_MAX - 1)) == 0);
+        frameSizeInfo.nbBlocks = (size_t)(frameSizeInfo.decompressedBound / ZSTD_BLOCKSIZE_MAX);
+    }
     return frameSizeInfo;
 }
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 480d65f67..22c8bba5b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1427,6 +1427,51 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size
  *           or an error code (if srcSize is too small) */
 ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
 
+/*! ZSTD_decompressionMargin() :
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ * and the input buffer must be at the end of the output buffer.
+ *
+ *  _______________________ Output Buffer ________________________
+ * |                                                              |
+ * |                                        ____ Input Buffer ____|
+ * |                                       |                      |
+ * v                                       v                      v
+ * |---------------------------------------|-----------|----------|
+ * ^                                                   ^          ^
+ * |___________________ Output_Size ___________________|_ Margin _|
+ *
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ * ZSTD_decompressDCtx().
+ * NOTE: This function supports multi-frame input.
+ *
+ * @param src The compressed frame(s)
+ * @param srcSize The size of the compressed frame(s)
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
+/*! ZSTD_DECOMPRESS_MARGIN() :
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ * the compressed frame, compute it from the original size and the blockSizeLog.
+ * See ZSTD_decompressionMargin() for details.
+ *
+ * WARNING: This macro does not support multi-frame input, the input must be a single
+ * zstd frame. If you need that support use the function, or implement it yourself.
+ *
+ * @param originalSize The original uncompressed size of the data.
+ * @param blockSize    The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+ *                     Unless you explicitly set the windowLog smaller than
+ *                     ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ */
+#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)(                                              \
+        ZSTD_FRAMEHEADERSIZE_MAX                                                              /* Frame header */ + \
+        4                                                                                         /* checksum */ + \
+        ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+        (blockSize)                                                                    /* One block of margin */   \
+    ))
+
 typedef enum {
   ZSTD_sf_noBlockDelimiters = 0,         /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
   ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 23a805af2..c2c69d950 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -26,6 +26,23 @@
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
 
+static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_t srcSize, int hasSmallBlocks)
+{
+    size_t margin = ZSTD_decompressionMargin(compressed, cSize);
+    if (!hasSmallBlocks) {
+        /* The macro should be correct in this case, but it may be smaller
+         * because of e.g. block splitting, so take the smaller of the two.
+         */
+        ZSTD_frameHeader zfh;
+        size_t marginM;
+        FUZZ_ZASSERT(ZSTD_getFrameHeader(&zfh, compressed, cSize));
+        marginM = ZSTD_DECOMPRESSION_MARGIN(srcSize, zfh.blockSizeMax);
+        if (marginM < margin)
+            margin = marginM;
+    }
+    return margin;
+}
+
 static size_t roundTripTest(void *result, size_t resultCapacity,
                             void *compressed, size_t compressedCapacity,
                             const void *src, size_t srcSize,
@@ -67,6 +84,25 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
     }
     dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
     FUZZ_ZASSERT(dSize);
+    FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
+    FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, result, dSize), "Corruption!");
+
+    {
+        size_t margin = getDecompressionMargin(compressed, cSize, srcSize, targetCBlockSize);
+        size_t const outputSize = srcSize + margin;
+        char* const output = (char*)FUZZ_malloc(outputSize);
+        char* const input = output + outputSize - cSize;
+        FUZZ_ASSERT(outputSize >= cSize);
+        memcpy(input, compressed, cSize);
+
+        dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
+        FUZZ_ZASSERT(dSize);
+        FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, srcSize), "Corruption!");
+
+        free(output);
+    }
+
     /* When superblock is enabled make sure we don't expand the block more than expected.
      * NOTE: This test is currently disabled because superblock mode can arbitrarily
      * expand the block in the worst case. Once superblock mode has been improved we can
@@ -120,13 +156,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         FUZZ_ASSERT(dctx);
     }
 
-    {
-        size_t const result =
-            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
-        FUZZ_ZASSERT(result);
-        FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
-        FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!");
-    }
+    roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
     free(rBuf);
     free(cBuf);
     FUZZ_dataProducer_free(producer);
diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index 8a28907b6..fae9ccbf4 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -166,6 +166,24 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         FUZZ_ZASSERT(rSize);
         FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!");
+
+        /* Test in-place decompression (note the macro doesn't work in this case) */
+        {
+            size_t const margin = ZSTD_decompressionMargin(cBuf, cSize);
+            size_t const outputSize = size + margin;
+            char* const output = (char*)FUZZ_malloc(outputSize);
+            char* const input = output + outputSize - cSize;
+            size_t dSize;
+            FUZZ_ASSERT(outputSize >= cSize);
+            memcpy(input, cBuf, cSize);
+
+            dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
+            FUZZ_ZASSERT(dSize);
+            FUZZ_ASSERT_MSG(dSize == size, "Incorrect regenerated size");
+            FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, size), "Corruption!");
+
+            free(output);
+        }
     }
 
     FUZZ_dataProducer_free(producer);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 3ad8ced5e..ce0bea573 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1220,6 +1220,60 @@ static int basicUnitTests(U32 const seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
+    cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
+    CHECK_Z(cSize);
+    CHECK_LT(CNBuffSize, cSize);
+    {
+        size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
+        size_t const outputSize = (CNBuffSize + margin);
+        char* output = malloc(outputSize);
+        char* input = output + outputSize - cSize;
+        CHECK_LT(cSize, CNBuffSize + margin);
+        CHECK(output != NULL);
+        CHECK_Z(margin);
+        CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
+        memcpy(input, compressedBuffer, cSize);
+
+        {
+            size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
+            CHECK_Z(dSize);
+            CHECK_EQ(dSize, CNBuffSize);
+        }
+        CHECK(!memcmp(output, CNBuffer, CNBuffSize));
+        free(output);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
+    cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
+    CHECK_Z(cSize);
+    {
+        size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
+        CHECK_Z(cSize2);
+        cSize += cSize2;
+    }
+    {
+        size_t const srcSize = (CNBuffSize / 3) * 2;
+        size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
+        size_t const outputSize = (CNBuffSize + margin);
+        char* output = malloc(outputSize);
+        char* input = output + outputSize - cSize;
+        CHECK_LT(cSize, CNBuffSize + margin);
+        CHECK(output != NULL);
+        CHECK_Z(margin);
+        memcpy(input, compressedBuffer, cSize);
+
+        {
+            size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
+            CHECK_Z(dSize);
+            CHECK_EQ(dSize, srcSize);
+        }
+        CHECK(!memcmp(output, CNBuffer, srcSize));
+        free(output);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
     {
         ZSTD_CCtx* const cctx = ZSTD_createCCtx();

From bcfb7ad03c47d3018eeaec2c68be9d37dc114151 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 12 Jan 2023 19:00:27 -0800
Subject: [PATCH 110/197] refactor timefn

The timer storage type is no longer dependent on OS.
This will make it possible to re-enable posix precise timers
since the timer storage type will no longer be sensible to #include order.
See #3168 for details of pbs of previous interface.

Suggestion by @terrelln
---
 programs/benchzstd.c         |  7 +---
 programs/dibio.c             |  9 ++--
 programs/fileio_common.h     |  2 +-
 programs/timefn.c            | 79 ++++++++++++++++++++----------------
 programs/timefn.h            | 52 +++++++-----------------
 tests/decodecorpus.c         | 27 +++++++-----
 tests/external_matchfinder.h |  2 +-
 tests/fuzzer.c               | 22 +++++-----
 8 files changed, 94 insertions(+), 106 deletions(-)

diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 285e401ef..7770dd2a5 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -387,12 +387,9 @@ BMK_benchMemAdvancedNoAlloc(
         RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
     }
 
-#if defined(UTIL_TIME_USES_C90_CLOCK)
-    if (adv->nbWorkers > 1) {
-        OUTPUTLEVEL(2, "Warning : time measurements restricted to C90 clock_t. \n")
-        OUTPUTLEVEL(2, "Warning : using C90 clock_t leads to incorrect measurements in multithreading mode. \n")
+    if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) {
+        OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n")
     }
-#endif
 
     /* Bench */
     {   U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
diff --git a/programs/dibio.c b/programs/dibio.c
index b21338cd8..26ebe5ca1 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -274,21 +274,20 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
     int n;
     memset(&fs, 0, sizeof(fs));
 
-    // We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX
+    /* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */
     assert( chunkSize <= SAMPLESIZE_MAX );
 
     for (n=0; n<nbFiles; n++) {
       S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
-      // TODO: is there a minimum sample size? What if the file is 1-byte?
+      /* TODO: is there a minimum sample size? What if the file is 1-byte? */
       if (fileSize == 0) {
         DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
         continue;
       }
 
       /* the case where we are breaking up files in sample chunks */
-      if (chunkSize > 0)
-      {
-        // TODO: is there a minimum sample size? Can we have a 1-byte sample?
+      if (chunkSize > 0) {
+        /* TODO: is there a minimum sample size? Can we have a 1-byte sample? */
         fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);
         fs.totalSizeToLoad += fileSize;
       }
diff --git a/programs/fileio_common.h b/programs/fileio_common.h
index 827a5a06b..55491b8e3 100644
--- a/programs/fileio_common.h
+++ b/programs/fileio_common.h
@@ -122,4 +122,4 @@ extern UTIL_time_t g_displayClock;
 #if defined (__cplusplus)
 }
 #endif
-#endif //ZSTD_FILEIO_COMMON_H
+#endif /* ZSTD_FILEIO_COMMON_H */
diff --git a/programs/timefn.c b/programs/timefn.c
index 08aa1cfcb..023e2136c 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -13,6 +13,7 @@
 
 #include "timefn.h"
 
+#include <time.h>     /* TIME_UTC, then struct timespec and clock_t */
 
 /*-****************************************
 *  Time functions
@@ -20,12 +21,11 @@
 
 #if defined(_WIN32)   /* Windows */
 
+#include <windows.h>  /* LARGE_INTEGER */
 #include <stdlib.h>   /* abort */
 #include <stdio.h>    /* perror */
 
-UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
 {
     static LARGE_INTEGER ticksPerSecond;
     static int init = 0;
@@ -36,16 +36,18 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         }
         init = 1;
     }
-    return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+    {   UTIL_time_t r;
+        r.t = (PTime)(QueryPerformanceCounter(&x).QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
+        return ;
+    }
 }
 
 
-
 #elif defined(__APPLE__) && defined(__MACH__)
 
-UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+#include <mach/mach_time.h> /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */
 
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
 {
     static mach_timebase_info_data_t rate;
     static int init = 0;
@@ -53,12 +55,18 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         mach_timebase_info(&rate);
         init = 1;
     }
-    return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
+    {   UTIL_time_t r;
+        r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom;
+        return r;
+    }
 }
 
 
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
-   Android also lacks it but does define TIME_UTC. */
+/* C11 requires timespec_get().
+ * However, FreeBSD 11 claims C11 compliance but lacks timespec_get().
+ * Double confirm by requiring the definition of TIME_UTC.
+ * However, some versions of Android manage to simultanously define TIME_UTC
+ * and lack timespec_get() support... */
 #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
     && defined(TIME_UTC) && !defined(__ANDROID__)
 
@@ -69,46 +77,38 @@ UTIL_time_t UTIL_getTime(void)
 {
     /* time must be initialized, othersize it may fail msan test.
      * No good reason, likely a limitation of timespec_get() for some target */
-    UTIL_time_t time = UTIL_TIME_INITIALIZER;
+    struct timespec time = { 0, 0 };
     if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
         perror("timefn::timespec_get");
         abort();
     }
-    return time;
-}
-
-static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t diff;
-    if (end.tv_nsec < begin.tv_nsec) {
-        diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
-        diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
-    } else {
-        diff.tv_sec = end.tv_sec - begin.tv_sec;
-        diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
+    {   UTIL_time_t r;
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+        return r;
     }
-    return diff;
-}
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    PTime nano = 0;
-    nano += 1000000000ULL * diff.tv_sec;
-    nano += diff.tv_nsec;
-    return nano;
 }
 
 
 #else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
 
-UTIL_time_t UTIL_getTime(void) { return clock(); }
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+UTIL_time_t UTIL_getTime(void)
+{
+    UTIL_time_t r;
+    r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC;
+    return r;
+}
+
+#define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
 
 #endif
 
 /* ==== Common functions, valid for all time API ==== */
 
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    return clockEnd.t - clockStart.t;
+}
+
 PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
 {
     return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
@@ -134,3 +134,12 @@ void UTIL_waitForNextTick(void)
         clockEnd = UTIL_getTime();
     } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
 }
+
+int UTIL_support_MT_measurements(void)
+{
+# if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED)
+    return 0;
+# else
+    return 1;
+# endif
+}
diff --git a/programs/timefn.h b/programs/timefn.h
index 3e4476a4e..b814ff8d8 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -18,7 +18,7 @@ extern "C" {
 
 
 /*-****************************************
-*  Local Types
+*  Types
 ******************************************/
 
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
@@ -32,40 +32,11 @@ extern "C" {
   typedef unsigned long long PTime;  /* does not support compilers without long long support */
 #endif
 
-
-
-/*-****************************************
-*  Time types (note: OS dependent)
-******************************************/
-#include <time.h>     /* TIME_UTC, then struct timespec and clock_t */
-
-#if defined(_WIN32)   /* Windows */
-
-    #include <windows.h>   /* LARGE_INTEGER */
-    typedef LARGE_INTEGER UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
-
-#elif defined(__APPLE__) && defined(__MACH__)
-
-    #include <mach/mach_time.h>
-    typedef PTime UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
-   Android also lacks it but does define TIME_UTC. */
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
-    && defined(TIME_UTC) && !defined(__ANDROID__)
-
-    typedef struct timespec UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { 0, 0 }
-
-#else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
-
-    #define UTIL_TIME_USES_C90_CLOCK
-    typedef clock_t UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-#endif
+/* UTIL_time_t contains a nanosecond time counter.
+ * The absolute value is not meaningful.
+ * It's only valid to compute the difference between 2 measurements. */
+typedef struct { PTime t; } UTIL_time_t;
+#define UTIL_TIME_INITIALIZER { 0 }
 
 
 /*-****************************************
@@ -73,16 +44,23 @@ extern "C" {
 ******************************************/
 
 UTIL_time_t UTIL_getTime(void);
+
+/* Timer resolution can be low on some platforms.
+ * To improve accuracy, it's recommended to wait for a new tick
+ * before starting benchmark measurements */
 void UTIL_waitForNextTick(void);
+/* tells if timefn will return correct time measurements
+ * in presence of multi-threaded workload.
+ * note : this is not the case if only C90 clock_t measurements are available */
+int UTIL_support_MT_measurements(void);
 
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
 
-#define SEC_TO_MICRO ((PTime)1000000)
 PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
 
-
+#define SEC_TO_MICRO ((PTime)1000000)  /* nb of microseconds in a second */
 
 
 #if defined (__cplusplus)
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 15e89d51f..86bbaf8bf 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -25,21 +25,13 @@
 #include "zdict.h"
 
 /* Direct access to internal compression functions is required */
-#include "zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
+#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
 
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
 
-#ifndef MIN
-    #define MIN(a, b) ((a) < (b) ? (a) : (b))
-#endif
-
-#ifndef MAX_PATH
-    #ifdef PATH_MAX
-        #define MAX_PATH PATH_MAX
-    #else
-        #define MAX_PATH 256
-    #endif
+#if !(defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */))
+# define inline  /* disable */
 #endif
 
 /*-************************************
@@ -71,6 +63,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
         }                                                                      \
     } while (0)
 
+
 /*-*******************************************************
 *  Random function
 *********************************************************/
@@ -176,6 +169,14 @@ const char* BLOCK_TYPES[] = {"raw", "rle", "compressed"};
 #define MIN_SEQ_LEN (3)
 #define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
 
+#ifndef MAX_PATH
+    #ifdef PATH_MAX
+        #define MAX_PATH PATH_MAX
+    #else
+        #define MAX_PATH 256
+    #endif
+#endif
+
 BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
 BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
 BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];
@@ -241,6 +242,10 @@ typedef enum {
   gt_block,      /* generate compressed blocks without block/frame headers */
 } genType_e;
 
+#ifndef MIN
+    #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
 /*-*******************************************************
 *  Global variables (set from command line)
 *********************************************************/
diff --git a/tests/external_matchfinder.h b/tests/external_matchfinder.h
index 041f73e4d..e4f7c95f0 100644
--- a/tests/external_matchfinder.h
+++ b/tests/external_matchfinder.h
@@ -32,4 +32,4 @@ size_t zstreamExternalMatchFinder(
   size_t windowSize
 );
 
-#endif // EXTERNAL_MATCHFINDER
+#endif /* EXTERNAL_MATCHFINDER */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ce0bea573..cb46dc45b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -476,7 +476,7 @@ static void test_compressBound(unsigned tnb)
             CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
     }   }
 
-    // Ensure error if srcSize too big
+    /* Ensure error if srcSize too big */
     {   size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
         CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
         CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
@@ -489,7 +489,7 @@ static void test_decompressBound(unsigned tnb)
 {
     DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
 
-    // Simple compression, with size : should provide size;
+    /* Simple compression, with size : should provide size; */
     {   const char example[] = "abcd";
         char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
         size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
@@ -497,7 +497,7 @@ static void test_decompressBound(unsigned tnb)
         CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
     }
 
-    // Simple small compression without size : should provide 1 block size
+    /* Simple small compression without size : should provide 1 block size */
     {   char cBuffer[ZSTD_COMPRESSBOUND(0)];
         ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
         ZSTD_inBuffer in = { NULL, 0, 0 };
@@ -510,14 +510,14 @@ static void test_decompressBound(unsigned tnb)
         ZSTD_freeCCtx(cctx);
     }
 
-    // Attempt to overflow 32-bit intermediate multiplication result
-    // This requires dBound >= 4 GB, aka 2^32.
-    // This requires 2^32 / 2^17 = 2^15 blocks
-    // => create 2^15 blocks (can be empty, or just 1 byte).
+    /* Attempt to overflow 32-bit intermediate multiplication result
+     * This requires dBound >= 4 GB, aka 2^32.
+     * This requires 2^32 / 2^17 = 2^15 blocks
+     * => create 2^15 blocks (can be empty, or just 1 byte). */
     {   const char input[] = "a";
         size_t const nbBlocks = (1 << 15) + 1;
         size_t blockNb;
-        size_t const outCapacity = 1 << 18; // large margin
+        size_t const outCapacity = 1 << 18; /* large margin */
         char* const outBuffer = malloc (outCapacity);
         ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
         ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -3535,7 +3535,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
     {
-        U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed)
+        U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
         U32 rand32 = FUZ_rand(&seed_copy);
         U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
         U32 lowbit_only_32 = 1;
@@ -3543,8 +3543,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         U32 highbit_only_32 = (U32)1 << 31;
         U64 highbit_only_64 = (U64)1 << 63;
         U32 i;
-        if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0
-        if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0
+        if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
+        if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
 
         /* Test ZSTD_countTrailingZeros32 */
         CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);

From a2ef23dec03a18978dc945cee557d280160f8cb6 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 12 Jan 2023 20:45:11 -0800
Subject: [PATCH 111/197] restore support of clock_gettime() for POSIX systems

This should notably allow posix systems with timespec_get()
to have access to a high resolution timer,
instead of falling back to C90's clock_t.
---
 programs/timefn.c | 39 +++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/programs/timefn.c b/programs/timefn.c
index 023e2136c..f941e57e6 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -12,8 +12,8 @@
 /* ===  Dependencies  === */
 
 #include "timefn.h"
-
-#include <time.h>     /* TIME_UTC, then struct timespec and clock_t */
+#include "platform.h" /* set _POSIX_C_SOURCE */
+#include <time.h>     /* CLOCK_MONOTONIC, TIME_UTC */
 
 /*-****************************************
 *  Time functions
@@ -37,8 +37,10 @@ UTIL_time_t UTIL_getTime(void)
         init = 1;
     }
     {   UTIL_time_t r;
-        r.t = (PTime)(QueryPerformanceCounter(&x).QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
-        return ;
+        LARGE_INTEGER x;
+        QueryPerformanceCounter(&x);
+        r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
+        return r;
     }
 }
 
@@ -61,10 +63,31 @@ UTIL_time_t UTIL_getTime(void)
     }
 }
 
+/* POSIX.1-2001 (optional) */
+#elif defined(CLOCK_MONOTONIC)
 
-/* C11 requires timespec_get().
- * However, FreeBSD 11 claims C11 compliance but lacks timespec_get().
- * Double confirm by requiring the definition of TIME_UTC.
+#include <stdlib.h>   /* abort */
+#include <stdio.h>    /* perror */
+
+UTIL_time_t UTIL_getTime(void)
+{
+    /* time must be initialized, othersize it may fail msan test.
+     * No good reason, likely a limitation of timespec_get() for some target */
+    struct timespec time = { 0, 0 };
+    if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
+        perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
+        abort();
+    }
+    {   UTIL_time_t r;
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+        return r;
+    }
+}
+
+
+/* C11 requires support of timespec_get().
+ * However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
+ * Double confirm timespec_get() support by checking the definition of TIME_UTC.
  * However, some versions of Android manage to simultanously define TIME_UTC
  * and lack timespec_get() support... */
 #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
@@ -79,7 +102,7 @@ UTIL_time_t UTIL_getTime(void)
      * No good reason, likely a limitation of timespec_get() for some target */
     struct timespec time = { 0, 0 };
     if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
-        perror("timefn::timespec_get");
+        perror("timefn::timespec_get(TIME_UTC)");
         abort();
     }
     {   UTIL_time_t r;

From 14b8defb86b3e25086fafb720a4b90048a7bd779 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Fri, 13 Jan 2023 07:00:50 -0800
Subject: [PATCH 112/197] move ZSTD_BLOCKSIZE_MAX_MIN to static linking only
 section

---
 lib/zstd.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 9d1d507ff..dfe420001 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -141,7 +141,6 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
 
 #define ZSTD_BLOCKSIZELOG_MAX  17
 #define ZSTD_BLOCKSIZE_MAX     (1<<ZSTD_BLOCKSIZELOG_MAX)
-#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 /***************************************
@@ -1203,6 +1202,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 #define ZSTD_TARGETLENGTH_MIN     0   /* note : comparing this constant to an unsigned results in a tautological test */
 #define ZSTD_STRATEGY_MIN        ZSTD_fast
 #define ZSTD_STRATEGY_MAX        ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 #define ZSTD_OVERLAPLOG_MIN       0

From f10922a8fa84d332a4d001bc2a1f659a00ae0213 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 13 Jan 2023 11:51:59 -0500
Subject: [PATCH 113/197] Disable Custom ASAN/MSAN Poisoning on MinGW Builds

Addresses #3240.
---
 lib/common/compiler.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index e423f2467..4ad55e601 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -281,6 +281,17 @@
 *  Sanitizer
 *****************************************************************/
 
+/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
+ * abundance of caution, disable our custom poisoning on mingw. */
+#ifdef __MINGW32__
+#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE
+#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1
+#endif
+#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE
+#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1
+#endif
+#endif
+
 #if ZSTD_MEMORY_SANITIZER
 /* Not all platforms that support msan provide sanitizers/msan_interface.h.
  * We therefore declare the functions we need ourselves, rather than trying to

From d78fbedd968988270c27c215ae5f592990f0bc1d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 13 Jan 2023 11:56:48 -0500
Subject: [PATCH 114/197] Don't Even Declare Poisoning Functions if Poisoning
 is Disabled

This guarantees that we won't accidentally forget to check the macro somewhere
where we use these functions.
---
 lib/common/compiler.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 4ad55e601..d4f2f285d 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -292,7 +292,7 @@
 #endif
 #endif
 
-#if ZSTD_MEMORY_SANITIZER
+#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE)
 /* Not all platforms that support msan provide sanitizers/msan_interface.h.
  * We therefore declare the functions we need ourselves, rather than trying to
  * include the header file... */
@@ -313,7 +313,7 @@ void __msan_poison(const volatile void *a, size_t size);
 intptr_t __msan_test_shadow(const volatile void *x, size_t size);
 #endif
 
-#if ZSTD_ADDRESS_SANITIZER
+#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE)
 /* Not all platforms that support asan provide sanitizers/asan_interface.h.
  * We therefore declare the functions we need ourselves, rather than trying to
  * include the header file... */

From 2086e7396e659143dfe0c6d292e63f1332ec496d Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Jan 2023 11:38:27 -0800
Subject: [PATCH 115/197] missing #include for Windows

---
 tests/fuzzer.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index cb46dc45b..43fa1158d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -25,7 +25,8 @@
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
 #include <string.h>       /* strcmp */
-#undef NDEBUG
+#include <time.h>         /* time(), time_t */
+#undef NDEBUG             /* always enable assert() */
 #include <assert.h>
 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "debug.h"        /* DEBUG_STATIC_ASSERT */

From 5d8cfa6b96a6442ab1251f9de3b47a0eb12561a0 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Fri, 13 Jan 2023 14:51:47 -0500
Subject: [PATCH 116/197] Deprecate advanced streaming functions (#3408)

* deprecate advanced streaming functions

* remove internal usage of the deprecated functions

* nit

* suppress warnings in tests/zstreamtest.c

* purge ZSTD_initDStream_usingDict

* nits

* c90 compat

* zstreamtest.c already disables deprecation warnings!

* fix initDStream() return value

* fix typo

* wasn't able to import private symbol properly, this commit works around that

* new strategy for zbuff

* undo zbuff deprecation warning changes

* move ZSTD_DISABLE_DEPRECATE_WARNINGS from .h to .c
---
 lib/decompress/zstd_decompress.c  | 6 +++++-
 lib/deprecated/zbuff_decompress.c | 2 ++
 lib/zstd.h                        | 6 +++---
 programs/fileio.c                 | 3 ++-
 zlibWrapper/examples/zwrapbench.c | 6 ++++--
 zlibWrapper/zstd_zlibwrapper.c    | 6 ++++--
 6 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index fd82a6540..770dd8b19 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1719,7 +1719,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 size_t ZSTD_initDStream(ZSTD_DStream* zds)
 {
     DEBUGLOG(4, "ZSTD_initDStream");
-    return ZSTD_initDStream_usingDDict(zds, NULL);
+    FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), "");
+    FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), "");
+    return ZSTD_startingInputLength(zds->format);
 }
 
 /* ZSTD_initDStream_usingDDict() :
@@ -1727,6 +1729,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
  * this function cannot fail */
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
 {
+    DEBUGLOG(4, "ZSTD_initDStream_usingDDict");
     FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
     FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
     return ZSTD_startingInputLength(dctx->format);
@@ -1737,6 +1740,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
  * this function cannot fail */
 size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
 {
+    DEBUGLOG(4, "ZSTD_resetDStream");
     FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
     return ZSTD_startingInputLength(dctx->format);
 }
diff --git a/lib/deprecated/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
index 51159ef53..12a66af74 100644
--- a/lib/deprecated/zbuff_decompress.c
+++ b/lib/deprecated/zbuff_decompress.c
@@ -13,6 +13,8 @@
 /* *************************************
 *  Dependencies
 ***************************************/
+#define ZSTD_DISABLE_DEPRECATE_WARNINGS  /* suppress warning on ZSTD_initDStream_usingDict */
+#include "../zstd.h"        /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
 #define ZBUFF_STATIC_LINKING_ONLY
 #include "zbuff.h"
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 22c8bba5b..f47c24676 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2510,8 +2510,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
  *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
  *
  * note: no dictionary will be used if dict == NULL or dictSize < 8
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 
 /*!
@@ -2521,8 +2521,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const vo
  *     ZSTD_DCtx_refDDict(zds, ddict);
  *
  * note : ddict is referenced, it must outlive decompression session
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 
 /*!
@@ -2531,8 +2531,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const Z
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
  *
  * re-use decompression parameters from previous init; saves dictionary loading
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
 
diff --git a/programs/fileio.c b/programs/fileio.c
index 1e3486683..bc0b1ac39 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1995,7 +1995,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
     /* dictionary */
     {   void* dictBuffer;
         size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs);
-        CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) );
+        CHECK( ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
+        CHECK( ZSTD_DCtx_loadDictionary(ress.dctx, dictBuffer, dictBufferSize) );
         free(dictBuffer);
     }
 
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index aef29be0b..3bc9a1ab0 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -428,8 +428,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                     ZSTD_DStream* zbd = ZSTD_createDStream();
                     size_t rSize;
                     if (zbd == NULL) EXM_THROW(1, "ZSTD_createDStream() allocation failure");
-                    rSize = ZSTD_initDStream_usingDict(zbd, dictBuffer, dictBufferSize);
-                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initDStream() failed : %s", ZSTD_getErrorName(rSize));
+                    rSize = ZSTD_DCtx_reset(zbd, ZSTD_reset_session_only);
+                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_reset() failed : %s", ZSTD_getErrorName(rSize));
+                    rSize = ZSTD_DCtx_loadDictionary(zbd, dictBuffer, dictBufferSize);
+                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_loadDictionary() failed : %s", ZSTD_getErrorName(rSize));
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index f31cb59db..479ddd4e0 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -706,8 +706,10 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary _Z_OF((z_streamp strm,
 
     {   ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
         if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
-        { size_t const initErr = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
-          if (ZSTD_isError(initErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
+        { size_t const resetErr = ZSTD_DCtx_reset(zwd->zbd, ZSTD_reset_session_only);
+          if (ZSTD_isError(resetErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
+        { size_t const loadErr = ZSTD_DCtx_loadDictionary(zwd->zbd, dictionary, dictLength);
+          if (ZSTD_isError(loadErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
         zwd->decompState = ZWRAP_useReset;
 
         if (zwd->totalInBytes == ZSTD_HEADERSIZE) {

From 3add5ca3ef874483cf50ef134eb8477d0300c95a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Jan 2023 17:22:19 -0500
Subject: [PATCH 117/197] Bump github/codeql-action from 2.1.37 to 2.1.38
 (#3428)

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.1.37 to 2.1.38.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/959cbb7472c4d4ad70cdfe6f4976053fe48ab394...515828d97454b8354517688ddc5b48402b723750)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/scorecards.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 6e99bc168..7fafe8e8a 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -59,6 +59,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@959cbb7472c4d4ad70cdfe6f4976053fe48ab394 # tag=v2.1.37
+        uses: github/codeql-action/upload-sarif@515828d97454b8354517688ddc5b48402b723750 # tag=v2.1.38
         with:
           sarif_file: results.sarif

From 018b68f332213e75aaf8fb20524c4306dad6b1aa Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Tue, 17 Jan 2023 12:10:15 -0500
Subject: [PATCH 118/197] fix msys2 symlink breakage in CI (#3429)

---
 .github/workflows/dev-short-tests.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index d0dba3209..5a29789f9 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -384,6 +384,7 @@ jobs:
   mingw-short-test:
     runs-on: windows-latest
     strategy:
+      fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
       matrix:
         include: [
           { compiler: gcc, msystem: MINGW32, cflags: "-Werror"},
@@ -401,18 +402,18 @@ jobs:
         install: make diffutils
         update: true
     # Based on https://ariya.io/2020/07/on-github-actions-with-msys2
-    - name: install mingw gcc
+    - name: install mingw gcc i686
       if: ${{ (matrix.msystem == 'MINGW32') && (matrix.compiler == 'gcc') }}
       run: pacman --noconfirm -S mingw-w64-i686-gcc
-    - name: install mingw gcc
+    - name: install mingw gcc x86_64
       if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'gcc') }}
       run: pacman --noconfirm -S mingw-w64-x86_64-gcc
-    - name: install mingw clang x86_64
-      if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'clang') }}
-      run: pacman --noconfirm -S mingw-w64-x86_64-clang
     - name: install mingw clang i686
       if: ${{ (matrix.msystem == 'MINGW32') && (matrix.compiler == 'clang') }}
       run: pacman --noconfirm -S mingw-w64-i686-clang
+    - name: install mingw clang x86_64
+      if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'clang') }}
+      run: pacman --noconfirm -S mingw-w64-x86_64-clang
     - name: run mingw tests
       run: |
         make -v
@@ -421,7 +422,7 @@ jobs:
         CFLAGS="${{ matrix.cflags }}" make -j allzstd
         echo "Testing $CC ${{ matrix.msystem }}"
         make clean
-        make check
+        MSYS="" make check
 
   visual-runtime-tests:
     runs-on: windows-latest

From 8353a4b095ca71dcf64a1507952001c8fa34cbee Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 17 Jan 2023 12:24:18 -0800
Subject: [PATCH 119/197] fix maxBlockSize resolution + add test cases

---
 lib/compress/zstd_compress.c |  5 +-
 tests/fuzzer.c               | 88 ++++++++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 2ff1fa209..0fa2e6e2d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1577,7 +1577,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         size_t maxBlockSize)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
-    size_t const blockSize = MIN(maxBlockSize, windowSize);
+    size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
     size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
@@ -1679,7 +1679,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-        size_t const blockSize = MIN(params->maxBlockSize, (size_t)1 << cParams.windowLog);
+        size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
         size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
                 ? ((size_t)1 << cParams.windowLog) + blockSize
                 : 0;
@@ -1964,6 +1964,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     assert(params->useRowMatchFinder != ZSTD_ps_auto);
     assert(params->useBlockSplitter != ZSTD_ps_auto);
     assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+    assert(params->maxBlockSize != 0);
     if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
         /* Adjust long distance matching parameters */
         ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 3ad8ced5e..ef1fef14f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1725,6 +1725,94 @@ static int basicUnitTests(U32 const seed, double compressibility)
                 if (!ZSTD_isError(r)) goto _output_error;
             }
             DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
+            {
+                // Test ZSTD_estimateCCtxSize_usingCCtxParams
+                {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3));
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+
+                // Test ZSTD_estimateCStreamSize_usingCCtxParams
+                  {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+            }
+            DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
+            {
+                // Test ZSTD_estimateCCtxSize_usingCCtxParams
+                {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t cctxSizeDefault;
+                    CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+                    cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+
+                // Test ZSTD_estimateCStreamSize_usingCCtxParams
+                  {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t cctxSizeDefault;
+                    CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+                    cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+            }
+            DISPLAYLEVEL(3, "OK \n");
         }
         free(staticCCtxBuffer);
         free(staticDCtxBuffer);

From 2ad6855ac1b90516b1670684bb20126f7b17ef4b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Jan 2023 14:01:06 -0800
Subject: [PATCH 120/197] `FIO_openSrcFile()` Returns File Stat

---
 programs/fileio.c | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index bc0b1ac39..f8bbb80f9 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -531,26 +531,26 @@ static int FIO_removeFile(const char* path)
 /** FIO_openSrcFile() :
  *  condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
  * @result : FILE* to `srcFileName`, or NULL if it fails */
-static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName)
+static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
 {
-    stat_t statbuf;
     int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0;
     assert(srcFileName != NULL);
+    assert(statbuf != NULL);
     if (!strcmp (srcFileName, stdinmark)) {
         DISPLAYLEVEL(4,"Using stdin for input \n");
         SET_BINARY_MODE(stdin);
         return stdin;
     }
 
-    if (!UTIL_stat(srcFileName, &statbuf)) {
+    if (!UTIL_stat(srcFileName, statbuf)) {
         DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
                         srcFileName, strerror(errno));
         return NULL;
     }
 
-    if (!UTIL_isRegularFileStat(&statbuf)
-     && !UTIL_isFIFOStat(&statbuf)
-     && !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf))
+    if (!UTIL_isRegularFileStat(statbuf)
+     && !UTIL_isFIFOStat(statbuf)
+     && !(allowBlockDevices && UTIL_isBlockDevStat(statbuf))
     ) {
         DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
                         srcFileName);
@@ -1709,6 +1709,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
 {
     int result;
     FILE* srcFile;
+    stat_t srcFileStat;
     DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
 
     /* ensure src is not a directory */
@@ -1732,7 +1733,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
         return 0;
     }
 
-    srcFile = FIO_openSrcFile(prefs, srcFileName);
+    srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
     if (srcFile == NULL) return 1;   /* srcFile could not be opened */
 
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
@@ -2524,6 +2525,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
 static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
 {
     FILE* srcFile;
+    stat_t srcFileStat;
     int result;
 
     if (UTIL_isDirectory(srcFileName)) {
@@ -2531,7 +2533,7 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs
         return 1;
     }
 
-    srcFile = FIO_openSrcFile(prefs, srcFileName);
+    srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
     if (srcFile==NULL) return 1;
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
 
@@ -2899,10 +2901,11 @@ static InfoError
 getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName)
 {
     InfoError status;
-    FILE* const srcFile = FIO_openSrcFile(NULL, inFileName);
+    stat_t srcFileStat;
+    FILE* const srcFile = FIO_openSrcFile(NULL, inFileName, &srcFileStat);
     ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName);
 
-    info->compressedSize = UTIL_getFileSize(inFileName);
+    info->compressedSize = UTIL_getFileSizeStat(&srcFileStat);
     status = FIO_analyzeFrames(info, srcFile);
 
     fclose(srcFile);

From 5653f96776640421783af3fe379741a1a2486706 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Jan 2023 14:05:15 -0800
Subject: [PATCH 121/197] Pass `stat_t` into `*_dstFile()` Functions

---
 programs/fileio.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index f8bbb80f9..9122a1036 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1626,6 +1626,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
                                         cRess_t ress,
                                         const char* dstFileName,
                                         const char* srcFileName,
+                                        const stat_t* srcFileStat,
                                         int compressionLevel)
 {
     int closeDstFile = 0;
@@ -1633,6 +1634,9 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
     stat_t statbuf;
     int transferMTime = 0;
     FILE *dstFile;
+
+    (void)srcFileStat;
+
     assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
     if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
         int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
@@ -1737,7 +1741,10 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
     if (srcFile == NULL) return 1;   /* srcFile could not be opened */
 
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
-    result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
+    result = FIO_compressFilename_dstFile(
+            fCtx, prefs, ress,
+            dstFileName, srcFileName,
+            &srcFileStat, compressionLevel);
     AIO_ReadPool_closeFile(ress.readCtx);
 
     if ( prefs->removeSrcFile   /* --rm */
@@ -2462,13 +2469,17 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
 static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
                                  FIO_prefs_t* const prefs,
                                  dRess_t ress,
-                                 const char* dstFileName, const char* srcFileName)
+                                 const char* dstFileName,
+                                 const char* srcFileName,
+                                 const stat_t* srcFileStat)
 {
     int result;
     stat_t statbuf;
     int releaseDstFile = 0;
     int transferMTime = 0;
 
+    (void)srcFileStat;
+
     if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
         FILE *dstFile;
         int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
@@ -2537,7 +2548,7 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs
     if (srcFile==NULL) return 1;
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
 
-    result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName);
+    result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName, &srcFileStat);
 
     AIO_ReadPool_setFile(ress.readCtx, NULL);
 

From a5ed28f1fb35bfd29053d5bc4d5ec7c8c1d25b83 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Jan 2023 14:08:22 -0800
Subject: [PATCH 122/197] Use Existing Src File Stat in `*_dstFile()` Funcs

One fewer `stat()` call to make per operation!
---
 programs/fileio.c                              | 18 ++++++------------
 .../compress-file-to-file.sh.stderr.exact      |  2 --
 .../decompress-file-to-file.sh.stderr.exact    |  2 --
 3 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 9122a1036..2067cfcda 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1631,20 +1631,16 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
 {
     int closeDstFile = 0;
     int result;
-    stat_t statbuf;
     int transferMTime = 0;
     FILE *dstFile;
 
-    (void)srcFileStat;
-
     assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
     if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
         int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
         if ( strcmp (srcFileName, stdinmark)
           && strcmp (dstFileName, stdoutmark)
-          && UTIL_stat(srcFileName, &statbuf)
-          && UTIL_isRegularFileStat(&statbuf) ) {
-            dstFilePermissions = statbuf.st_mode;
+          && UTIL_isRegularFileStat(srcFileStat) ) {
+            dstFilePermissions = srcFileStat->st_mode;
             transferMTime = 1;
         }
 
@@ -1671,7 +1667,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
             result=1;
         }
         if (transferMTime) {
-            UTIL_utime(dstFileName, &statbuf);
+            UTIL_utime(dstFileName, srcFileStat);
         }
         if ( (result != 0)  /* operation failure */
           && strcmp(dstFileName, stdoutmark)  /* special case : don't remove() stdout */
@@ -2474,7 +2470,6 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
                                  const stat_t* srcFileStat)
 {
     int result;
-    stat_t statbuf;
     int releaseDstFile = 0;
     int transferMTime = 0;
 
@@ -2485,9 +2480,8 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
         int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
         if ( strcmp(srcFileName, stdinmark)   /* special case : don't transfer permissions from stdin */
           && strcmp(dstFileName, stdoutmark)
-          && UTIL_stat(srcFileName, &statbuf)
-          && UTIL_isRegularFileStat(&statbuf) ) {
-            dstFilePermissions = statbuf.st_mode;
+          && UTIL_isRegularFileStat(srcFileStat) ) {
+            dstFilePermissions = srcFileStat->st_mode;
             transferMTime = 1;
         }
 
@@ -2514,7 +2508,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
         }
 
         if (transferMTime) {
-            UTIL_utime(dstFileName, &statbuf);
+            UTIL_utime(dstFileName, srcFileStat);
         }
 
         if ( (result != 0)  /* operation failure */
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
index 34bb704c7..7297894c7 100644
--- a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -12,8 +12,6 @@ Trace:FileStat:  < 1
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_stat(file)
 Trace:FileStat: < 1
-Trace:FileStat: > UTIL_stat(file)
-Trace:FileStat: < 1
 Trace:FileStat: > UTIL_isSameFile(file, file.zst)
 Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
index a2d7ad282..2af60eb43 100644
--- a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -10,8 +10,6 @@ Trace:FileStat:  < 1
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_stat(file.zst)
 Trace:FileStat: < 1
-Trace:FileStat: > UTIL_stat(file.zst)
-Trace:FileStat: < 1
 Trace:FileStat: > UTIL_isSameFile(file.zst, file)
 Trace:FileStat:  > UTIL_stat(file.zst)
 Trace:FileStat:  < 1

From 860548cd5b6a833b83b47a14baf13291e7c5bb14 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 20 Dec 2022 16:25:24 -0800
Subject: [PATCH 123/197] [tests] Fix version test determinism

The dictionary source files were taken from the `dev` branch before this
commit, which could introduce non-determinism on PR jobs. Instead take
the sources from the PR checkout.

This PR also adds stderr logging, and verbose output for the jobs that
are failing, to help catch the failure if it occurs again.
---
 tests/test-zstd-versions.py | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index 7117c1952..88b0578eb 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -29,8 +29,19 @@ test_dat_src = 'README.md'
 test_dat = 'test_dat'
 head = 'vdevel'
 dict_source = 'dict_source'
-dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
-dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+dict_globs = [
+    'programs/*.c',
+    'lib/common/*.c',
+    'lib/compress/*.c',
+    'lib/decompress/*.c',
+    'lib/dictBuilder/*.c',
+    'lib/legacy/*.c',
+    'programs/*.h',
+    'lib/common/*.h',
+    'lib/compress/*.h',
+    'lib/dictBuilder/*.h',
+    'lib/legacy/*.h'
+]
 
 
 def execute(command, print_output=False, print_error=True, param_shell=False):
@@ -85,6 +96,7 @@ def create_dict(tag, dict_source_path):
             result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
         if result == 0:
             print(dict_name + ' created')
+            assert os.path.isfile(dict_name)
         else:
             raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
     else:
@@ -103,12 +115,15 @@ def zstd(tag, args, input_file, output_file):
             print("Running: '{}', input={}, output={}" .format(
                 ' '.join(cmd), input_file, output_file
             ))
-            subprocess.check_call(cmd, stdin=i, stdout=o)
+            result = subprocess.run(cmd, stdin=i, stdout=o, stderr=subprocess.PIPE)
+            print("Stderr: {}".format(result.stderr.decode("ascii")))
+            result.check_returncode()
 
 
 def dict_compress_sample(tag, sample):
     dict_name = 'dict.' + tag
-    zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
+    verbose = ['-v', '-v', '-v']
+    zstd(tag, ['-D', dict_name, '-1'] + verbose, sample, sample + '_01_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
@@ -246,8 +261,12 @@ if __name__ == '__main__':
     # copy *.c and *.h to a temporary directory ("dict_source")
     if not os.path.isdir(dict_source_path):
         os.mkdir(dict_source_path)
-        print('cp ' + dict_files + ' ' + dict_source_path)
-        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+        for dict_glob in dict_globs:
+            files = glob.glob(dict_glob, root_dir=base_dir)
+            for file in files:
+                file = os.path.join(base_dir, file)
+                print("copying " + file + " to " + dict_source_path)
+                shutil.copy(file, dict_source_path)
 
     print('-----------------------------------------------')
     print('Compress test.dat by all released zstd')

From 0382076af716e78f86a764475eece59a8eeb6272 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Jan 2023 14:50:31 -0800
Subject: [PATCH 124/197] Re-Use `stat_t` in `FIO_compressFilename_srcFile()`

---
 programs/fileio.c                             | 40 +++++++++++--------
 programs/util.c                               | 36 +++++++++++++++--
 programs/util.h                               |  1 +
 .../compress-file-to-file.sh.stderr.exact     |  6 +--
 .../compress-file-to-stdout.sh.stderr.exact   |  6 +--
 .../compress-stdin-to-file.sh.stderr.exact    |  4 --
 .../compress-stdin-to-stdout.sh.stderr.exact  |  4 --
 .../decompress-file-to-file.sh.stderr.exact   |  2 +
 .../decompress-file-to-stdout.sh.stderr.exact |  2 +
 9 files changed, 67 insertions(+), 34 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 2067cfcda..0e1345596 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -662,23 +662,23 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
  * @return : loaded size
  *  if fileName==NULL, returns 0 and a NULL pointer
  */
-static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs)
+static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
 {
     FILE* fileHandle;
     U64 fileSize;
-    stat_t statbuf;
 
     assert(bufferPtr != NULL);
+    assert(dictFileStat != NULL);
     *bufferPtr = NULL;
     if (fileName == NULL) return 0;
 
     DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
 
-    if (!UTIL_stat(fileName, &statbuf)) {
+    if (!UTIL_stat(fileName, dictFileStat)) {
         EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
     }
 
-    if (!UTIL_isRegularFileStat(&statbuf)) {
+    if (!UTIL_isRegularFileStat(dictFileStat)) {
         EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
     }
 
@@ -688,7 +688,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
         EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
     }
 
-    fileSize = UTIL_getFileSizeStat(&statbuf);
+    fileSize = UTIL_getFileSizeStat(dictFileStat);
     {
         size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
         if (fileSize >  dictSizeMax) {
@@ -869,6 +869,7 @@ typedef struct {
     void* dictBuffer;
     size_t dictBufferSize;
     const char* dictFileName;
+    stat_t dictFileStat;
     ZSTD_CStream* cctx;
     WritePoolCtx_t *writeCtx;
     ReadPoolCtx_t *readCtx;
@@ -927,7 +928,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
         unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
         FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
     }
-    ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs);   /* works with dictFileName==NULL */
+    ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat);   /* works with dictFileName==NULL */
 
     ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
     ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
@@ -1712,16 +1713,22 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
     stat_t srcFileStat;
     DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
 
-    /* ensure src is not a directory */
-    if (UTIL_isDirectory(srcFileName)) {
-        DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
-        return 1;
-    }
+    if (strcmp(srcFileName, stdinmark)) {
+        if (UTIL_stat(srcFileName, &srcFileStat)) {
+            /* failure to stat at all is handled during opening */
 
-    /* ensure src is not the same as dict (if present) */
-    if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) {
-        DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
-        return 1;
+            /* ensure src is not a directory */
+            if (UTIL_isDirectoryStat(&srcFileStat)) {
+                DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
+                return 1;
+            }
+
+            /* ensure src is not the same as dict (if present) */
+            if (ress.dictFileName != NULL && UTIL_isSameFileStat(srcFileName, ress.dictFileName, &srcFileStat, &ress.dictFileStat)) {
+                DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
+                return 1;
+            }
+        }
     }
 
     /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
@@ -1998,7 +2005,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
 
     /* dictionary */
     {   void* dictBuffer;
-        size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs);
+        stat_t statbuf;
+        size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs, &statbuf);
         CHECK( ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
         CHECK( ZSTD_DCtx_loadDictionary(ress.dctx, dictBuffer, dictBufferSize) );
         free(dictBuffer);
diff --git a/programs/util.c b/programs/util.c
index 21d7d3b46..7402b75d2 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -272,11 +272,15 @@ int UTIL_isDirectory(const char* infilename)
 
 int UTIL_isDirectoryStat(const stat_t* statbuf)
 {
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isDirectoryStat()");
 #if defined(_MSC_VER)
-    return (statbuf->st_mode & _S_IFDIR) != 0;
+    ret = (statbuf->st_mode & _S_IFDIR) != 0;
 #else
-    return S_ISDIR(statbuf->st_mode) != 0;
+    ret = S_ISDIR(statbuf->st_mode) != 0;
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_compareStr(const void *p1, const void *p2) {
@@ -299,8 +303,32 @@ int UTIL_isSameFile(const char* fName1, const char* fName2)
         stat_t file2Stat;
         ret =  UTIL_stat(fName1, &file1Stat)
             && UTIL_stat(fName2, &file2Stat)
-            && (file1Stat.st_dev == file2Stat.st_dev)
-            && (file1Stat.st_ino == file2Stat.st_ino);
+            && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat);
+    }
+#endif
+    UTIL_TRACE_RET(ret);
+    return ret;
+}
+
+int UTIL_isSameFileStat(
+        const char* fName1, const char* fName2,
+        const stat_t* file1Stat, const stat_t* file2Stat)
+{
+    int ret;
+    assert(fName1 != NULL); assert(fName2 != NULL);
+    UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2);
+#if defined(_MSC_VER) || defined(_WIN32)
+    /* note : Visual does not support file identification by inode.
+     *        inode does not work on Windows, even with a posix layer, like msys2.
+     *        The following work-around is limited to detecting exact name repetition only,
+     *        aka `filename` is considered different from `subdir/../filename` */
+    (void)file1Stat;
+    (void)file2Stat;
+    ret = !strcmp(fName1, fName2);
+#else
+    {
+        ret =  (file1Stat->st_dev == file2Stat->st_dev)
+            && (file1Stat->st_ino == file2Stat->st_ino);
     }
 #endif
     UTIL_TRACE_RET(ret);
diff --git a/programs/util.h b/programs/util.h
index 88ee1bad1..4ec54137d 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -171,6 +171,7 @@ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions);
 int UTIL_isRegularFile(const char* infilename);
 int UTIL_isDirectory(const char* infilename);
 int UTIL_isSameFile(const char* file1, const char* file2);
+int UTIL_isSameFileStat(const char* file1, const char* file2, const stat_t* file1Stat, const stat_t* file2Stat);
 int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
 int UTIL_isLink(const char* infilename);
 int UTIL_isFIFO(const char* infilename);
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
index 7297894c7..873233633 100644
--- a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -6,9 +6,9 @@ Trace:FileStat: > UTIL_getFileSize(file)
 Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
 Trace:FileStat: < 65537
-Trace:FileStat: > UTIL_isDirectory(file)
-Trace:FileStat:  > UTIL_stat(file)
-Trace:FileStat:  < 1
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_stat(file)
 Trace:FileStat: < 1
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
index e01922f84..e86c4eae3 100644
--- a/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
@@ -6,9 +6,9 @@ Trace:FileStat: > UTIL_getFileSize(file)
 Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
 Trace:FileStat: < 65537
-Trace:FileStat: > UTIL_isDirectory(file)
-Trace:FileStat:  > UTIL_stat(file)
-Trace:FileStat:  < 1
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_stat(file)
 Trace:FileStat: < 1
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
index dce9005f4..00afd97c3 100644
--- a/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
@@ -6,10 +6,6 @@ Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
 Trace:FileStat:  > UTIL_stat(/*stdin*\)
 Trace:FileStat:  < 0
 Trace:FileStat: < -1
-Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
-Trace:FileStat:  > UTIL_stat(/*stdin*\)
-Trace:FileStat:  < 0
-Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
 Trace:FileStat:  > UTIL_stat(/*stdin*\)
 Trace:FileStat:  < 0
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
index d47bdcf09..2e44511d8 100644
--- a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
@@ -6,10 +6,6 @@ Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
 Trace:FileStat:  > UTIL_stat(/*stdin*\)
 Trace:FileStat:  < 0
 Trace:FileStat: < -1
-Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
-Trace:FileStat:  > UTIL_stat(/*stdin*\)
-Trace:FileStat:  < 0
-Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
 Trace:FileStat:  > UTIL_stat(/*stdout*\)
 Trace:FileStat:  < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
index 2af60eb43..53c3333eb 100644
--- a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -7,6 +7,8 @@ Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isDirectory(file.zst)
 Trace:FileStat:  > UTIL_stat(file.zst)
 Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_isDirectoryStat()
+Trace:FileStat:  < 0
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_stat(file.zst)
 Trace:FileStat: < 1
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
index 87fc5e97c..bbf66506b 100644
--- a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
@@ -5,6 +5,8 @@ Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isDirectory(file.zst)
 Trace:FileStat:  > UTIL_stat(file.zst)
 Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_isDirectoryStat()
+Trace:FileStat:  < 0
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_stat(file.zst)
 Trace:FileStat: < 1

From 1e3eba65a647dbd35739f82b9afc2021af7542f3 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Jan 2023 15:08:15 -0800
Subject: [PATCH 125/197] Copy Permissions from Source File

---
 programs/fileio.c                             | 30 ++++++++++---------
 .../compress-file-to-file.sh.stderr.exact     | 10 ++++++-
 .../decompress-file-to-file.sh.stderr.exact   | 10 ++++++-
 tests/playTests.sh                            | 10 -------
 4 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 0e1345596..ce3c0b055 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -114,11 +114,15 @@ char const* FIO_lzmaVersion(void)
 #define FNSPACE 30
 
 /* Default file permissions 0666 (modulated by umask) */
+/* Temporary restricted file permissions are used when we're going to
+ * chmod/chown at the end of the operation. */
 #if !defined(_WIN32)
 /* These macros aren't defined on windows. */
 #define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)
+#define TEMPORARY_FILE_PERMISSIONS (S_IRUSR|S_IWUSR)
 #else
 #define DEFAULT_FILE_PERMISSIONS (0666)
+#define TEMPORARY_FILE_PERMISSIONS (0600)
 #endif
 
 /*-************************************
@@ -1632,22 +1636,22 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
 {
     int closeDstFile = 0;
     int result;
-    int transferMTime = 0;
+    int transferStat = 0;
     FILE *dstFile;
 
     assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
     if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
-        int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
+        int dstFileInitialPermissions = DEFAULT_FILE_PERMISSIONS;
         if ( strcmp (srcFileName, stdinmark)
           && strcmp (dstFileName, stdoutmark)
           && UTIL_isRegularFileStat(srcFileStat) ) {
-            dstFilePermissions = srcFileStat->st_mode;
-            transferMTime = 1;
+            transferStat = 1;
+            dstFileInitialPermissions = TEMPORARY_FILE_PERMISSIONS;
         }
 
         closeDstFile = 1;
         DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
-        dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions);
+        dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
         if (dstFile==NULL) return 1;  /* could not open dstFileName */
         AIO_WritePool_setFile(ress.writeCtx, dstFile);
         /* Must only be added after FIO_openDstFile() succeeds.
@@ -1667,8 +1671,8 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
             DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
             result=1;
         }
-        if (transferMTime) {
-            UTIL_utime(dstFileName, srcFileStat);
+        if (transferStat) {
+            UTIL_setFileStat(dstFileName, srcFileStat);
         }
         if ( (result != 0)  /* operation failure */
           && strcmp(dstFileName, stdoutmark)  /* special case : don't remove() stdout */
@@ -2479,9 +2483,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
 {
     int result;
     int releaseDstFile = 0;
-    int transferMTime = 0;
-
-    (void)srcFileStat;
+    int transferStat = 0;
 
     if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
         FILE *dstFile;
@@ -2489,8 +2491,8 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
         if ( strcmp(srcFileName, stdinmark)   /* special case : don't transfer permissions from stdin */
           && strcmp(dstFileName, stdoutmark)
           && UTIL_isRegularFileStat(srcFileStat) ) {
-            dstFilePermissions = srcFileStat->st_mode;
-            transferMTime = 1;
+            transferStat = 1;
+            dstFilePermissions = TEMPORARY_FILE_PERMISSIONS;
         }
 
         releaseDstFile = 1;
@@ -2515,8 +2517,8 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
             result = 1;
         }
 
-        if (transferMTime) {
-            UTIL_utime(dstFileName, srcFileStat);
+        if (transferStat) {
+            UTIL_setFileStat(dstFileName, srcFileStat);
         }
 
         if ( (result != 0)  /* operation failure */
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
index 873233633..8944e4cb4 100644
--- a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -30,5 +30,13 @@ Trace:FileStat: > UTIL_getFileSize(file)
 Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
 Trace:FileStat: < 65537
-Trace:FileStat: > UTIL_utime(file.zst)
+Trace:FileStat: > UTIL_setFileStat(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_utime(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat:  > UTIL_chmod(file.zst, 420)
+Trace:FileStat:   > chmod
+Trace:FileStat:   < 0
+Trace:FileStat:  < 0
 Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
index 53c3333eb..3b9d69922 100644
--- a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -26,5 +26,13 @@ Trace:FileStat: > UTIL_isRegularFile(file)
 Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
 Trace:FileStat: < 1
-Trace:FileStat: > UTIL_utime(file)
+Trace:FileStat: > UTIL_setFileStat(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_utime(file)
+Trace:FileStat:  < 0
+Trace:FileStat:  > UTIL_chmod(file, 420)
+Trace:FileStat:   > chmod
+Trace:FileStat:   < 0
+Trace:FileStat:  < 0
 Trace:FileStat: < 0
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 7c111e7a3..06e6ba951 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -562,19 +562,9 @@ if [ "$isWindows" = false ] ; then
     zstd -f -d tmp1.zst -o tmp1.out
     assertFilePermissions tmp1.out 400
 
-    rm -f tmp1.zst tmp1.out
-
     umask 0666
     chmod 0666 tmp1 tmp2
 
-    println "test : respect umask when copying permissions in file -> file compression "
-    zstd -f tmp1 -o tmp1.zst
-    assertFilePermissions tmp1.zst 0
-    println "test : respect umask when copying permissions in file -> file decompression "
-    chmod 0666 tmp1.zst
-    zstd -f -d tmp1.zst -o tmp1.out
-    assertFilePermissions tmp1.out 0
-
     rm -f tmp1.zst tmp1.out
 
     println "test : respect umask when compressing from stdin input "

From 0d2d46022336b267e87d6bfb078ccc34d0ca8aad Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Jan 2023 16:37:30 -0800
Subject: [PATCH 126/197] Mimic gzip chown(gid), chmod(), chown(uid) Behavior

Avoids a race condition in which we unintentionally open up permissions to
the wrong group.
---
 programs/util.c                                  | 16 ++++++++++++++--
 .../cli-tests/file-stat/compress-file-to-file.sh |  1 +
 .../compress-file-to-file.sh.stderr.exact        |  2 +-
 .../file-stat/decompress-file-to-file.sh         |  1 +
 .../decompress-file-to-file.sh.stderr.exact      |  2 +-
 5 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 7402b75d2..d0411e68d 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -249,11 +249,23 @@ int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
     /* set access and modification times */
     res += UTIL_utime(filename, statbuf);
 
+    /* Mimic gzip's behavior:
+     *
+     * "Change the group first, then the permissions, then the owner.
+     * That way, the permissions will be correct on systems that allow
+     * users to give away files, without introducing a security hole.
+     * Security depends on permissions not containing the setuid or
+     * setgid bits." */
+
 #if !defined(_WIN32)
-    res += chown(filename, statbuf->st_uid, statbuf->st_gid);  /* Copy ownership */
+    res += chown(filename, -1, statbuf->st_gid);  /* Apply group ownership */
 #endif
 
-    res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 07777);  /* Copy file permissions */
+    res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 0777);  /* Copy file permissions */
+
+#if !defined(_WIN32)
+    res += chown(filename, statbuf->st_uid, -1);  /* Apply user ownership */
+#endif
 
     errno = 0;
     UTIL_TRACE_RET(-res);
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh b/tests/cli-tests/file-stat/compress-file-to-file.sh
index 949f34ff1..c5f590031 100755
--- a/tests/cli-tests/file-stat/compress-file-to-file.sh
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh
@@ -3,6 +3,7 @@
 set -e
 
 datagen > file
+chmod 642 file
 
 zstd file -q --trace-file-stat -o file.zst
 zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
index 8944e4cb4..49a8152f3 100644
--- a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -35,7 +35,7 @@ Trace:FileStat:  > UTIL_stat(file.zst)
 Trace:FileStat:  < 1
 Trace:FileStat:  > UTIL_utime(file.zst)
 Trace:FileStat:  < 0
-Trace:FileStat:  > UTIL_chmod(file.zst, 420)
+Trace:FileStat:  > UTIL_chmod(file.zst, 418)
 Trace:FileStat:   > chmod
 Trace:FileStat:   < 0
 Trace:FileStat:  < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh b/tests/cli-tests/file-stat/decompress-file-to-file.sh
index 3e08c247b..9e68f8f33 100755
--- a/tests/cli-tests/file-stat/decompress-file-to-file.sh
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh
@@ -3,5 +3,6 @@
 set -e
 
 datagen | zstd -q > file.zst
+chmod 642 file.zst
 
 zstd -dq --trace-file-stat file.zst
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
index 3b9d69922..65547537a 100644
--- a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -31,7 +31,7 @@ Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
 Trace:FileStat:  > UTIL_utime(file)
 Trace:FileStat:  < 0
-Trace:FileStat:  > UTIL_chmod(file, 420)
+Trace:FileStat:  > UTIL_chmod(file, 418)
 Trace:FileStat:   > chmod
 Trace:FileStat:   < 0
 Trace:FileStat:  < 0

From 7a8c8f3fe7477b3c12b03999f38f1034b278ff93 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Jan 2023 11:15:53 -0800
Subject: [PATCH 127/197] Easy: Print Mode as Octal in `chmod()` Trace

---
 programs/util.c                                                 | 2 +-
 tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact | 2 +-
 .../cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index d0411e68d..bfa2abed0 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -185,7 +185,7 @@ int UTIL_isRegularFileStat(const stat_t* statbuf)
 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
 {
     stat_t localStatBuf;
-    UTIL_TRACE_CALL("UTIL_chmod(%s, %u)", filename, (unsigned)permissions);
+    UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
     if (statbuf == NULL) {
         if (!UTIL_stat(filename, &localStatBuf)) {
             UTIL_TRACE_RET(0);
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
index 49a8152f3..a1ad09ef7 100644
--- a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -35,7 +35,7 @@ Trace:FileStat:  > UTIL_stat(file.zst)
 Trace:FileStat:  < 1
 Trace:FileStat:  > UTIL_utime(file.zst)
 Trace:FileStat:  < 0
-Trace:FileStat:  > UTIL_chmod(file.zst, 418)
+Trace:FileStat:  > UTIL_chmod(file.zst, 0642)
 Trace:FileStat:   > chmod
 Trace:FileStat:   < 0
 Trace:FileStat:  < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
index 65547537a..d264c63e4 100644
--- a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -31,7 +31,7 @@ Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
 Trace:FileStat:  > UTIL_utime(file)
 Trace:FileStat:  < 0
-Trace:FileStat:  > UTIL_chmod(file, 418)
+Trace:FileStat:  > UTIL_chmod(file, 0642)
 Trace:FileStat:   > chmod
 Trace:FileStat:   < 0
 Trace:FileStat:  < 0

From bce0382c828b502e5d9db6e58f43c25abf16ea02 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Thu, 19 Jan 2023 10:41:24 -0500
Subject: [PATCH 128/197] Bugfixes for the External Matchfinder API (#3433)

* external matchfinder bugfixes + tests

* small doc fix
---
 lib/compress/zstd_compress.c | 33 ++++++++++++++--------
 lib/zstd.h                   | 21 ++++++++++++--
 tests/zstreamtest.c          | 55 +++++++++++++++++++++++++++++++-----
 3 files changed, 88 insertions(+), 21 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0fa2e6e2d..3a48e7dcd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1599,7 +1599,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 
     size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
     size_t const externalSeqSpace = useExternalMatchFinder
-        ? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+        ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
         : 0;
 
     size_t const neededSpace =
@@ -3158,7 +3158,6 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
                                                                                     zc->appliedParams.useRowMatchFinder,
                                                                                     dictMode);
-            assert(zc->externalMatchCtx.mFinder == NULL);
             ms->ldmSeqStore = NULL;
             lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
         }
@@ -5945,6 +5944,13 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
 
 #ifdef ZSTD_MULTITHREAD
+    /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
+    RETURN_ERROR_IF(
+        params.useExternalMatchFinder == 1 && params.nbWorkers >= 1,
+        parameter_combination_unsupported,
+        "External matchfinder isn't supported with nbWorkers >= 1"
+    );
+
     if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
         params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
     }
@@ -6774,14 +6780,19 @@ void ZSTD_registerExternalMatchFinder(
     ZSTD_CCtx* zc, void* mState,
     ZSTD_externalMatchFinder_F* mFinder
 ) {
-    ZSTD_externalMatchCtx emctx = {
-        mState,
-        mFinder,
+    if (mFinder != NULL) {
+        ZSTD_externalMatchCtx emctx = {
+            mState,
+            mFinder,
 
-        /* seqBuffer is allocated later (from the cwskp) */
-        NULL, /* seqBuffer */
-        0 /* seqBufferCapacity */
-    };
-    zc->externalMatchCtx = emctx;
-    zc->requestedParams.useExternalMatchFinder = 1;
+            /* seqBuffer is allocated later (from the cwskp) */
+            NULL, /* seqBuffer */
+            0 /* seqBufferCapacity */
+        };
+        zc->externalMatchCtx = emctx;
+        zc->requestedParams.useExternalMatchFinder = 1;
+    } else {
+        ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+        zc->requestedParams.useExternalMatchFinder = 0;
+    }
 }
diff --git a/lib/zstd.h b/lib/zstd.h
index 38dfb3a5d..a2e023452 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2840,8 +2840,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* bloc
  * externalMatchState.
  *
  * *** LIMITATIONS ***
- * External matchfinders are compatible with all zstd compression APIs. There are
- * only two limitations.
+ * External matchfinders are compatible with all zstd compression APIs which respect
+ * advanced parameters. However, there are three limitations:
  *
  * First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
  * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
@@ -2863,7 +2863,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* bloc
  *     APIs, work with the external matchfinder, but the external matchfinder won't
  *     receive any history from the previous block. Each block is an independent chunk.
  *
- * Long-term, we plan to overcome both limitations. There is no technical blocker to
+ * Third, multi-threading within a single compression is not supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external matchfinder is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
  * overcoming them. It is purely a question of engineering effort.
  */
 
@@ -2886,6 +2890,17 @@ typedef size_t ZSTD_externalMatchFinder_F (
  * compressions. It will remain set until the user explicitly resets compression
  * parameters.
  *
+ * External matchfinder registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on
+ * compression APIs which respect advanced parameters, such as compress2() and
+ * compressStream(). Older compression APIs such as compressCCtx(), which predate
+ * the introduction of "advanced parameters", will ignore any external matchfinder
+ * setting.
+ *
+ * The external matchfinder can be "cleared" by registering a NULL external
+ * matchfinder function pointer. This removes all limitations described above in
+ * the "LIMITATIONS" section of the API docs.
+ *
  * The user is strongly encouraged to read the full API documentation (above)
  * before calling this function. */
 ZSTDLIB_STATIC_API void
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 2febfff91..4a621692d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1846,15 +1846,11 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 
         CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
 
-        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
 
         /* Reference external matchfinder outside the test loop to
          * check that the reference is preserved across compressions */
-        ZSTD_registerExternalMatchFinder(
-            zc,
-            &externalMatchState,
-            zstreamExternalMatchFinder
-        );
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
 
         for (enableFallback = 0; enableFallback < 1; enableFallback++) {
             size_t testCaseId;
@@ -1916,11 +1912,56 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
         }
 
         /* Test that reset clears the external matchfinder */
-        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
         externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
         CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
         CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
 
+        /* Test that registering mFinder == NULL clears the external matchfinder */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
+        CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+        /* Test that external matchfinder doesn't interact with older APIs */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
+
+        /* Test that compression returns the correct error with LDM */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+
+#ifdef ZSTD_MULTITHREAD
+        /* Test that compression returns the correct error with nbWorkers > 0 */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+#endif
+
         free(dstBuf);
         free(checkBuf);
     }

From 3f9f568aa6401c1902138012760d4c53780f7bb4 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Thu, 19 Jan 2023 13:18:01 -0800
Subject: [PATCH 129/197] Fuzz the external matchfinder API

---
 tests/fuzz/Makefile       | 30 ++++++++++++++++++++++--------
 tests/fuzz/zstd_helpers.c | 18 ++++++++++++++++++
 2 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 35a19dd89..4b5102ef1 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -34,9 +34,14 @@ ZSTDDIR = ../../lib
 PRGDIR = ../../programs
 CONTRIBDIR = ../../contrib
 
+# TODO(embg) make it possible to plug in an arbitrary matchfinder as a .o file
+MATCHFINDER_DIR = $(CONTRIBDIR)/externalMatchfinder
+MATCHFINDER_SRC = $(MATCHFINDER_DIR)/matchfinder.c
+
 FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 	-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
-	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
+	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(MATCHFINDER_DIR) \
+	-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
 FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
 	-Wstrict-prototypes -Wundef \
@@ -69,7 +74,8 @@ FUZZ_SRC       := \
 	$(ZSTDCOMMON_SRC) \
 	$(ZSTDCOMP_SRC) \
 	$(ZSTDDICT_SRC) \
-	$(ZSTDLEGACY_SRC)
+	$(ZSTDLEGACY_SRC) \
+	$(MATCHFINDER_SRC)
 FUZZ_SRC := $(sort $(wildcard $(FUZZ_SRC)))
 
 FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC))
@@ -78,9 +84,10 @@ FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2))
 FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3))
 FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4))
 FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5))
-FUZZ_D_OBJ7 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ6))
-FUZZ_D_OBJ8 := $(FUZZ_D_OBJ7:.c=.o)
-FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ8:.S=.o)
+FUZZ_D_OBJ7 := $(subst $(MATCHFINDER_DIR)/,d_matchfinder_,$(FUZZ_D_OBJ6))
+FUZZ_D_OBJ8 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ7))
+FUZZ_D_OBJ9 := $(FUZZ_D_OBJ8:.c=.o)
+FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ9:.S=.o)
 
 FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC))
 FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1))
@@ -88,9 +95,10 @@ FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2
 FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3))
 FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4))
 FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5))
-FUZZ_RT_OBJ7 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ6))
-FUZZ_RT_OBJ8 := $(FUZZ_RT_OBJ7:.c=.o)
-FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ8:.S=.o)
+FUZZ_RT_OBJ7 := $(subst $(MATCHFINDER_DIR)/,rt_matchfinder_,$(FUZZ_RT_OBJ6))
+FUZZ_RT_OBJ8 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ7))
+FUZZ_RT_OBJ9 := $(FUZZ_RT_OBJ8:.c=.o)
+FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ9:.S=.o)
 
 .PHONY: default all clean cleanall
 
@@ -143,6 +151,9 @@ rt_prg_%.o: $(PRGDIR)/%.c
 rt_fuzz_%.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
 
+rt_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
 d_lib_common_%.o: $(ZSTDDIR)/common/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 
@@ -167,6 +178,9 @@ d_prg_%.o: $(PRGDIR)/%.c
 d_fuzz_%.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 
+d_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
 simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o
 	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
 
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 2797ed726..c9fcdd5bf 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -17,6 +17,7 @@
 #include "fuzz_helpers.h"
 #include "zstd.h"
 #include "zdict.h"
+#include "matchfinder.h"
 
 const int kMinClevel = -3;
 const int kMaxClevel = 19;
@@ -70,6 +71,17 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer)
     return params;
 }
 
+static void setExternalMatchFinderParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
+    ZSTD_registerExternalMatchFinder(
+        cctx,
+        NULL,
+        simpleExternalMatchFinder
+    );
+    setRand(cctx, ZSTD_c_enableMatchFinderFallback, 0, 1, producer);
+    FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0));
+    FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
+}
+
 void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer)
 {
     ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer);
@@ -122,6 +134,12 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer);
     }
+
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 0) {
+        setExternalMatchFinderParams(cctx, producer);
+    } else {
+        ZSTD_registerExternalMatchFinder(cctx, NULL, NULL);
+    }
 }
 
 FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer)

From ea684c335ab54bafa967a290a92d66b3d8f80648 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 18 Jan 2023 15:38:36 -0800
Subject: [PATCH 130/197] added c89 build test to CI

---
 .github/workflows/dev-short-tests.yml |  9 +++++++
 Makefile                              |  2 +-
 lib/common/pool.c                     |  4 ++-
 lib/compress/zstd_compress.c          | 36 ++++++++++++++++-----------
 lib/compress/zstd_compress_internal.h |  2 +-
 lib/decompress/zstd_decompress.c      | 20 +++++++++------
 lib/dictBuilder/cover.c               | 25 +++++++++++--------
 programs/fileio.c                     | 32 ++++++++++++++++++------
 8 files changed, 87 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 5a29789f9..f5c555231 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -47,6 +47,15 @@ jobs:
         APT_PACKAGES="gcc-multilib" make apt-install
         CFLAGS="-mx32 -O1 -fstack-protector" make check V=1
 
+  build-c89:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: ensure zstd can be build with c89/c90 compilers (+ long long support + variadic macros)
+      run: |
+        make c89build V=1
+
+
   gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
diff --git a/Makefile b/Makefile
index d87fc76eb..75a47a308 100644
--- a/Makefile
+++ b/Makefile
@@ -404,7 +404,7 @@ cmakebuild:
 
 c89build: clean
 	$(CC) -v
-	CFLAGS="-std=c89 -Werror -O0" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd
 
 gnu90build: clean
 	$(CC) -v
diff --git a/lib/common/pool.c b/lib/common/pool.c
index 3466b85b8..f3d9d0854 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -271,7 +271,9 @@ static int isQueueFull(POOL_ctx const* ctx) {
 static void
 POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
 {
-    POOL_job const job = {function, opaque};
+    POOL_job job;
+    job.function = function;
+    job.opaque = opaque;
     assert(ctx != NULL);
     if (ctx->shutdown) return;
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3a48e7dcd..7710151de 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3939,7 +3939,9 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end
  */
 static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
 {
-    seqStoreSplits splits = {partitions, 0};
+    seqStoreSplits splits;
+    splits.splitLocations = partitions;
+    splits.idx = 0;
     if (nbSeq <= 4) {
         DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq);
         /* Refuse to try and split anything with less than 4 sequences */
@@ -6111,13 +6113,20 @@ size_t ZSTD_compressStream2_simpleArgs (
                       const void* src, size_t srcSize, size_t* srcPos,
                             ZSTD_EndDirective endOp)
 {
-    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
-    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
+    ZSTD_outBuffer output;
+    ZSTD_inBuffer  input;
+    output.dst = dst;
+    output.size = dstCapacity;
+    output.pos = *dstPos;
+    input.src = src;
+    input.size = srcSize;
+    input.pos = *srcPos;
     /* ZSTD_compressStream2() will check validity of dstPos and srcPos */
-    size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
-    *dstPos = output.pos;
-    *srcPos = input.pos;
-    return cErr;
+    {   size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
+        *dstPos = output.pos;
+        *srcPos = input.pos;
+        return cErr;
+    }
 }
 
 size_t ZSTD_compress2(ZSTD_CCtx* cctx,
@@ -6781,14 +6790,11 @@ void ZSTD_registerExternalMatchFinder(
     ZSTD_externalMatchFinder_F* mFinder
 ) {
     if (mFinder != NULL) {
-        ZSTD_externalMatchCtx emctx = {
-            mState,
-            mFinder,
-
-            /* seqBuffer is allocated later (from the cwskp) */
-            NULL, /* seqBuffer */
-            0 /* seqBufferCapacity */
-        };
+        ZSTD_externalMatchCtx emctx;
+        emctx.mState = mState;
+        emctx.mFinder = mFinder;
+        emctx.seqBuffer = NULL;
+        emctx.seqBufferCapacity = 0;
         zc->externalMatchCtx = emctx;
         zc->requestedParams.useExternalMatchFinder = 1;
     } else {
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 3b888acfa..c93e85fb5 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -494,7 +494,7 @@ typedef enum {
                                  * In this mode we take both the source size and the dictionary size
                                  * into account when selecting and adjusting the parameters.
                                  */
-    ZSTD_cpm_unknown = 3,       /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
+    ZSTD_cpm_unknown = 3        /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
                                  * We don't know what these parameters are for. We default to the legacy
                                  * behavior of taking both the source size and the dict size into account
                                  * when selecting and adjusting parameters.
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 770dd8b19..10d29f298 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2324,11 +2324,17 @@ size_t ZSTD_decompressStream_simpleArgs (
                             void* dst, size_t dstCapacity, size_t* dstPos,
                       const void* src, size_t srcSize, size_t* srcPos)
 {
-    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
-    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
-    /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
-    size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
-    *dstPos = output.pos;
-    *srcPos = input.pos;
-    return cErr;
+    ZSTD_outBuffer output;
+    ZSTD_inBuffer  input;
+    output.dst = dst;
+    output.size = dstCapacity;
+    output.pos = *dstPos;
+    input.src = src;
+    input.size = srcSize;
+    input.pos = *srcPos;
+    {   size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
+        *dstPos = output.pos;
+        *srcPos = input.pos;
+        return cErr;
+    }
 }
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 18c8ec538..9e5e7d5b5 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -951,9 +951,17 @@ void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
   }
 }
 
+static COVER_dictSelection_t setDictSelection(BYTE* buf, size_t s, size_t csz)
+{
+    COVER_dictSelection_t ds;
+    ds.dictContent = buf;
+    ds.dictSize = s;
+    ds.totalCompressedSize = csz;
+    return ds;
+}
+
 COVER_dictSelection_t COVER_dictSelectionError(size_t error) {
-    COVER_dictSelection_t selection = { NULL, 0, error };
-    return selection;
+    return setDictSelection(NULL, 0, error);
 }
 
 unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) {
@@ -1006,9 +1014,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
   }
 
   if (params.shrinkDict == 0) {
-    COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
     free(candidateDictBuffer);
-    return selection;
+    return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize);
   }
 
   largestDict = dictContentSize;
@@ -1041,19 +1048,15 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
     }
 
     if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) {
-      COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize };
       free(largestDictbuffer);
-      return selection;
+      return setDictSelection( candidateDictBuffer, dictContentSize, totalCompressedSize );
     }
     dictContentSize *= 2;
   }
   dictContentSize = largestDict;
   totalCompressedSize = largestCompressed;
-  {
-    COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
-    free(candidateDictBuffer);
-    return selection;
-  }
+  free(candidateDictBuffer);
+  return setDictSelection( largestDictbuffer, dictContentSize, totalCompressedSize );
 }
 
 /**
diff --git a/programs/fileio.c b/programs/fileio.c
index bc0b1ac39..23472c9b1 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -860,6 +860,24 @@ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t*
     return error;
 }
 
+static ZSTD_inBuffer setInBuffer(const void* buf, size_t s, size_t pos)
+{
+    ZSTD_inBuffer i;
+    i.src = buf;
+    i.size = s;
+    i.pos = pos;
+    return i;
+}
+
+static ZSTD_outBuffer setOutBuffer(void* buf, size_t s, size_t pos)
+{
+    ZSTD_outBuffer o;
+    o.dst = buf;
+    o.size = s;
+    o.pos = pos;
+    return o;
+}
+
 #ifndef ZSTD_NOCOMPRESS
 
 /* **********************************************************************
@@ -1278,7 +1296,6 @@ FIO_compressLz4Frame(cRess_t* ress,
 }
 #endif
 
-
 static unsigned long long
 FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                       FIO_prefs_t* const prefs,
@@ -1342,7 +1359,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
         size_t stillToFlush;
         /* Fill input Buffer */
         size_t const inSize = AIO_ReadPool_fillBuffer(ress.readCtx, ZSTD_CStreamInSize());
-        ZSTD_inBuffer inBuff = { ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 };
+        ZSTD_inBuffer inBuff = setInBuffer( ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 );
         DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize);
         *readsize += inSize;
 
@@ -1354,7 +1371,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
             || (directive == ZSTD_e_end && stillToFlush != 0) ) {
 
             size_t const oldIPos = inBuff.pos;
-            ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
+            ZSTD_outBuffer outBuff = setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
             size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
             CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
             AIO_ReadPool_consumeBytes(ress.readCtx, inBuff.pos - oldIPos);
@@ -2094,8 +2111,8 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
 
     /* Main decompression Loop */
     while (1) {
-        ZSTD_inBuffer  inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 };
-        ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
+        ZSTD_inBuffer  inBuff = setInBuffer( ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 );
+        ZSTD_outBuffer outBuff= setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
         size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
         UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
         if (ZSTD_isError(readSizeHint)) {
@@ -2758,7 +2775,8 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
 
     if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
         DISPLAY_PROGRESS("\r%79s\r", "");
-        DISPLAY_SUMMARY("%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
+        DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n",
+            fCtx->nbFilesProcessed, (unsigned long long)fCtx->totalBytesOutput);
     }
 
     FIO_freeDResources(ress);
@@ -2787,7 +2805,7 @@ typedef enum {
   info_frame_error=1,
   info_not_zstd=2,
   info_file_error=3,
-  info_truncated_input=4,
+  info_truncated_input=4
 } InfoError;
 
 #define ERROR_IF(c,n,...) {             \

From cd272d7a2d832aec3c4332b32eeeffe29916fbfc Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 18 Jan 2023 14:12:27 -0800
Subject: [PATCH 131/197] added cygwin tests to github actions

---
 .github/workflows/dev-short-tests.yml | 28 ++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 5a29789f9..de561c716 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -445,9 +445,35 @@ jobs:
         COPY build\VS2010\bin\${{matrix.platform}}_${{matrix.configuration}}\*.exe tests\
         CD tests
         sh -e playTests.sh
-        DIR
         .\fuzzer.exe -T2m
 
+  # Following instructions at: https://github.com/marketplace/actions/install-cygwin-action
+  cygwin-tests:
+    runs-on: windows-latest
+    steps:
+    - run: git config --global core.autocrlf input
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - uses: cygwin/cygwin-install-action@f5e0f048310c425e84bc789f493a828c6dc80a25 # tag=master
+      with:
+        platform: x86_64
+        packages: >-
+          autoconf,
+          automake,
+          gcc-g++,
+          make,
+          mingw64-x86_64-gcc-g++,
+          patch,
+          perl
+    - name: cygwin tests
+      shell: C:\cygwin\bin\bash.exe --noprofile --norc -eo pipefail '{0}'
+      run: >-
+        export PATH=/usr/bin:$(cygpath ${SYSTEMROOT})/system32 &&
+        export CFLAGS="-Werror -O1" &&
+        ls &&
+        make -j allzstd &&
+        make -C tests fuzzer &&
+        ./tests/fuzzer.exe -v -T1m
+
   intel-cet-compatibility:
     runs-on: ubuntu-latest
     steps:

From f593e54ee18dcceb2c0d4529e1edfe4e6ea823eb Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Fri, 20 Jan 2023 11:41:53 -0500
Subject: [PATCH 132/197] Enable if == 1 rather than if == 0

Co-authored-by: Nick Terrell <nickrterrell@gmail.com>
---
 tests/fuzz/zstd_helpers.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index c9fcdd5bf..43a26405f 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -135,7 +135,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
       setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer);
     }
 
-    if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 0) {
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 1) {
         setExternalMatchFinderParams(cctx, producer);
     } else {
         ZSTD_registerExternalMatchFinder(cctx, NULL, NULL);

From aa385ece13b0a847380303a3918609213eea45e0 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Fri, 20 Jan 2023 10:32:35 -0800
Subject: [PATCH 133/197] fix sequence validation and bounds check in
 ZSTD_copySequencesToSeqStore()

---
 lib/compress/zstd_compress.c | 18 ++++----
 tests/zstreamtest.c          | 84 ++++++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+), 8 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3a48e7dcd..c7a573371 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -6156,8 +6156,8 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
  * @returns a ZSTD error code if sequence is not valid
  */
 static size_t
-ZSTD_validateSequence(U32 offCode, U32 matchLength,
-                      size_t posInSrc, U32 windowLog, size_t dictSize)
+ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
+                      size_t posInSrc, U32 windowLog, size_t dictSize, int useExternalMatchFinder)
 {
     U32 const windowSize = 1u << windowLog;
     /* posInSrc represents the amount of data the decoder would decode up to this point.
@@ -6168,6 +6168,8 @@ ZSTD_validateSequence(U32 offCode, U32 matchLength,
     size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
     RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), corruption_detected, "Offset too large!");
     RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small");
+    /* Validate maxNbSeq is large enough for the given matchLength and minMatch */
+    RETURN_ERROR_IF(!useExternalMatchFinder && minMatch >= 4 && matchLength < 4, corruption_detected, "Matchlength too small for the minMatch");
     return 0;
 }
 
@@ -6220,11 +6222,11 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
         DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
         if (cctx->appliedParams.validateSequences) {
             seqPos->posInSrc += litLength + matchLength;
-            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, seqPos->posInSrc,
-                                                cctx->appliedParams.cParams.windowLog, dictSize),
+            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+                                                cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
                                                 "Sequence validation failed");
         }
-        RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, memory_allocation,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
@@ -6332,12 +6334,12 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
 
         if (cctx->appliedParams.validateSequences) {
             seqPos->posInSrc += litLength + matchLength;
-            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, seqPos->posInSrc,
-                                                   cctx->appliedParams.cParams.windowLog, dictSize),
+            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+                                                   cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
                                                    "Sequence validation failed");
         }
         DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
-        RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, memory_allocation,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 4a621692d..cb5f7fea7 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2066,6 +2066,90 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    /* Test Sequence Validation */
+    DISPLAYLEVEL(3, "test%3i : Testing sequence validation: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+
+        /* Test minMatch >= 4, matchLength < 4 */
+        {
+            size_t srcSize = 11;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 4;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+
+            memset(src, 'x', srcSize);
+
+            sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
+            sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[3] = (ZSTD_Sequence) {0, 1, 0, 0};
+
+            /* Test with sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+            /* Test without sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+
+            free(sequences);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+        { /* Test case with two additional sequences */
+            size_t srcSize = 19;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 7;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+
+            memset(src, 'x', srcSize);
+
+            sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
+            sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[3] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[4] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[5] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[6] = (ZSTD_Sequence) {0, 0, 0, 0};
+
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+
+            free(sequences);
+        }
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);

From 666944fbe6bb5e6d84a343ad4df525af79720165 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 19 Jan 2023 16:27:24 -0800
Subject: [PATCH 134/197] Cap hashLog & chainLog to ensure that we only use 32
 bits of hash

* Cap shortCache chainLog to 24
* Cap row match finder hashLog so that rowLog <= 24
* Add unit tests to expose all cases. The row match finder unit tests
  are only run in 64-bit mode, because they allocate ~1GB.

Fixes #3336
---
 lib/compress/zstd_compress.c | 41 +++++++++++++++---
 lib/compress/zstd_lazy.c     |  1 -
 lib/compress/zstd_lazy.h     |  4 +-
 tests/fuzzer.c               | 84 ++++++++++++++++++++++++++++++++++++
 tests/zstreamtest.c          | 21 +++++++++
 5 files changed, 144 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3a48e7dcd..e0bcbfb16 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1412,7 +1412,8 @@ static ZSTD_compressionParameters
 ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
                             unsigned long long srcSize,
                             size_t dictSize,
-                            ZSTD_cParamMode_e mode)
+                            ZSTD_cParamMode_e mode,
+                            ZSTD_paramSwitch_e useRowMatchFinder)
 {
     const U64 minSrcSize = 513; /* (1<<9) + 1 */
     const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
@@ -1465,11 +1466,40 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
         cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* minimum wlog required for valid frame header */
 
+    /* We can't use more than 32 bits of hash in total, so that means that we require:
+     * (hashLog + 8) <= 32 && (chainLog + 8) <= 32
+     */
     if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) {
         U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS;
         if (cPar.hashLog > maxShortCacheHashLog) {
             cPar.hashLog = maxShortCacheHashLog;
         }
+        if (cPar.chainLog > maxShortCacheHashLog) {
+            cPar.chainLog = maxShortCacheHashLog;
+        }
+    }
+
+
+    /* At this point, we aren't 100% sure if we are using the row match finder.
+     * Unless it is explicitly disabled, conservatively assume that it is enabled.
+     * In this case it will only be disabled for small sources, so shrinking the
+     * hash log a little bit shouldn't result in any ratio loss.
+     */
+    if (useRowMatchFinder == ZSTD_ps_auto)
+        useRowMatchFinder = ZSTD_ps_enable;
+
+    /* We can't hash more than 32-bits in total. So that means that we require:
+     * (hashLog - rowLog + 8) <= 32
+     */
+    if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) {
+        /* Switch to 32-entry rows if searchLog is 5 (or more) */
+        U32 const rowLog = BOUNDED(4, cPar.searchLog, 6);
+        U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS;
+        U32 const maxHashLog = maxRowHashLog + rowLog;
+        assert(cPar.hashLog >= rowLog);
+        if (cPar.hashLog > maxHashLog) {
+            cPar.hashLog = maxHashLog;
+        }
     }
 
     return cPar;
@@ -1482,7 +1512,7 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
 {
     cPar = ZSTD_clampCParams(cPar);   /* resulting cPar is necessarily valid (all parameters within range) */
     if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN;
-    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown);
+    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto);
 }
 
 static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
@@ -1513,7 +1543,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
     ZSTD_overrideCParams(&cParams, &CCtxParams->cParams);
     assert(!ZSTD_checkCParams(cParams));
     /* srcSizeHint == 0 means 0 */
-    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode);
+    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder);
 }
 
 static size_t
@@ -2185,7 +2215,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         }
 
         params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize,
-                                                     cdict->dictContentSize, ZSTD_cpm_attachDict);
+                                                     cdict->dictContentSize, ZSTD_cpm_attachDict,
+                                                     params.useRowMatchFinder);
         params.cParams.windowLog = windowLog;
         params.useRowMatchFinder = cdict->useRowMatchFinder;    /* cdict overrides */
         FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
@@ -6740,7 +6771,7 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
             cp.targetLength = (unsigned)(-clampedCompressionLevel);
         }
         /* refine parameters based on srcSize & dictSize */
-        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode);
+        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto);
     }
 }
 
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 810bf011c..a24734272 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -759,7 +759,6 @@ size_t ZSTD_HcFindBestMatch(
 ***********************************/
 /* Constants for row-based hash */
 #define ZSTD_ROW_HASH_TAG_OFFSET 16     /* byte offset of hashes in the match state's tagTable from the beginning of a row */
-#define ZSTD_ROW_HASH_TAG_BITS 8        /* nb bits to use for the tag */
 #define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
 #define ZSTD_ROW_HASH_MAX_ENTRIES 64    /* absolute maximum number of entries per row, for all configurations */
 
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index c24f1c794..3bde67331 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -25,6 +25,8 @@ extern "C" {
  */
 #define ZSTD_LAZY_DDSS_BUCKET_LOG 2
 
+#define ZSTD_ROW_HASH_TAG_BITS 8        /* nb bits to use for the tag */
+
 U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
 void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
 
@@ -116,7 +118,7 @@ size_t ZSTD_compressBlock_lazy2_extDict_row(
 size_t ZSTD_compressBlock_btlazy2_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize);
-        
+
 
 #if defined (__cplusplus)
 }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 4a091c897..e02d06872 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -2832,6 +2832,90 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
+        DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
+        {
+            ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
+            ZSTD_customMem customMem = {NULL, NULL, NULL};
+            ZSTD_DCtx* dctx = ZSTD_createDCtx();
+            ZSTD_CDict* cdict;
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
+            /* Set windowLog to 25 so hash/chain logs don't get sized down */
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
+            /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
+            cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
+            CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
+            CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
+            cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
+            CHECK_Z(cSize);
+            CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
+            ZSTD_freeCDict(cdict);
+            ZSTD_freeDCtx(dctx);
+            ZSTD_freeCCtxParams(cctxParams);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
+        {
+            ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
+            ZSTD_customMem customMem = {NULL, NULL, NULL};
+            ZSTD_DCtx* dctx = ZSTD_createDCtx();
+            ZSTD_CDict* cdict;
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
+            /* Set windowLog to 25 so hash/chain logs don't get sized down */
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
+            /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
+            cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
+            CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
+            CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
+            cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
+            CHECK_Z(cSize);
+            CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
+            ZSTD_freeCDict(cdict);
+            ZSTD_freeDCtx(dctx);
+            ZSTD_freeCCtxParams(cctxParams);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
+        if (MEM_64bits()) {
+            ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
+            ZSTD_customMem customMem = {NULL, NULL, NULL};
+            ZSTD_DCtx* dctx = ZSTD_createDCtx();
+            ZSTD_CDict* cdict;
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
+            /* Force enable row based match finder, and disable dedicated dict search. */
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
+            /* Set windowLog to 29 so hash/chain logs don't get sized down */
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
+            /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
+            CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
+            cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
+            CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
+            CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
+            cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
+            CHECK_Z(cSize);
+            CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
+            ZSTD_freeCDict(cdict);
+            ZSTD_freeDCtx(dctx);
+            ZSTD_freeCCtxParams(cctxParams);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
         { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
         dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 4a621692d..b3531a19e 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1566,6 +1566,27 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
     CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
+    if (MEM_64bits()) {
+        ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
+        ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
+        /* Force enable the row based match finder */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
+        /* Set windowLog to 29 so the hashLog doesn't get sized down */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
+        /* Compress with continue first so the hashLog doesn't get sized down */
+        CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
+        CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
+        cSize = out.pos;
+        CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
     {
         int windowLog;

From 667eb6d4fd1677aa0de109673868ff47b0532e45 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 19 Jan 2023 12:21:31 -0800
Subject: [PATCH 135/197] [versions-test] Work around bug in dictionary builder
 for older versions

Older versions of zstandard have a bug in the dictionary builder, that
can cause dictionary building to fail. The process still exits 0, but
the dictionary is not created.

For reference, the bug is that it creates a dictionary that starts with
the zstd dictionary magic, in the process of writing the dictionary header,
but the header isn't fully written yet, and zstd fails compressions in
this case, because the dictionary is malformated. We fixed this later on
by trying to load the dictionary as a zstd dictionary, but if that fails
we fallback to content only (by default).

The fix is to:
1. Make the dictionary determinsitic by sorting the input files.
   Previously the bug would only sometimes occur, when the input files
   were in a particular order.
2. If dictionary creation fails, fallback to the `head` dictionary.
---
 tests/test-zstd-versions.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index 88b0578eb..d6784d615 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -85,18 +85,23 @@ def get_git_tags():
     return tags
 
 
-def create_dict(tag, dict_source_path):
+def create_dict(tag, dict_source_path, fallback_tag=None):
     dict_name = 'dict.' + tag
     if not os.path.isfile(dict_name):
         cFiles = glob.glob(dict_source_path + "/*.c")
         hFiles = glob.glob(dict_source_path + "/*.h")
+        # Ensure the dictionary builder is deterministic
+        files = sorted(cFiles + hFiles)
         if tag == 'v0.5.0':
-            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
         else:
-            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
-        if result == 0:
+            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        if result == 0 and os.path.isfile(dict_name):
             print(dict_name + ' created')
-            assert os.path.isfile(dict_name)
+        elif fallback_tag is not None:
+            fallback_dict_name = 'dict.' + fallback_tag
+            print('creating dictionary ' + dict_name + ' failed, falling back to ' + fallback_dict_name)
+            shutil.copy(fallback_dict_name, dict_name)
         else:
             raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
     else:
@@ -272,10 +277,11 @@ if __name__ == '__main__':
     print('Compress test.dat by all released zstd')
     print('-----------------------------------------------')
 
+    create_dict(head, dict_source_path)
     for tag in tags:
         print(tag)
         if tag >= 'v0.5.0':
-            create_dict(tag, dict_source_path)
+            create_dict(tag, dict_source_path, head)
             dict_compress_sample(tag, test_dat)
             remove_duplicates()
             decompress_dict(tag)

From 0cc1b0cb224041b33eaef1ca97e1711fd5181b6a Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 18 Jan 2023 15:30:30 -0800
Subject: [PATCH 136/197] Delete unused Huffman functions

Remove all Huffman functions that aren't used by zstd.
---
 contrib/linux-kernel/Makefile           |   1 -
 lib/common/entropy_common.c             |   1 -
 lib/common/huf.h                        | 169 ++----------
 lib/common/zstd_internal.h              |   1 -
 lib/compress/huf_compress.c             |  87 +------
 lib/compress/zstd_compress.c            |   1 -
 lib/compress/zstd_compress_superblock.c |   7 +-
 lib/decompress/huf_decompress.c         | 332 ++----------------------
 lib/decompress/zstd_ddict.c             |   1 -
 lib/decompress/zstd_decompress.c        |   9 +-
 lib/decompress/zstd_decompress_block.c  |   5 +-
 lib/dictBuilder/zdict.c                 |   8 +-
 tests/decodecorpus.c                    |  10 +-
 tests/fuzz/huf_decompress.c             |   2 -
 tests/fuzz/huf_round_trip.c             |   4 +-
 15 files changed, 57 insertions(+), 581 deletions(-)

diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index 61a52ce20..63dd15d95 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -32,7 +32,6 @@ libzstd:
 		-DZSTD_LEGACY_SUPPORT=0 \
 		-DZSTD_STATIC_LINKING_ONLY \
 		-DFSE_STATIC_LINKING_ONLY \
-		-DHUF_STATIC_LINKING_ONLY \
 		-DXXH_STATIC_LINKING_ONLY \
 		-D__GNUC__ \
 		-D__linux__=1 \
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index 4bf8242ff..e40921966 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -19,7 +19,6 @@
 #include "error_private.h"       /* ERR_*, ERROR */
 #define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
 #include "fse.h"
-#define HUF_STATIC_LINKING_ONLY  /* HUF_TABLELOG_ABSOLUTEMAX */
 #include "huf.h"
 #include "bits.h"                /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
 
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 9683f915a..d085d7741 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -21,102 +21,23 @@ extern "C" {
 
 /* *** Dependencies *** */
 #include "zstd_deps.h"    /* size_t */
-
-
-/* *** library symbols visibility *** */
-/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
- *        HUF symbols remain "private" (internal symbols for library only).
- *        Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
-#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
-#  define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
-#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
-#  define HUF_PUBLIC_API __declspec(dllexport)
-#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
-#  define HUF_PUBLIC_API __declspec(dllimport)  /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
-#else
-#  define HUF_PUBLIC_API
-#endif
-
-
-/* ========================== */
-/* ***  simple functions  *** */
-/* ========================== */
-
-/** HUF_compress() :
- *  Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
- * 'dst' buffer must be already allocated.
- *  Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
- * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
- * @return : size of compressed data (<= `dstCapacity`).
- *  Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
- *                   if HUF_isError(return), compression failed (more details using HUF_getErrorName())
- */
-HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
-                             const void* src, size_t srcSize);
-
-/** HUF_decompress() :
- *  Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
- *  into already allocated buffer 'dst', of minimum size 'dstSize'.
- * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
- *  Note : in contrast with FSE, HUF_decompress can regenerate
- *         RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
- *         because it knows size to regenerate (originalSize).
- * @return : size of regenerated data (== originalSize),
- *           or an error code, which can be tested using HUF_isError()
- */
-HUF_PUBLIC_API size_t HUF_decompress(void* dst,  size_t originalSize,
-                               const void* cSrc, size_t cSrcSize);
-
-
-/* ***   Tool functions *** */
-#define HUF_BLOCKSIZE_MAX (128 * 1024)                  /**< maximum input size for a single block compressed with HUF_compress */
-HUF_PUBLIC_API size_t HUF_compressBound(size_t size);   /**< maximum compressed size (worst case) */
-
-/* Error Management */
-HUF_PUBLIC_API unsigned    HUF_isError(size_t code);       /**< tells if a return value is an error code */
-HUF_PUBLIC_API const char* HUF_getErrorName(size_t code);  /**< provides error code string (useful for debugging) */
-
-
-/* ***   Advanced function   *** */
-
-/** HUF_compress2() :
- *  Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
- * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
- * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
-HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
-                               const void* src, size_t srcSize,
-                               unsigned maxSymbolValue, unsigned tableLog);
-
-/** HUF_compress4X_wksp() :
- *  Same as HUF_compress2(), but uses externally allocated @workSpace.
- * @workSpace's size, aka @wkspSize, must be >= HUF_WORKSPACE_SIZE
- * @srcSize must be >= 6
- */
-#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
-#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
-HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     unsigned maxSymbolValue, unsigned tableLog,
-                                     void* workSpace, size_t wkspSize);
-
-#endif   /* HUF_H_298734234 */
-
-/* ******************************************************************
- *  WARNING !!
- *  The following section contains advanced and experimental definitions
- *  which shall never be used in the context of a dynamic library,
- *  because they are not guaranteed to remain stable in the future.
- *  Only consider them in association with static linking.
- * *****************************************************************/
-#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
-#define HUF_H_HUF_STATIC_LINKING_ONLY
-
-/* *** Dependencies *** */
-#include "mem.h"   /* U32 */
+#include "mem.h"          /* U32 */
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
 
 
+/* ***   Tool functions *** */
+#define HUF_BLOCKSIZE_MAX (128 * 1024)   /**< maximum input size for a single block compressed with HUF_compress */
+size_t HUF_compressBound(size_t size);   /**< maximum compressed size (worst case) */
+
+/* Error Management */
+unsigned    HUF_isError(size_t code);       /**< tells if a return value is an error code */
+const char* HUF_getErrorName(size_t code);  /**< provides error code string (useful for debugging) */
+
+
+#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
+#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
+
 /* *** Constants *** */
 #define HUF_TABLELOG_MAX      12      /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
 #define HUF_TABLELOG_DEFAULT  11      /* default tableLog value when none specified */
@@ -156,20 +77,6 @@ typedef U32 HUF_DTable;
 /* ****************************************
 *  Advanced decompression functions
 ******************************************/
-size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
-#endif
-
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
-size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
-#endif
 
 
 /* ****************************************
@@ -196,10 +103,7 @@ unsigned HUF_minTableLog(unsigned symbolCardinality);
 unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
  size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode); /* table is used as scratch space for building and testing tables, not a return value */
-size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);   /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
 size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
-size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
 size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
 int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
@@ -227,7 +131,7 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
  * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
  */
-#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
+#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
 #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
 size_t HUF_buildCTable_wksp (HUF_CElt* tree,
                        const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
@@ -291,31 +195,11 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
 #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
 #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
 
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-
 
 /* ====================== */
 /* single stream variants */
 /* ====================== */
 
-size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */
-size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
 /** HUF_compress1X_repeat() :
  *  Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
@@ -330,28 +214,9 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
                        HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
                        int suspectUncompressible, HUF_depth_mode depthMode);
 
-size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
+size_t HUF_decompress1X_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
 #ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
-#endif
-
-size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
-size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
-#endif
-
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);   /**< automatic selection of sing or double symbol decoder, based on DTable */
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+size_t HUF_decompress1X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);   /**< double-symbols decoder */
 #endif
 
 /* BMI2 variants.
@@ -370,7 +235,7 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
 size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
 #endif
 
-#endif /* HUF_STATIC_LINKING_ONLY */
+#endif   /* HUF_H_298734234 */
 
 #if defined (__cplusplus)
 }
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 12e1106a1..94670a03d 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -28,7 +28,6 @@
 #include "../zstd.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
 #ifndef XXH_STATIC_LINKING_ONLY
 #  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 75200851b..e04ccd2eb 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -29,7 +29,6 @@
 #include "hist.h"
 #define FSE_STATIC_LINKING_ONLY   /* FSE_optimalTableLog_internal */
 #include "../common/fse.h"        /* header compression */
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/error_private.h"
 #include "../common/bits.h"       /* ZSTD_highbit32 */
@@ -236,6 +235,8 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
     U32 n;
     HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
 
+    HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp));
+
     /* check conditions */
     if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
     if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
@@ -265,16 +266,6 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
     return ((maxSymbolValue+1)/2) + 1;
 }
 
-/*! HUF_writeCTable() :
-    `CTable` : Huffman tree to save, using huf representation.
-    @return : size of saved CTable */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize,
-                        const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
-{
-    HUF_WriteCTableWksp wksp;
-    return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp));
-}
-
 
 size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights)
 {
@@ -486,7 +477,7 @@ typedef struct {
     U16 curr;
 } rankPos;
 
-typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
+typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)];
 
 /* Number of buckets available for HUF_sort() */
 #define RANK_POSITION_TABLE_SIZE 192
@@ -745,6 +736,8 @@ HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue
     nodeElt* const huffNode = huffNode0+1;
     int nonNullRank;
 
+    HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables));
+
     DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1);
 
     /* safety checks */
@@ -1133,11 +1126,6 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
 
 #endif
 
-size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
-{
-    return HUF_compress1X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
-}
-
 size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
 {
     return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
@@ -1193,11 +1181,6 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     return (size_t)(op-ostart);
 }
 
-size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
-{
-    return HUF_compress4X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
-}
-
 size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
 {
     return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
@@ -1422,18 +1405,6 @@ HUF_compress_internal (void* dst, size_t dstSize,
                                        nbStreams, table->CTable, bmi2);
 }
 
-
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
-                      const void* src, size_t srcSize,
-                      unsigned maxSymbolValue, unsigned huffLog,
-                      void* workSpace, size_t wkspSize)
-{
-    return HUF_compress_internal(dst, dstSize, src, srcSize,
-                                 maxSymbolValue, huffLog, HUF_singleStream,
-                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
-}
-
 size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
@@ -1448,21 +1419,6 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                                  repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
 }
 
-/* HUF_compress4X_repeat():
- * compress input using 4 streams.
- * provide workspace to generate compression tables */
-size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
-                      const void* src, size_t srcSize,
-                      unsigned maxSymbolValue, unsigned huffLog,
-                      void* workSpace, size_t wkspSize)
-{
-    DEBUGLOG(5, "HUF_compress4X_wksp (srcSize = %zu)", srcSize);
-    return HUF_compress_internal(dst, dstSize, src, srcSize,
-                                 maxSymbolValue, huffLog, HUF_fourStreams,
-                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
-}
-
 /* HUF_compress4X_repeat():
  * compress input using 4 streams.
  * consider skipping quickly
@@ -1480,36 +1436,3 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
                                  workSpace, wkspSize,
                                  hufTable, repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
 }
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-/** HUF_buildCTable() :
- * @return : maxNbBits
- *  Note : count is used before tree is written, so they can safely overlap
- */
-size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits)
-{
-    HUF_buildCTable_wksp_tables workspace;
-    return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace));
-}
-
-size_t HUF_compress1X (void* dst, size_t dstSize,
-                 const void* src, size_t srcSize,
-                 unsigned maxSymbolValue, unsigned huffLog)
-{
-    U64 workSpace[HUF_WORKSPACE_SIZE_U64];
-    return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
-}
-
-size_t HUF_compress2 (void* dst, size_t dstSize,
-                const void* src, size_t srcSize,
-                unsigned maxSymbolValue, unsigned huffLog)
-{
-    U64 workSpace[HUF_WORKSPACE_SIZE_U64];
-    return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
-}
-
-size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
-    return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
-}
-#endif
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4e653ccfb..a0463581a 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -16,7 +16,6 @@
 #include "hist.h"           /* HIST_countFast_wksp */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "zstd_compress_internal.h"
 #include "zstd_compress_sequences.h"
diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index 1971ec646..f8624ee8e 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -54,8 +54,6 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
     symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
     size_t cLitSize = 0;
 
-    (void)bmi2; /* TODO bmi2... */
-
     DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
 
     *entropyWritten = 0;
@@ -77,9 +75,8 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
         DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
     }
 
-    /* TODO bmi2 */
-    {   const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable)
-                                          : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable);
+    {   const size_t cSize = singleStream ? HUF_compress1X_usingCTable_bmi2(op, oend-op, literals, litSize, hufTable, bmi2)
+                                          : HUF_compress4X_usingCTable_bmi2(op, oend-op, literals, litSize, hufTable, bmi2);
         op += cSize;
         cLitSize += cSize;
         if (cSize == 0 || ERR_isError(cSize)) {
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index e5c35dae9..260398962 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -19,7 +19,6 @@
 #include "../common/compiler.h"
 #include "../common/bitstream.h"  /* BIT_* */
 #include "../common/fse.h"        /* to compress headers */
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/error_private.h"
 #include "../common/zstd_internal.h"
@@ -336,12 +335,6 @@ typedef struct {
         BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
 } HUF_ReadDTableX1_Workspace;
 
-
-size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
-{
-    return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
 size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2)
 {
     U32 tableLog = 0;
@@ -753,42 +746,6 @@ static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize,
 #endif
 }
 
-
-size_t HUF_decompress1X1_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 0) return ERROR(GENERIC);
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    const BYTE* ip = (const BYTE*) cSrc;
-
-    size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
-    if (HUF_isError(hSize)) return hSize;
-    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
-    ip += hSize; cSrcSize -= hSize;
-
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
-}
-
-
-size_t HUF_decompress4X1_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 0) return ERROR(GENERIC);
-    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
 static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
                                    void* workSpace, size_t wkspSize, int bmi2)
@@ -803,14 +760,6 @@ static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size
     return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
 }
 
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0);
-}
-
-
 #endif /* HUF_FORCE_DECOMPRESS_X2 */
 
 
@@ -1046,13 +995,6 @@ typedef struct {
     U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
 } HUF_ReadDTableX2_Workspace;
 
-size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
-                       const void* src, size_t srcSize,
-                             void* workSpace, size_t wkspSize)
-{
-    return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
 size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
                        const void* src, size_t srcSize,
                              void* workSpace, size_t wkspSize, int bmi2)
@@ -1466,40 +1408,19 @@ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize,
 
 HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
 
-size_t HUF_decompress1X2_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 1) return ERROR(GENERIC);
-    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
+size_t HUF_decompress1X2_DCtx_wksp_bmi2(HUF_DTable* DCtx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
+                                   void* workSpace, size_t wkspSize, int bmi2)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
-                                               workSpace, wkspSize);
+    size_t const hSize = HUF_readDTableX2_wksp_bmi2(DCtx, cSrc, cSrcSize,
+                                               workSpace, wkspSize, bmi2);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
-}
-
-
-size_t HUF_decompress4X2_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 1) return ERROR(GENERIC);
-    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
+    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, bmi2);
 }
 
 static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
@@ -1508,8 +1429,8 @@ static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
-                                         workSpace, wkspSize);
+    size_t hSize = HUF_readDTableX2_wksp_bmi2(dctx, cSrc, cSrcSize,
+                                         workSpace, wkspSize, bmi2);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
@@ -1517,14 +1438,6 @@ static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size
     return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
 }
 
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-
 #endif /* HUF_FORCE_DECOMPRESS_X1 */
 
 
@@ -1532,44 +1445,6 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* Universal decompression selectors */
 /* ***********************************/
 
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
-                                    const void* cSrc, size_t cSrcSize,
-                                    const HUF_DTable* DTable)
-{
-    DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-    (void)dtd;
-    assert(dtd.tableType == 0);
-    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-    (void)dtd;
-    assert(dtd.tableType == 1);
-    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
-    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
-                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
-                                    const void* cSrc, size_t cSrcSize,
-                                    const HUF_DTable* DTable)
-{
-    DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-    (void)dtd;
-    assert(dtd.tableType == 0);
-    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-    (void)dtd;
-    assert(dtd.tableType == 1);
-    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
-    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
-                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
 
 #if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
 typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
@@ -1624,36 +1499,9 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
 #endif
 }
 
-
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
-                                     size_t dstSize, const void* cSrc,
-                                     size_t cSrcSize, void* workSpace,
-                                     size_t wkspSize)
-{
-    /* validation checks */
-    if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize == 0) return ERROR(corruption_detected);
-
-    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-        (void)algoNb;
-        assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-        (void)algoNb;
-        assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#else
-        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                            cSrcSize, workSpace, wkspSize):
-                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#endif
-    }
-}
-
-size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+size_t HUF_decompress1X_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
                                   const void* cSrc, size_t cSrcSize,
-                                  void* workSpace, size_t wkspSize)
+                                  void* workSpace, size_t wkspSize, int bmi2)
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -1665,18 +1513,18 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 #if defined(HUF_FORCE_DECOMPRESS_X1)
         (void)algoNb;
         assert(algoNb == 0);
-        return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+        return HUF_decompress1X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, bmi2);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
         (void)algoNb;
         assert(algoNb == 1);
-        return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+        return HUF_decompress1X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, bmi2);
 #else
-        return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize):
-                        HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+        return algoNb ? HUF_decompress1X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, bmi2):
+                        HUF_decompress1X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, bmi2);
 #endif
     }
 }
@@ -1751,149 +1599,3 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds
 #endif
     }
 }
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_readDTableX1_wksp(DTable, src, srcSize,
-                                 workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
-                              const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
-{
-  U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-  return HUF_readDTableX2_wksp(DTable, src, srcSize,
-                               workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
-                              const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
-                              const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-
-size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
-    static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };
-#endif
-
-    /* validation checks */
-    if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize > dstSize) return ERROR(corruption_detected);   /* invalid */
-    if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; }   /* not compressed */
-    if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; }   /* RLE */
-
-    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-        (void)algoNb;
-        assert(algoNb == 0);
-        return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-        (void)algoNb;
-        assert(algoNb == 1);
-        return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);
-#else
-        return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
-#endif
-    }
-}
-
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    /* validation checks */
-    if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize > dstSize) return ERROR(corruption_detected);   /* invalid */
-    if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; }   /* not compressed */
-    if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; }   /* RLE */
-
-    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-        (void)algoNb;
-        assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-        (void)algoNb;
-        assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
-#else
-        return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
-                        HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
-#endif
-    }
-}
-
-size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                         workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
-                             const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                      workSpace, sizeof(workSpace));
-}
-#endif
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index 3647ad0a7..ad5c34a7f 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -19,7 +19,6 @@
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "zstd_decompress_internal.h"
 #include "zstd_ddict.h"
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 10d29f298..f0e4f99d2 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -59,7 +59,6 @@
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
 #include "../common/zstd_internal.h"  /* blockProperties_t */
@@ -1436,13 +1435,13 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
         size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
 #ifdef HUF_FORCE_DECOMPRESS_X1
         /* in minimal huffman, we always use X1 variants */
-        size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
+        size_t const hSize = HUF_readDTableX1_wksp_bmi2(entropy->hufTable,
                                                 dictPtr, dictEnd - dictPtr,
-                                                workspace, workspaceSize);
+                                                workspace, workspaceSize, /* bmi2 */ 0);
 #else
-        size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
+        size_t const hSize = HUF_readDTableX2_wksp_bmi2(entropy->hufTable,
                                                 dictPtr, (size_t)(dictEnd - dictPtr),
-                                                workspace, workspaceSize);
+                                                workspace, workspaceSize, /* bmi2 */ 0);
 #endif
         RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
         dictPtr += hSize;
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 853ddeae1..131114896 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -20,7 +20,6 @@
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/zstd_internal.h"
 #include "zstd_decompress_internal.h"   /* ZSTD_DCtx */
@@ -193,10 +192,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 } else {
                     if (singleStream) {
 #if defined(HUF_FORCE_DECOMPRESS_X2)
-                        hufSuccess = HUF_decompress1X_DCtx_wksp(
+                        hufSuccess = HUF_decompress1X_DCtx_wksp_bmi2(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace));
+                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
 #else
                         hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index f41d505f8..f22e04df2 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -44,7 +44,6 @@
 #ifndef ZDICT_STATIC_LINKING_ONLY
 #  define ZDICT_STATIC_LINKING_ONLY
 #endif
-#define HUF_STATIC_LINKING_ONLY
 
 #include "../common/mem.h"           /* read */
 #include "../common/fse.h"           /* FSE_normalizeCount, FSE_writeNCount */
@@ -676,6 +675,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
     size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles);
     BYTE* dstPtr = (BYTE*)dstBuffer;
+    U32 wksp[HUF_CTABLE_WORKSPACE_SIZE_U32];
 
     /* init */
     DEBUGLOG(4, "ZDICT_analyzeEntropy");
@@ -716,7 +716,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     }   }
 
     /* analyze, build stats, starting with literals */
-    {   size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
+    {   size_t maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
         if (HUF_isError(maxNbBits)) {
             eSize = maxNbBits;
             DISPLAYLEVEL(1, " HUF_buildCTable error \n");
@@ -725,7 +725,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
         if (maxNbBits==8) {  /* not compressible : will fail on HUF_writeCTable() */
             DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n");
             ZDICT_flatLit(countLit);  /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */
-            maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
+            maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
             assert(maxNbBits==9);
         }
         huffLog = (U32)maxNbBits;
@@ -766,7 +766,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     llLog = (U32)errorCode;
 
     /* write result to buffer */
-    {   size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
+    {   size_t const hhSize = HUF_writeCTable_wksp(dstPtr, maxDstSize, hufTable, 255, huffLog, wksp, sizeof(wksp));
         if (HUF_isError(hhSize)) {
             eSize = hhSize;
             DISPLAYLEVEL(1, "HUF_writeCTable error \n");
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 86bbaf8bf..68c147196 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -460,7 +460,7 @@ static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t ds
     }
 
     /* Write table description header */
-    {   size_t const hSize = HUF_writeCTable (op, dstSize, hufTable, maxSymbolValue, huffLog);
+    {   size_t const hSize = HUF_writeCTable_wksp (op, dstSize, hufTable, maxSymbolValue, huffLog, WKSP, sizeof(WKSP));
         if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
         op += hSize;
     }
@@ -562,12 +562,12 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
         do {
             compressedSize =
                     sizeFormat == 0
-                            ? HUF_compress1X_usingCTable(
+                            ? HUF_compress1X_usingCTable_bmi2(
                                       op, opend - op, LITERAL_BUFFER, litSize,
-                                      frame->stats.hufTable)
-                            : HUF_compress4X_usingCTable(
+                                      frame->stats.hufTable, /* bmi2 */ 0)
+                            : HUF_compress4X_usingCTable_bmi2(
                                       op, opend - op, LITERAL_BUFFER, litSize,
-                                      frame->stats.hufTable);
+                                      frame->stats.hufTable, /* bmi2 */ 0);
             CHECKERR(compressedSize);
             /* this only occurs when it could not compress or similar */
         } while (compressedSize <= 0);
diff --git a/tests/fuzz/huf_decompress.c b/tests/fuzz/huf_decompress.c
index 13f040a17..25cc9bdb0 100644
--- a/tests/fuzz/huf_decompress.c
+++ b/tests/fuzz/huf_decompress.c
@@ -13,8 +13,6 @@
  * compares the result with the original, and calls abort() on corruption.
  */
 
-#define HUF_STATIC_LINKING_ONLY
-
 #include <stddef.h>
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/tests/fuzz/huf_round_trip.c b/tests/fuzz/huf_round_trip.c
index 0993ecd14..19b3bea4f 100644
--- a/tests/fuzz/huf_round_trip.c
+++ b/tests/fuzz/huf_round_trip.c
@@ -13,8 +13,6 @@
  * compares the result with the original, and calls abort() on corruption.
  */
 
-#define HUF_STATIC_LINKING_ONLY
-
 #include <stddef.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -98,7 +96,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     if (symbols == 0) {
         FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
     } else {
-        size_t const ret = HUF_readDTableX2_wksp(dt, cBuf, tableSize, wksp, wkspSize);
+        size_t const ret = HUF_readDTableX2_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2);
         if (ERR_getErrorCode(ret) == ZSTD_error_tableLog_tooLarge) {
             FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
         } else {

From 329169189c248696c197ef9b88eb97c315bfb831 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 18 Jan 2023 16:30:10 -0800
Subject: [PATCH 137/197] Replace Huffman boolean args with flags bit set

---
 lib/common/huf.h                        |  66 +++++++----
 lib/compress/huf_compress.c             |  65 +++++------
 lib/compress/zstd_compress.c            |   8 +-
 lib/compress/zstd_compress_literals.c   |  13 ++-
 lib/compress/zstd_compress_superblock.c |   5 +-
 lib/decompress/huf_decompress.c         | 148 +++++++++++-------------
 lib/decompress/zstd_decompress.c        |   8 +-
 lib/decompress/zstd_decompress_block.c  |  21 ++--
 tests/decodecorpus.c                    |   8 +-
 tests/fullbench.c                       |   7 +-
 tests/fuzz/huf_decompress.c             |  15 ++-
 tests/fuzz/huf_round_trip.c             |  25 ++--
 12 files changed, 208 insertions(+), 181 deletions(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index d085d7741..3d574bbec 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -78,15 +78,43 @@ typedef U32 HUF_DTable;
 *  Advanced decompression functions
 ******************************************/
 
+/**
+ * Huffman flags bitset.
+ * For all flags, 0 is the default value.
+ */
+typedef enum {
+    /**
+     * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
+     * Otherwise: Ignored.
+     */
+    HUF_flags_bmi2 = (1 << 0),
+    /**
+     * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
+     * If unset: Use heuristic to find the table depth.
+     */
+    HUF_flags_optimalDepth = (1 << 1),
+    /**
+     * If set: If the previous table can encode the input, always reuse the previous table.
+     * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
+     */
+    HUF_flags_preferRepeat = (1 << 2),
+    /**
+     * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
+     * If unset: Always histogram the entire input.
+     */
+    HUF_flags_suspectUncompressible = (1 << 3),
+    /**
+     * If set: Don't use assembly implementations
+     * If unset: Allow using assembly implementations
+     */
+    HUF_flags_disableAsm = (1 << 4),
+} HUF_flags_e;
+
 
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
 #define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
-typedef enum {
-   HUF_depth_fast, /** Use heuristic to find the table depth**/
-   HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
- } HUF_depth_mode;
 
 /*! HUF_compress() does the following:
  *  1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
@@ -102,9 +130,9 @@ typedef enum {
 unsigned HUF_minTableLog(unsigned symbolCardinality);
 unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
- size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode); /* table is used as scratch space for building and testing tables, not a return value */
+ size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
 size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
-size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
 size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
 int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
 
@@ -124,8 +152,7 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
                        const void* src, size_t srcSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,    /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                       int suspectUncompressible, HUF_depth_mode depthMode);
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@@ -157,7 +184,7 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
                           U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
                           const void* src, size_t srcSize,
                           void* workspace, size_t wkspSize,
-                          int bmi2);
+                          int flags);
 
 /** HUF_readCTable() :
  *  Loading a CTable saved with HUF_writeCTable() */
@@ -200,7 +227,7 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
 /* single stream variants */
 /* ====================== */
 
-size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
 /** HUF_compress1X_repeat() :
  *  Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
  *  If it uses hufTable it does not modify hufTable or repeat.
@@ -211,28 +238,27 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
                        const void* src, size_t srcSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,   /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                       int suspectUncompressible, HUF_depth_mode depthMode);
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
 
-size_t HUF_decompress1X_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);   /**< double-symbols decoder */
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);   /**< double-symbols decoder */
 #endif
 
 /* BMI2 variants.
  * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
  */
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
 #ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
 
 #endif   /* HUF_H_298734234 */
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index e04ccd2eb..29871877a 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1105,9 +1105,9 @@ HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize,
 static size_t
 HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, const int bmi2)
+                              const HUF_CElt* CTable, const int flags)
 {
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
         return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
     }
     return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
@@ -1118,23 +1118,23 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
 static size_t
 HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, const int bmi2)
+                              const HUF_CElt* CTable, const int flags)
 {
-    (void)bmi2;
+    (void)flags;
     return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
 }
 
 #endif
 
-size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
 {
-    return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
+    return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
 }
 
 static size_t
 HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, int bmi2)
+                              const HUF_CElt* CTable, int flags)
 {
     size_t const segmentSize = (srcSize+3)/4;   /* first 3 segments */
     const BYTE* ip = (const BYTE*) src;
@@ -1148,7 +1148,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     op += 6;   /* jumpTable */
 
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart, (U16)cSize);
         op += cSize;
@@ -1156,7 +1156,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
 
     ip += segmentSize;
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart+2, (U16)cSize);
         op += cSize;
@@ -1164,7 +1164,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
 
     ip += segmentSize;
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart+4, (U16)cSize);
         op += cSize;
@@ -1173,7 +1173,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     ip += segmentSize;
     assert(op <= oend);
     assert(ip <= iend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         op += cSize;
     }
@@ -1181,9 +1181,9 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     return (size_t)(op-ostart);
 }
 
-size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
 {
-    return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
+    return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
 }
 
 typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
@@ -1191,11 +1191,11 @@ typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
 static size_t HUF_compressCTable_internal(
                 BYTE* const ostart, BYTE* op, BYTE* const oend,
                 const void* src, size_t srcSize,
-                HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
+                HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags)
 {
     size_t const cSize = (nbStreams==HUF_singleStream) ?
-                         HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) :
-                         HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2);
+                         HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) :
+                         HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags);
     if (HUF_isError(cSize)) { return cSize; }
     if (cSize==0) { return 0; }   /* uncompressible */
     op += cSize;
@@ -1243,12 +1243,12 @@ unsigned HUF_optimalTableLog(
             void* workSpace, size_t wkspSize,
             HUF_CElt* table,
       const unsigned* count,
-            HUF_depth_mode depthMode)
+            int flags)
 {
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
     assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
 
-    if (depthMode != HUF_depth_optimal) {
+    if (!(flags & HUF_flags_optimalDepth)) {
         /* cheap evaluation, based on FSE */
         return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
     }
@@ -1300,8 +1300,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
                        unsigned maxSymbolValue, unsigned huffLog,
                        HUF_nbStreams_e nbStreams,
                        void* workSpace, size_t wkspSize,
-                       HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
-                 const int bmi2, int suspectUncompressible, HUF_depth_mode depthMode)
+                       HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags)
 {
     HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
     BYTE* const ostart = (BYTE*)dst;
@@ -1322,15 +1321,15 @@ HUF_compress_internal (void* dst, size_t dstSize,
     if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
 
     /* Heuristic : If old table is valid, use it for small inputs */
-    if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
+    if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) {
         return HUF_compressCTable_internal(ostart, op, oend,
                                            src, srcSize,
-                                           nbStreams, oldHufTable, bmi2);
+                                           nbStreams, oldHufTable, flags);
     }
 
     /* If uncompressible data is suspected, do a smaller sampling first */
     DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2);
-    if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
+    if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
         size_t largestTotal = 0;
         DEBUGLOG(5, "input suspected incompressible : sampling to check");
         {   unsigned maxSymbolValueBegin = maxSymbolValue;
@@ -1358,14 +1357,14 @@ HUF_compress_internal (void* dst, size_t dstSize,
         *repeat = HUF_repeat_none;
     }
     /* Heuristic : use existing table for small inputs */
-    if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
+    if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) {
         return HUF_compressCTable_internal(ostart, op, oend,
                                            src, srcSize,
-                                           nbStreams, oldHufTable, bmi2);
+                                           nbStreams, oldHufTable, flags);
     }
 
     /* Build Huffman Tree */
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, depthMode);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags);
     {   size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
                                             maxSymbolValue, huffLog,
                                             &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
@@ -1390,7 +1389,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
             if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
                 return HUF_compressCTable_internal(ostart, op, oend,
                                                    src, srcSize,
-                                                   nbStreams, oldHufTable, bmi2);
+                                                   nbStreams, oldHufTable, flags);
         }   }
 
         /* Use the new huffman table */
@@ -1402,21 +1401,20 @@ HUF_compress_internal (void* dst, size_t dstSize,
     }
     return HUF_compressCTable_internal(ostart, op, oend,
                                        src, srcSize,
-                                       nbStreams, table->CTable, bmi2);
+                                       nbStreams, table->CTable, flags);
 }
 
 size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
-                      int bmi2, int suspectUncompressible, HUF_depth_mode depthMode)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
 {
     DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
                                  maxSymbolValue, huffLog, HUF_singleStream,
                                  workSpace, wkspSize, hufTable,
-                                 repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
+                                 repeat, flags);
 }
 
 /* HUF_compress4X_repeat():
@@ -1427,12 +1425,11 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                      int suspectUncompressible, HUF_depth_mode depthMode)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
 {
     DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
                                  maxSymbolValue, huffLog, HUF_fourStreams,
                                  workSpace, wkspSize,
-                                 hufTable, repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
+                                 hufTable, repeat, flags);
 }
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index a0463581a..fbb051ccc 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3369,7 +3369,7 @@ ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
                                      ZSTD_hufCTablesMetadata_t* hufMetadata,
                                const int literalsCompressionIsDisabled,
                                      void* workspace, size_t wkspSize,
-                                     HUF_depth_mode depthMode)
+                                     int hufFlags)
 {
     BYTE* const wkspStart = (BYTE*)workspace;
     BYTE* const wkspEnd = wkspStart + wkspSize;
@@ -3430,7 +3430,7 @@ ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
 
     /* Build Huffman Tree */
     ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, depthMode);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags);
     assert(huffLog <= LitHufLog);
     {   size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
                                                     maxSymbolValue, huffLog,
@@ -3538,14 +3538,14 @@ size_t ZSTD_buildBlockEntropyStats(
 {
     size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
     int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
-    HUF_depth_mode const depthMode = huf_useOptDepth ? HUF_depth_optimal : HUF_depth_fast;
+    int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0;
 
     entropyMetadata->hufMetadata.hufDesSize =
         ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
                                             &prevEntropy->huf, &nextEntropy->huf,
                                             &entropyMetadata->hufMetadata,
                                             ZSTD_literalsCompressionIsDisabled(cctxParams),
-                                            workspace, wkspSize, depthMode);
+                                            workspace, wkspSize, hufFlags);
 
     FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
     entropyMetadata->fseMetadata.fseTablesSize =
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index 0dbc2c4d4..bfd4f11ab 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -160,9 +160,13 @@ size_t ZSTD_compressLiterals (
 
     RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
     {   HUF_repeat repeat = prevHuf->repeatMode;
-        int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
-        HUF_depth_mode const depthMode = (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD) ? HUF_depth_optimal : HUF_depth_fast;
-        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, int, HUF_depth_mode);
+        int const flags = 0
+            | (bmi2 ? HUF_flags_bmi2 : 0)
+            | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
+            | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
+            | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
+
+        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
         huf_compress_f huf_compress;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
         huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
@@ -171,8 +175,7 @@ size_t ZSTD_compressLiterals (
                                 HUF_SYMBOLVALUE_MAX, LitHufLog,
                                 entropyWorkspace, entropyWorkspaceSize,
                                 (HUF_CElt*)nextHuf->CTable,
-                                &repeat, preferRepeat,
-                                bmi2, suspectUncompressible, depthMode);
+                                &repeat, flags);
         DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
         if (repeat != HUF_repeat_none) {
             /* reused the existing table */
diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index f8624ee8e..638c4acbe 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -75,8 +75,9 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
         DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
     }
 
-    {   const size_t cSize = singleStream ? HUF_compress1X_usingCTable_bmi2(op, oend-op, literals, litSize, hufTable, bmi2)
-                                          : HUF_compress4X_usingCTable_bmi2(op, oend-op, literals, litSize, hufTable, bmi2);
+    {   int const flags = bmi2 ? HUF_flags_bmi2 : 0;
+        const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags)
+                                          : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags);
         op += cSize;
         cLitSize += cSize;
         if (cSize == 0 || ERR_isError(cSize)) {
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 260398962..400a4e44d 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -62,12 +62,6 @@
 # define HUF_NEED_BMI2_FUNCTION 0
 #endif
 
-#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
-# define HUF_NEED_DEFAULT_FUNCTION 1
-#else
-# define HUF_NEED_DEFAULT_FUNCTION 0
-#endif
-
 /* **************************************************************
 *  Error Management
 ****************************************************************/
@@ -105,9 +99,9 @@
     }                                                                       \
                                                                             \
     static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
-                     size_t cSrcSize, HUF_DTable const* DTable, int bmi2)   \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
     {                                                                       \
-        if (bmi2) {                                                         \
+        if (flags & HUF_flags_bmi2) {                                       \
             return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);         \
         }                                                                   \
         return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable);          \
@@ -117,9 +111,9 @@
 
 #define HUF_DGEN(fn)                                                        \
     static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
-                     size_t cSrcSize, HUF_DTable const* DTable, int bmi2)   \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
     {                                                                       \
-        (void)bmi2;                                                         \
+        (void)flags;                                                        \
         return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
     }
 
@@ -335,7 +329,7 @@ typedef struct {
         BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
 } HUF_ReadDTableX1_Workspace;
 
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags)
 {
     U32 tableLog = 0;
     U32 nbSymbols = 0;
@@ -350,7 +344,7 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
     DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
     /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */   /* is not necessary, even though some analyzer complain ... */
 
-    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2);
+    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags);
     if (HUF_isError(iSize)) return iSize;
 
 
@@ -653,13 +647,11 @@ size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo
 }
 #endif
 
-#if HUF_NEED_DEFAULT_FUNCTION
 static
 size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable) {
     return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
 }
-#endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
@@ -725,39 +717,39 @@ typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
 HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
 
 static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
-                    size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
-        return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm))
+            return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
 # endif
+        return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
     }
 #else
-    (void)bmi2;
+    (void)flags;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
-    return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm))
+        return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
 #endif
+    return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
-static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
+static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize, int bmi2)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
+    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 
 #endif /* HUF_FORCE_DECOMPRESS_X2 */
@@ -995,9 +987,9 @@ typedef struct {
     U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
 } HUF_ReadDTableX2_Workspace;
 
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
                        const void* src, size_t srcSize,
-                             void* workSpace, size_t wkspSize, int bmi2)
+                             void* workSpace, size_t wkspSize, int flags)
 {
     U32 tableLog, maxW, nbSymbols;
     DTableDesc dtd = HUF_getDTableDesc(DTable);
@@ -1019,7 +1011,7 @@ size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
     if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
     /* ZSTD_memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
 
-    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2);
+    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags);
     if (HUF_isError(iSize)) return iSize;
 
     /* check result */
@@ -1322,13 +1314,11 @@ size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo
 }
 #endif
 
-#if HUF_NEED_DEFAULT_FUNCTION
 static
 size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable) {
     return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
 }
-#endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
@@ -1385,57 +1375,57 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm(
 #endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
 
 static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
-                    size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
-        return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm))
+            return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
 # endif
+        return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
     }
 #else
-    (void)bmi2;
+    (void)flags;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
-    return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm))
+        return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
 #endif
+    return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
 HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
 
-size_t HUF_decompress1X2_DCtx_wksp_bmi2(HUF_DTable* DCtx, void* dst, size_t dstSize,
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize, int bmi2)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX2_wksp_bmi2(DCtx, cSrc, cSrcSize,
-                                               workSpace, wkspSize, bmi2);
+    size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
+                                               workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, bmi2);
+    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags);
 }
 
-static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
+static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize, int bmi2)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t hSize = HUF_readDTableX2_wksp_bmi2(dctx, cSrc, cSrcSize,
-                                         workSpace, wkspSize, bmi2);
+    size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
+                                         workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
+    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 
 #endif /* HUF_FORCE_DECOMPRESS_X1 */
@@ -1499,9 +1489,9 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
 #endif
 }
 
-size_t HUF_decompress1X_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                   const void* cSrc, size_t cSrcSize,
-                                  void* workSpace, size_t wkspSize, int bmi2)
+                                  void* workSpace, size_t wkspSize, int flags)
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -1513,72 +1503,72 @@ size_t HUF_decompress1X_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSi
 #if defined(HUF_FORCE_DECOMPRESS_X1)
         (void)algoNb;
         assert(algoNb == 0);
-        return HUF_decompress1X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
         (void)algoNb;
         assert(algoNb == 1);
-        return HUF_decompress1X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags);
 #else
-        return algoNb ? HUF_decompress1X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize, bmi2):
-                        HUF_decompress1X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize, bmi2);
+        return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags):
+                        HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags);
 #endif
     }
 }
 
 
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
 {
     DTableDesc const dtd = HUF_getDTableDesc(DTable);
 #if defined(HUF_FORCE_DECOMPRESS_X1)
     (void)dtd;
     assert(dtd.tableType == 0);
-    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
     (void)dtd;
     assert(dtd.tableType == 1);
-    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #else
-    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
-                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #endif
 }
 
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
+    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 #endif
 
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
 {
     DTableDesc const dtd = HUF_getDTableDesc(DTable);
 #if defined(HUF_FORCE_DECOMPRESS_X1)
     (void)dtd;
     assert(dtd.tableType == 0);
-    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
     (void)dtd;
     assert(dtd.tableType == 1);
-    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #else
-    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
-                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #endif
 }
 
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -1588,14 +1578,14 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds
 #if defined(HUF_FORCE_DECOMPRESS_X1)
         (void)algoNb;
         assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
         (void)algoNb;
         assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #else
-        return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
-                        HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) :
+                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #endif
     }
 }
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index f0e4f99d2..7e6ecd7df 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1435,13 +1435,13 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
         size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
 #ifdef HUF_FORCE_DECOMPRESS_X1
         /* in minimal huffman, we always use X1 variants */
-        size_t const hSize = HUF_readDTableX1_wksp_bmi2(entropy->hufTable,
+        size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
                                                 dictPtr, dictEnd - dictPtr,
-                                                workspace, workspaceSize, /* bmi2 */ 0);
+                                                workspace, workspaceSize, /* flags */ 0);
 #else
-        size_t const hSize = HUF_readDTableX2_wksp_bmi2(entropy->hufTable,
+        size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
                                                 dictPtr, (size_t)(dictEnd - dictPtr),
-                                                workspace, workspaceSize, /* bmi2 */ 0);
+                                                workspace, workspaceSize, /* flags */ 0);
 #endif
         RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
         dictPtr += hSize;
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 131114896..fbe4bf674 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -141,6 +141,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 U32 const lhc = MEM_readLE32(istart);
                 size_t hufSuccess;
                 size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+                int const flags = ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0;
                 switch(lhlCode)
                 {
                 case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
@@ -180,33 +181,33 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
 
                 if (litEncType==set_repeat) {
                     if (singleStream) {
-                        hufSuccess = HUF_decompress1X_usingDTable_bmi2(
+                        hufSuccess = HUF_decompress1X_usingDTable(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
-                            dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
+                            dctx->HUFptr, flags);
                     } else {
                         assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
-                        hufSuccess = HUF_decompress4X_usingDTable_bmi2(
+                        hufSuccess = HUF_decompress4X_usingDTable(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
-                            dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
+                            dctx->HUFptr, flags);
                     }
                 } else {
                     if (singleStream) {
 #if defined(HUF_FORCE_DECOMPRESS_X2)
-                        hufSuccess = HUF_decompress1X_DCtx_wksp_bmi2(
+                        hufSuccess = HUF_decompress1X_DCtx_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
+                            sizeof(dctx->workspace), flags);
 #else
-                        hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2(
+                        hufSuccess = HUF_decompress1X1_DCtx_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
+                            sizeof(dctx->workspace), flags);
 #endif
                     } else {
-                        hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2(
+                        hufSuccess = HUF_decompress4X_hufOnly_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
+                            sizeof(dctx->workspace), flags);
                     }
                 }
                 if (dctx->litBufferLocation == ZSTD_split)
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 68c147196..148088a76 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -562,12 +562,12 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
         do {
             compressedSize =
                     sizeFormat == 0
-                            ? HUF_compress1X_usingCTable_bmi2(
+                            ? HUF_compress1X_usingCTable(
                                       op, opend - op, LITERAL_BUFFER, litSize,
-                                      frame->stats.hufTable, /* bmi2 */ 0)
-                            : HUF_compress4X_usingCTable_bmi2(
+                                      frame->stats.hufTable, /* flags */ 0)
+                            : HUF_compress4X_usingCTable(
                                       op, opend - op, LITERAL_BUFFER, litSize,
-                                      frame->stats.hufTable, /* bmi2 */ 0);
+                                      frame->stats.hufTable, /* flags */ 0);
             CHECKERR(compressedSize);
             /* this only occurs when it could not compress or similar */
         } while (compressedSize <= 0);
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 25c6d2b40..3a72d89d3 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -171,6 +171,7 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
                 size_t lhSize, litSize, litCSize;
                 U32 const lhlCode = (istart[0] >> 2) & 3;
                 U32 const lhc = MEM_readLE32(istart);
+                int const flags = ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0;
                 switch(lhlCode)
                 {
                 case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
@@ -195,16 +196,16 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
                 RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
                 RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
 #ifndef HUF_FORCE_DECOMPRESS_X2
-                return HUF_readDTableX1_wksp_bmi2(
+                return HUF_readDTableX1_wksp(
                         dctx->entropy.hufTable,
                         istart+lhSize, litCSize,
                         dctx->workspace, sizeof(dctx->workspace),
-                        ZSTD_DCtx_get_bmi2(dctx));
+                        flags);
 #else
                 return HUF_readDTableX2_wksp(
                         dctx->entropy.hufTable,
                         istart+lhSize, litCSize,
-                        dctx->workspace, sizeof(dctx->workspace));
+                        dctx->workspace, sizeof(dctx->workspace), flags);
 #endif
             }
         }
diff --git a/tests/fuzz/huf_decompress.c b/tests/fuzz/huf_decompress.c
index 25cc9bdb0..a9b3540ab 100644
--- a/tests/fuzz/huf_decompress.c
+++ b/tests/fuzz/huf_decompress.c
@@ -28,7 +28,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     /* Select random parameters: #streams, X1 or X2 decoding, bmi2 */
     int const streams = FUZZ_dataProducer_int32Range(producer, 0, 1);
     int const symbols = FUZZ_dataProducer_int32Range(producer, 0, 1);
-    int const bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1);
+    int const flags = 0
+        | (ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_bmi2 : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0);
     /* Select a random cBufSize - it may be too small */
     size_t const dBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 8 * size + 500);
     size_t const maxTableLog = FUZZ_dataProducer_uint32Range(producer, 1, HUF_TABLELOG_MAX);
@@ -40,18 +45,18 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     size = FUZZ_dataProducer_remainingBytes(producer);
 
     if (symbols == 0) {
-        size_t const err = HUF_readDTableX1_wksp_bmi2(dt, src, size, wksp, wkspSize, bmi2);
+        size_t const err = HUF_readDTableX1_wksp(dt, src, size, wksp, wkspSize, flags);
         if (ZSTD_isError(err))
             goto _out;
     } else {
-        size_t const err = HUF_readDTableX2_wksp_bmi2(dt, src, size, wksp, wkspSize, bmi2);
+        size_t const err = HUF_readDTableX2_wksp(dt, src, size, wksp, wkspSize, flags);
         if (ZSTD_isError(err))
             goto _out;
     }
     if (streams == 0)
-        HUF_decompress1X_usingDTable_bmi2(dBuf, dBufSize, src, size, dt, bmi2);
+        HUF_decompress1X_usingDTable(dBuf, dBufSize, src, size, dt, flags);
     else
-        HUF_decompress4X_usingDTable_bmi2(dBuf, dBufSize, src, size, dt, bmi2);
+        HUF_decompress4X_usingDTable(dBuf, dBufSize, src, size, dt, flags);
 
 _out:
     free(dt);
diff --git a/tests/fuzz/huf_round_trip.c b/tests/fuzz/huf_round_trip.c
index 19b3bea4f..9967c57bf 100644
--- a/tests/fuzz/huf_round_trip.c
+++ b/tests/fuzz/huf_round_trip.c
@@ -44,7 +44,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     /* Select random parameters: #streams, X1 or X2 decoding, bmi2 */
     int const streams = FUZZ_dataProducer_int32Range(producer, 0, 1);
     int const symbols = FUZZ_dataProducer_int32Range(producer, 0, 1);
-    int const bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1);
+    int const flags = 0
+        | (ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_bmi2 : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0);
     /* Select a random cBufSize - it may be too small */
     size_t const cBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 4 * size);
     /* Select a random tableLog - we'll adjust it up later */
@@ -81,9 +86,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     HUF_DTable* dt = (HUF_DTable*)FUZZ_malloc(HUF_DTABLE_SIZE(tableLog) * sizeof(HUF_DTable));
     dt[0] = tableLog * 0x01000001;
 
-    HUF_depth_mode depthMode = rand() & 1 ? HUF_depth_fast : HUF_depth_optimal;
-
-    tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, depthMode);
+    tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, flags);
     FUZZ_ASSERT(tableLog <= 12);
     tableLog = HUF_buildCTable_wksp(ct, count, maxSymbol, tableLog, wksp, wkspSize);
     FUZZ_ZASSERT(tableLog);
@@ -94,11 +97,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     }
     FUZZ_ZASSERT(tableSize);
     if (symbols == 0) {
-        FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
+        FUZZ_ZASSERT(HUF_readDTableX1_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags));
     } else {
-        size_t const ret = HUF_readDTableX2_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2);
+        size_t const ret = HUF_readDTableX2_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags);
         if (ERR_getErrorCode(ret) == ZSTD_error_tableLog_tooLarge) {
-            FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
+            FUZZ_ZASSERT(HUF_readDTableX1_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags));
         } else {
             FUZZ_ZASSERT(ret);
         }
@@ -107,15 +110,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     size_t cSize;
     size_t rSize;
     if (streams == 0) {
-        cSize = HUF_compress1X_usingCTable_bmi2(cBuf, cBufSize, src, size, ct, bmi2);
+        cSize = HUF_compress1X_usingCTable(cBuf, cBufSize, src, size, ct, flags);
         FUZZ_ZASSERT(cSize);
         if (cSize != 0)
-            rSize = HUF_decompress1X_usingDTable_bmi2(rBuf, size, cBuf, cSize, dt, bmi2);
+            rSize = HUF_decompress1X_usingDTable(rBuf, size, cBuf, cSize, dt, flags);
     } else {
-        cSize = HUF_compress4X_usingCTable_bmi2(cBuf, cBufSize, src, size, ct, bmi2);
+        cSize = HUF_compress4X_usingCTable(cBuf, cBufSize, src, size, ct, flags);
         FUZZ_ZASSERT(cSize);
         if (cSize != 0)
-            rSize = HUF_decompress4X_usingDTable_bmi2(rBuf, size, cBuf, cSize, dt, bmi2);
+            rSize = HUF_decompress4X_usingDTable(rBuf, size, cBuf, cSize, dt, flags);
     }
     if (cSize != 0) {
         FUZZ_ZASSERT(rSize);

From 87e169d05d75db5112df20361de33f7398ef27d5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Jan 2023 17:32:49 -0500
Subject: [PATCH 138/197] Add Additional Flags to PGO Build

In GCC, we can add a couple more flags to give us confidence that the profile
data is actually being found and used.

Also, my system for example doesn't have a binary installed under the name
`llvm-profdata`, but it does have, e.g., `llvm-profdata-13`, etc. So this
commit adds a variable that can be overridden.
---
 programs/Makefile | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index b79b60b4f..fcff41dc1 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -231,9 +231,12 @@ zstd-dll : zstd
 
 ## zstd-pgo: zstd executable optimized with PGO.
 .PHONY: zstd-pgo
+zstd-pgo : LLVM_PROFDATA?=llvm-profdata
+zstd-pgo : PROF_GENERATE_FLAGS=-fprofile-generate $(if $(findstring gcc,$(CC)),-fprofile-dir=.)
+zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Werror=missing-profile -Wno-error=coverage-mismatch)
 zstd-pgo :
 	$(MAKE) clean HASH_DIR=$(HASH_DIR)
-	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-generate
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_GENERATE_FLAGS)"
 	./zstd -b19i1 $(PROFILE_WITH)
 	./zstd -b16i1 $(PROFILE_WITH)
 	./zstd -b9i2 $(PROFILE_WITH)
@@ -245,8 +248,8 @@ ifndef BUILD_DIR
 else
 	$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
 endif
-	case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
-	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-use
+	case $(CC) in *clang*) if ! [ -e default.profdata ]; then $(LLVM_PROFDATA) merge -output=default.profdata default*.profraw; fi ;; esac
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_USE_FLAGS)"
 
 ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
 CLEAN += zstd-small zstd-frugal

From aab3dd4312d98b2eff868012a50b1523f41a1a9a Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Jan 2023 17:36:24 -0500
Subject: [PATCH 139/197] Add PGO Build Jobs to CI

---
 .github/workflows/dev-short-tests.yml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 5a29789f9..3f6675f0d 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -500,6 +500,29 @@ jobs:
       run: |
         make clangbuild
 
+  clang-pgo:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: Build PGO Zstd with Clang
+      env:
+        CC: clang-14
+        LLVM_PROFDATA: llvm-profdata-14
+      run: |
+        make -C programs zstd-pgo
+        ./programs/zstd -b
+
+  gcc-pgo:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: Build PGO Zstd with GCC
+      env:
+        CC: gcc
+      run: |
+        make -C programs zstd-pgo
+        ./programs/zstd -b
+
 
 # For reference : icc tests
 # icc tests are currently failing on Github Actions, likely to issues during installation stage

From b4467c10611018eef3189ffc6faf681c8b4eca6b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 20 Jan 2023 14:14:14 -0800
Subject: [PATCH 140/197] Fix bufferless API with attached dictionary

Fixes #3102.
---
 lib/compress/zstd_compress_internal.h |  7 ++++++-
 tests/fuzzer.c                        | 21 +++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index c93e85fb5..79068bcb8 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -1172,10 +1172,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
                     (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
         assert(blockEndIdx >= loadedDictEnd);
 
-        if (blockEndIdx > loadedDictEnd + maxDist) {
+        if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) {
             /* On reaching window size, dictionaries are invalidated.
              * For simplification, if window size is reached anywhere within next block,
              * the dictionary is invalidated for the full block.
+             *
+             * We also have to invalidate the dictionary if ZSTD_window_update() has detected
+             * non-contiguous segments, which means that loadedDictEnd != window->dictLimit.
+             * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use
+             * dictMatchState, so setting it to NULL is not a problem.
              */
             DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
             *loadedDictEndPtr = 0;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index e02d06872..bd3ca0e02 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -2592,6 +2592,27 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
+        DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
+        {   ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
+            ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+            ZSTD_frameParameters const fParams = { 0, 1, 0 };
+            size_t cBlockSize;
+            cSize = 0;
+            CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
+            cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
+            CHECK_Z(cBlockSize);
+            cSize += cBlockSize;
+            cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
+            CHECK_Z(cBlockSize);
+            cSize += cBlockSize;
+
+            CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
+
+            ZSTD_freeCDict(cdict);
+            ZSTD_freeDCtx(dctx);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);

From d9280afb7dfb39aa6417e7465c3905f2de864bbd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 20 Jan 2023 18:04:20 -0800
Subject: [PATCH 141/197] fixed minor c89 warning

introduced due to parallel merges
---
 lib/common/huf.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index 3d574bbec..5ab79db7a 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -107,7 +107,7 @@ typedef enum {
      * If set: Don't use assembly implementations
      * If unset: Allow using assembly implementations
      */
-    HUF_flags_disableAsm = (1 << 4),
+    HUF_flags_disableAsm = (1 << 4)
 } HUF_flags_e;
 
 

From cee6bec9fa6aa249f2df9f84165b682eb793eab4 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 20 Jan 2023 17:53:05 -0800
Subject: [PATCH 142/197] refactor : --rm is ignored with stdout

`zstd` CLI has progressively moved to the policy of
ignoring `--rm` command when the output is `stdout`.
The primary drive is to feature a behavior more consistent with `gzip`,
when `--rm` is the default, but is also ignored when output is `stdout`.
Other policies are certainly possible, but would break from this `gzip` convention.

The new policy was inconsistenly enforced, depending on the exact list of commands.
For example, it was possible to circumvent it by using `-c --rm` in this order,
which would re-establish source removal.

- Update the CLI so that it necessarily catch these situations and ensure that `--rm` is always disabled when output is `stdout`.
- Added a warning message in this case (for verbosity 3 `-v`).
- Added an `assert()`, which controls that `--rm` is no longer active with `stdout`
- Added tests, which control the behavior, even when `--rm` is added after `-c`
- Removed some legacy code which where trying to apply a specific policy for the `stdout` + `--rm` case, which is no longer possible
---
 programs/fileio.c  | 14 +++++---------
 programs/fileio.h  |  2 +-
 programs/zstdcli.c | 39 +++++++++++++++++++++++----------------
 tests/playTests.sh | 10 ++++++++--
 4 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 367ade415..2d79203de 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -362,7 +362,7 @@ void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictID
 
 void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; }
 
-void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); }
+void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag) { prefs->removeSrcFile = (flag!=0); }
 
 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
 
@@ -833,7 +833,7 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
  *         If -q is specified with --rm, zstd will abort pre-emptively
  *         If neither flag is specified, zstd will prompt the user for confirmation to proceed.
  * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q).
- * However, if the output is stdout, we will always abort rather than displaying the warning prompt.
+ *         Note : --rm in combination with stdout is not allowed.
  */
 static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff)
 {
@@ -850,14 +850,10 @@ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t*
             } else {
                 DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
             }
-            DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate the original directory tree. \n")
+            DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate original file names nor directory structure. \n")
             if (prefs->removeSrcFile) {
-                if (fCtx->hasStdoutOutput) {
-                    DISPLAYLEVEL(1, "Aborting. Use -f if you really want to delete the files and output to stdout\n");
-                    error = 1;
-                } else {
-                    error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
-                }
+                assert(fCtx->hasStdoutOutput == 0); /* not possible : never erase source files when output == stdout */
+                error = (g_display_prefs.displayLevel > displayLevelCutoff) && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
             }
         }
     }
diff --git a/programs/fileio.h b/programs/fileio.h
index e37398ded..291d4d414 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -86,7 +86,7 @@ void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch);
 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
 void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
 void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
-void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
+void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag);
 void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
 void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
 void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 9937a9c97..b3386988d 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -850,7 +850,8 @@ int main(int argCount, const char* argv[])
         defaultLogicalCores = 0,
         showDefaultCParams = 0,
         ultra=0,
-        contentSize=1;
+        contentSize=1,
+        removeSrcFile=0;
     unsigned nbWorkers = 0;
     double compressibility = 0.5;
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
@@ -909,17 +910,17 @@ int main(int argCount, const char* argv[])
     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
     if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZ)) {   /* behave like gzip */
-        suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
+        suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); removeSrcFile=1;
         dictCLevel = cLevel = 6;  /* gzip default is -6 */
     }
-    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); }                                                     /* behave like gunzip, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; }                                                     /* behave like gunzip, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); }    /* behave like lzma */
-    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }          /* behave like xz */
-    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }     /* behave like unxz, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); }                                       /* behave like lz4 */
-    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); }                                   /* behave like unlz4, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); removeSrcFile=1; }    /* behave like lzma */
+    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); removeSrcFile=1; }          /* behave like xz */
+    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); removeSrcFile=1; }     /* behave like unxz, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); }                        /* behave like lz4 */
+    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); }                    /* behave like unlz4, also supports multiple formats */
     memset(&compressionParams, 0, sizeof(compressionParams));
 
     /* init crash handler */
@@ -956,7 +957,7 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); }
                 if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
                 if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
-                if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; FIO_setRemoveSrcFile(prefs, 0); g_displayLevel-=(g_displayLevel==2); continue; }
+                if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; removeSrcFile=0; continue; }
                 if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
                 if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
                 if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
@@ -969,8 +970,8 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}
                 if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
                 if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; }
-                if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
-                if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
+                if (!strcmp(argument, "--keep")) { removeSrcFile=0; continue; }
+                if (!strcmp(argument, "--rm")) { removeSrcFile=1; continue; }
                 if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
                 if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; }
                 if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
@@ -1167,7 +1168,7 @@ int main(int argCount, const char* argv[])
                         operation=zom_decompress; argument++; break;
 
                     /* Force stdout, even if stdout==console */
-                case 'c': forceStdout=1; outFileName=stdoutmark; FIO_setRemoveSrcFile(prefs, 0); argument++; break;
+                case 'c': forceStdout=1; outFileName=stdoutmark; removeSrcFile=0; argument++; break;
 
                     /* do not store filename - gzip compatibility - nothing to do */
                 case 'n': argument++; break;
@@ -1185,7 +1186,7 @@ int main(int argCount, const char* argv[])
                 case 'q': g_displayLevel--; argument++; break;
 
                     /* keep source file (default) */
-                case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break;
+                case 'k': removeSrcFile=0; argument++; break;
 
                     /* Checksum */
                 case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break;
@@ -1434,7 +1435,7 @@ int main(int argCount, const char* argv[])
     }
 
 #ifndef ZSTD_NODECOMPRESS
-    if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); }  /* test mode */
+    if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; removeSrcFile=0; }  /* test mode */
 #endif
 
     /* No input filename ==> use stdin and stdout */
@@ -1496,9 +1497,15 @@ int main(int argCount, const char* argv[])
 
     /* No status message in pipe mode (stdin - stdout) */
     hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
-
     if ((hasStdout || !UTIL_isConsole(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
 
+    /* don't remove source files when output is stdout */;
+    if (hasStdout && removeSrcFile) {
+        DISPLAYLEVEL(3, "Note: src files are not removed when output is stdout \n");
+        removeSrcFile = 0;
+    }
+    FIO_setRemoveSrcFile(prefs, removeSrcFile);
+
     /* IO Stream/File */
     FIO_setHasStdoutOutput(fCtx, hasStdout);
     FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 06e6ba951..d0764d4d3 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -388,6 +388,14 @@ zstd -f --rm tmp
 test ! -f tmp  # tmp should no longer be present
 zstd -f -d --rm tmp.zst
 test ! -f tmp.zst   # tmp.zst should no longer be present
+println "test: --rm is disabled when output is stdout"
+test -f tmp
+zstd --rm tmp -c > $INTOVOID
+test -f tmp # tmp shall still be there
+zstd -f --rm tmp -c > $INTOVOID
+test -f tmp # tmp shall still be there
+zstd -f tmp -c > $INTOVOID --rm
+test -f tmp # tmp shall still be there
 println "test : should quietly not remove non-regular file"
 println hello > tmp
 zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
@@ -450,8 +458,6 @@ cp tmp_rm3.zst tmp_rm4.zst
 echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
 test ! -f tmp_rm3.zst
 test ! -f tmp_rm4.zst
-echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm && die "compressing multiple files to stdout with --rm should fail unless -f is specified"
-echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm -v && die "compressing multiple files to stdout with --rm should fail unless -f is specified"
 println gooder > tmpexists1
 zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID
 

From b6fd91ba84633f1e077f98a2df69bab8b430600e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 20 Jan 2023 18:07:55 -0800
Subject: [PATCH 143/197] update man

---
 programs/zstd.1.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 45a88a347..3b7f24f79 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -241,8 +241,10 @@ the last one takes effect.
     occur when the output destination is stdout and the force (`-f`) option is
     set.
 * `--rm`:
-    remove source file(s) after successful compression or decompression. If used in combination with
-    `-o`, will trigger a confirmation prompt (which can be silenced with `-f`), as this is a destructive operation.
+    remove source file(s) after successful compression or decompression.
+    This command is silently ignored if output is `stdout`.
+    If used in combination with `-o`,
+    triggers a confirmation prompt (which can be silenced with `-f`), as this is a destructive operation.
 * `-k`, `--keep`:
     keep source file(s) after successful compression or decompression.
     This is the default behavior.

From 3663faa05abd2527ec1ec27723aca698265a1a61 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Jan 2023 05:10:26 +0000
Subject: [PATCH 144/197] Bump github/codeql-action from 2.1.38 to 2.1.39

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.1.38 to 2.1.39.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/515828d97454b8354517688ddc5b48402b723750...a34ca99b4610d924e04c68db79e503e1f79f9f02)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/scorecards.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 16b0f846c..a2277d761 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -59,6 +59,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@515828d97454b8354517688ddc5b48402b723750 # tag=v2.1.38
+        uses: github/codeql-action/upload-sarif@a34ca99b4610d924e04c68db79e503e1f79f9f02 # tag=v2.1.39
         with:
           sarif_file: results.sarif

From 638d502002b6fbecb3e8ccc79b159766e1244a83 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 23 Jan 2023 07:55:11 -0800
Subject: [PATCH 145/197] modify sequence compression api fuzzer

---
 .gitignore                            |   1 +
 lib/compress/zstd_compress.c          |  81 ++++++++++---
 lib/compress/zstd_compress_internal.h |   3 +
 lib/zstd.h                            |  38 +++++-
 programs/benchzstd.c                  |   7 +-
 programs/dibio.c                      |   9 +-
 programs/fileio_common.h              |   2 +-
 programs/timefn.c                     | 106 +++++++++++------
 programs/timefn.h                     |  52 +++------
 tests/decodecorpus.c                  |  27 +++--
 tests/external_matchfinder.h          |   2 +-
 tests/fuzz/sequence_compression_api.c | 159 +++++++++++++++++---------
 tests/fuzz/zstd_helpers.c             |   1 +
 tests/fuzzer.c                        | 113 ++++++++++++++++--
 tests/test-zstd-versions.py           |  31 ++++-
 tests/zstreamtest.c                   | 154 +++++++++++++++++++++++--
 16 files changed, 587 insertions(+), 199 deletions(-)

diff --git a/.gitignore b/.gitignore
index a136ea394..cefdfe923 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,6 +26,7 @@ tmp*
 *.zstd
 dictionary.
 dictionary
+sequence_fuzz_dictionary
 NUL
 
 # Build artefacts
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b12cacc77..3a48e7dcd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -288,6 +288,15 @@ static int ZSTD_resolveExternalSequenceValidation(int mode) {
 #endif
 }
 
+/* Resolves maxBlockSize to the default if no value is present. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+    if (maxBlockSize == 0) {
+        return ZSTD_BLOCKSIZE_MAX;
+    } else {
+        return maxBlockSize;
+    }
+}
+
 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
  * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
@@ -312,6 +321,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
     cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+    cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -377,6 +387,7 @@ ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
     cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
     cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+    cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -604,6 +615,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = 1;
         return bounds;
 
+    case ZSTD_c_maxBlockSize:
+        bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+        bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+        return bounds;
+
     default:
         bounds.error = ERROR(parameter_unsupported);
         return bounds;
@@ -670,6 +686,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
     case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
     default:
         return 0;
     }
@@ -727,6 +744,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
     case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
         break;
 
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -964,6 +982,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->enableMatchFinderFallback = value;
         return CCtxParams->enableMatchFinderFallback;
 
+    case ZSTD_c_maxBlockSize:
+        if (value!=0)    /* 0 ==> default */
+            BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+        CCtxParams->maxBlockSize = value;
+        return CCtxParams->maxBlockSize;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -1102,6 +1126,9 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_enableMatchFinderFallback:
         *value = CCtxParams->enableMatchFinderFallback;
         break;
+    case ZSTD_c_maxBlockSize:
+        *value = (int)CCtxParams->maxBlockSize;
+        break;
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
@@ -1546,10 +1573,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const size_t buffInSize,
         const size_t buffOutSize,
         const U64 pledgedSrcSize,
-        int useExternalMatchFinder)
+        int useExternalMatchFinder,
+        size_t maxBlockSize)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+    size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
     size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
@@ -1571,7 +1599,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 
     size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
     size_t const externalSeqSpace = useExternalMatchFinder
-        ? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+        ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
         : 0;
 
     size_t const neededSpace =
@@ -1601,7 +1629,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
      * be needed. However, we still allocate two 0-sized buffers, which can
      * take space under ASAN. */
     return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
-        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
 }
 
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1651,7 +1679,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+        size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
         size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
                 ? ((size_t)1 << cParams.windowLog) + blockSize
                 : 0;
@@ -1662,7 +1690,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
-            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
     }
 }
 
@@ -1936,6 +1964,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     assert(params->useRowMatchFinder != ZSTD_ps_auto);
     assert(params->useBlockSplitter != ZSTD_ps_auto);
     assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+    assert(params->maxBlockSize != 0);
     if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
         /* Adjust long distance matching parameters */
         ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
@@ -1944,7 +1973,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     }
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+        size_t const blockSize = MIN(params->maxBlockSize, windowSize);
         size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
         size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
@@ -1962,7 +1991,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         size_t const neededSpace =
             ZSTD_estimateCCtxSize_usingCCtxParams_internal(
                 &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
-                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder);
+                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder, params->maxBlockSize);
         int resizeWorkspace;
 
         FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
@@ -2340,6 +2369,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
         params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
         params.ldmParams = srcCCtx->appliedParams.ldmParams;
         params.fParams = fParams;
+        params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
         ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
                                 /* loadedDictSize */ 0,
                                 ZSTDcrp_leaveDirty, zbuff);
@@ -3128,7 +3158,6 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
                                                                                     zc->appliedParams.useRowMatchFinder,
                                                                                     dictMode);
-            assert(zc->externalMatchCtx.mFinder == NULL);
             ms->ldmSeqStore = NULL;
             lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
         }
@@ -4474,7 +4503,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
     ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
     assert(!ZSTD_checkCParams(cParams));
-    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
+    return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
 }
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -5912,8 +5941,16 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
     params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+    params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
 
 #ifdef ZSTD_MULTITHREAD
+    /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
+    RETURN_ERROR_IF(
+        params.useExternalMatchFinder == 1 && params.nbWorkers >= 1,
+        parameter_combination_unsupported,
+        "External matchfinder isn't supported with nbWorkers >= 1"
+    );
+
     if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
         params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
     }
@@ -6103,6 +6140,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
         /* Reset to the original values. */
         cctx->requestedParams.inBufferMode = originalInBufferMode;
         cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
         FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
         if (result != 0) {  /* compression not completed, due to lack of output space */
             assert(oPos == dstCapacity);
@@ -6742,14 +6780,19 @@ void ZSTD_registerExternalMatchFinder(
     ZSTD_CCtx* zc, void* mState,
     ZSTD_externalMatchFinder_F* mFinder
 ) {
-    ZSTD_externalMatchCtx emctx = {
-        mState,
-        mFinder,
+    if (mFinder != NULL) {
+        ZSTD_externalMatchCtx emctx = {
+            mState,
+            mFinder,
 
-        /* seqBuffer is allocated later (from the cwskp) */
-        NULL, /* seqBuffer */
-        0 /* seqBufferCapacity */
-    };
-    zc->externalMatchCtx = emctx;
-    zc->requestedParams.useExternalMatchFinder = 1;
+            /* seqBuffer is allocated later (from the cwskp) */
+            NULL, /* seqBuffer */
+            0 /* seqBufferCapacity */
+        };
+        zc->externalMatchCtx = emctx;
+        zc->requestedParams.useExternalMatchFinder = 1;
+    } else {
+        ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+        zc->requestedParams.useExternalMatchFinder = 0;
+    }
 }
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index bb5e25083..3b888acfa 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -355,6 +355,9 @@ struct ZSTD_CCtx_params_s {
      * Users can't set this externally.
      * It is set internally in ZSTD_registerExternalMatchFinder(). */
     int useExternalMatchFinder;
+
+    /* Adjust the max block size*/
+    size_t maxBlockSize;
 };  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
 
 #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
diff --git a/lib/zstd.h b/lib/zstd.h
index f47c24676..a2e023452 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -479,6 +479,7 @@ typedef enum {
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
      * ZSTD_c_enableMatchFinderFallback
+     * ZSTD_c_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -499,7 +500,8 @@ typedef enum {
      ZSTD_c_experimentalParam14=1011,
      ZSTD_c_experimentalParam15=1012,
      ZSTD_c_experimentalParam16=1013,
-     ZSTD_c_experimentalParam17=1014
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015
 } ZSTD_cParameter;
 
 typedef struct {
@@ -1200,6 +1202,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 #define ZSTD_TARGETLENGTH_MIN     0   /* note : comparing this constant to an unsigned results in a tautological test */
 #define ZSTD_STRATEGY_MIN        ZSTD_fast
 #define ZSTD_STRATEGY_MAX        ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 #define ZSTD_OVERLAPLOG_MIN       0
@@ -2112,6 +2115,18 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  * documentation (below) before setting this parameter. */
 #define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
 
+/*  ZSTD_c_maxBlockSize
+ *  Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ *  The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ *  This parameter can be used to set an upper bound on the blocksize
+ *  that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ *  bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ *  compressBound() innacurate). Only currently meant to be used for testing.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
 /*! ZSTD_CCtx_getParameter() :
  *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  *  and store it into int* value.
@@ -2825,8 +2840,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* bloc
  * externalMatchState.
  *
  * *** LIMITATIONS ***
- * External matchfinders are compatible with all zstd compression APIs. There are
- * only two limitations.
+ * External matchfinders are compatible with all zstd compression APIs which respect
+ * advanced parameters. However, there are three limitations:
  *
  * First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
  * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
@@ -2848,7 +2863,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* bloc
  *     APIs, work with the external matchfinder, but the external matchfinder won't
  *     receive any history from the previous block. Each block is an independent chunk.
  *
- * Long-term, we plan to overcome both limitations. There is no technical blocker to
+ * Third, multi-threading within a single compression is not supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external matchfinder is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
  * overcoming them. It is purely a question of engineering effort.
  */
 
@@ -2871,6 +2890,17 @@ typedef size_t ZSTD_externalMatchFinder_F (
  * compressions. It will remain set until the user explicitly resets compression
  * parameters.
  *
+ * External matchfinder registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on
+ * compression APIs which respect advanced parameters, such as compress2() and
+ * compressStream(). Older compression APIs such as compressCCtx(), which predate
+ * the introduction of "advanced parameters", will ignore any external matchfinder
+ * setting.
+ *
+ * The external matchfinder can be "cleared" by registering a NULL external
+ * matchfinder function pointer. This removes all limitations described above in
+ * the "LIMITATIONS" section of the API docs.
+ *
  * The user is strongly encouraged to read the full API documentation (above)
  * before calling this function. */
 ZSTDLIB_STATIC_API void
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 285e401ef..7770dd2a5 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -387,12 +387,9 @@ BMK_benchMemAdvancedNoAlloc(
         RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
     }
 
-#if defined(UTIL_TIME_USES_C90_CLOCK)
-    if (adv->nbWorkers > 1) {
-        OUTPUTLEVEL(2, "Warning : time measurements restricted to C90 clock_t. \n")
-        OUTPUTLEVEL(2, "Warning : using C90 clock_t leads to incorrect measurements in multithreading mode. \n")
+    if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) {
+        OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n")
     }
-#endif
 
     /* Bench */
     {   U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
diff --git a/programs/dibio.c b/programs/dibio.c
index b21338cd8..26ebe5ca1 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -274,21 +274,20 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
     int n;
     memset(&fs, 0, sizeof(fs));
 
-    // We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX
+    /* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */
     assert( chunkSize <= SAMPLESIZE_MAX );
 
     for (n=0; n<nbFiles; n++) {
       S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
-      // TODO: is there a minimum sample size? What if the file is 1-byte?
+      /* TODO: is there a minimum sample size? What if the file is 1-byte? */
       if (fileSize == 0) {
         DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
         continue;
       }
 
       /* the case where we are breaking up files in sample chunks */
-      if (chunkSize > 0)
-      {
-        // TODO: is there a minimum sample size? Can we have a 1-byte sample?
+      if (chunkSize > 0) {
+        /* TODO: is there a minimum sample size? Can we have a 1-byte sample? */
         fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);
         fs.totalSizeToLoad += fileSize;
       }
diff --git a/programs/fileio_common.h b/programs/fileio_common.h
index 827a5a06b..55491b8e3 100644
--- a/programs/fileio_common.h
+++ b/programs/fileio_common.h
@@ -122,4 +122,4 @@ extern UTIL_time_t g_displayClock;
 #if defined (__cplusplus)
 }
 #endif
-#endif //ZSTD_FILEIO_COMMON_H
+#endif /* ZSTD_FILEIO_COMMON_H */
diff --git a/programs/timefn.c b/programs/timefn.c
index 08aa1cfcb..f941e57e6 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -12,7 +12,8 @@
 /* ===  Dependencies  === */
 
 #include "timefn.h"
-
+#include "platform.h" /* set _POSIX_C_SOURCE */
+#include <time.h>     /* CLOCK_MONOTONIC, TIME_UTC */
 
 /*-****************************************
 *  Time functions
@@ -20,12 +21,11 @@
 
 #if defined(_WIN32)   /* Windows */
 
+#include <windows.h>  /* LARGE_INTEGER */
 #include <stdlib.h>   /* abort */
 #include <stdio.h>    /* perror */
 
-UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
 {
     static LARGE_INTEGER ticksPerSecond;
     static int init = 0;
@@ -36,16 +36,20 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         }
         init = 1;
     }
-    return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+    {   UTIL_time_t r;
+        LARGE_INTEGER x;
+        QueryPerformanceCounter(&x);
+        r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
+        return r;
+    }
 }
 
 
-
 #elif defined(__APPLE__) && defined(__MACH__)
 
-UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+#include <mach/mach_time.h> /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */
 
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
 {
     static mach_timebase_info_data_t rate;
     static int init = 0;
@@ -53,12 +57,39 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         mach_timebase_info(&rate);
         init = 1;
     }
-    return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
+    {   UTIL_time_t r;
+        r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom;
+        return r;
+    }
+}
+
+/* POSIX.1-2001 (optional) */
+#elif defined(CLOCK_MONOTONIC)
+
+#include <stdlib.h>   /* abort */
+#include <stdio.h>    /* perror */
+
+UTIL_time_t UTIL_getTime(void)
+{
+    /* time must be initialized, othersize it may fail msan test.
+     * No good reason, likely a limitation of timespec_get() for some target */
+    struct timespec time = { 0, 0 };
+    if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
+        perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
+        abort();
+    }
+    {   UTIL_time_t r;
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+        return r;
+    }
 }
 
 
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
-   Android also lacks it but does define TIME_UTC. */
+/* C11 requires support of timespec_get().
+ * However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
+ * Double confirm timespec_get() support by checking the definition of TIME_UTC.
+ * However, some versions of Android manage to simultanously define TIME_UTC
+ * and lack timespec_get() support... */
 #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
     && defined(TIME_UTC) && !defined(__ANDROID__)
 
@@ -69,46 +100,38 @@ UTIL_time_t UTIL_getTime(void)
 {
     /* time must be initialized, othersize it may fail msan test.
      * No good reason, likely a limitation of timespec_get() for some target */
-    UTIL_time_t time = UTIL_TIME_INITIALIZER;
+    struct timespec time = { 0, 0 };
     if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
-        perror("timefn::timespec_get");
+        perror("timefn::timespec_get(TIME_UTC)");
         abort();
     }
-    return time;
-}
-
-static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t diff;
-    if (end.tv_nsec < begin.tv_nsec) {
-        diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
-        diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
-    } else {
-        diff.tv_sec = end.tv_sec - begin.tv_sec;
-        diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
+    {   UTIL_time_t r;
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+        return r;
     }
-    return diff;
-}
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    PTime nano = 0;
-    nano += 1000000000ULL * diff.tv_sec;
-    nano += diff.tv_nsec;
-    return nano;
 }
 
 
 #else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
 
-UTIL_time_t UTIL_getTime(void) { return clock(); }
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+UTIL_time_t UTIL_getTime(void)
+{
+    UTIL_time_t r;
+    r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC;
+    return r;
+}
+
+#define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
 
 #endif
 
 /* ==== Common functions, valid for all time API ==== */
 
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    return clockEnd.t - clockStart.t;
+}
+
 PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
 {
     return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
@@ -134,3 +157,12 @@ void UTIL_waitForNextTick(void)
         clockEnd = UTIL_getTime();
     } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
 }
+
+int UTIL_support_MT_measurements(void)
+{
+# if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED)
+    return 0;
+# else
+    return 1;
+# endif
+}
diff --git a/programs/timefn.h b/programs/timefn.h
index 3e4476a4e..b814ff8d8 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -18,7 +18,7 @@ extern "C" {
 
 
 /*-****************************************
-*  Local Types
+*  Types
 ******************************************/
 
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
@@ -32,40 +32,11 @@ extern "C" {
   typedef unsigned long long PTime;  /* does not support compilers without long long support */
 #endif
 
-
-
-/*-****************************************
-*  Time types (note: OS dependent)
-******************************************/
-#include <time.h>     /* TIME_UTC, then struct timespec and clock_t */
-
-#if defined(_WIN32)   /* Windows */
-
-    #include <windows.h>   /* LARGE_INTEGER */
-    typedef LARGE_INTEGER UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
-
-#elif defined(__APPLE__) && defined(__MACH__)
-
-    #include <mach/mach_time.h>
-    typedef PTime UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
-   Android also lacks it but does define TIME_UTC. */
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
-    && defined(TIME_UTC) && !defined(__ANDROID__)
-
-    typedef struct timespec UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { 0, 0 }
-
-#else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
-
-    #define UTIL_TIME_USES_C90_CLOCK
-    typedef clock_t UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-#endif
+/* UTIL_time_t contains a nanosecond time counter.
+ * The absolute value is not meaningful.
+ * It's only valid to compute the difference between 2 measurements. */
+typedef struct { PTime t; } UTIL_time_t;
+#define UTIL_TIME_INITIALIZER { 0 }
 
 
 /*-****************************************
@@ -73,16 +44,23 @@ extern "C" {
 ******************************************/
 
 UTIL_time_t UTIL_getTime(void);
+
+/* Timer resolution can be low on some platforms.
+ * To improve accuracy, it's recommended to wait for a new tick
+ * before starting benchmark measurements */
 void UTIL_waitForNextTick(void);
+/* tells if timefn will return correct time measurements
+ * in presence of multi-threaded workload.
+ * note : this is not the case if only C90 clock_t measurements are available */
+int UTIL_support_MT_measurements(void);
 
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
 
-#define SEC_TO_MICRO ((PTime)1000000)
 PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
 
-
+#define SEC_TO_MICRO ((PTime)1000000)  /* nb of microseconds in a second */
 
 
 #if defined (__cplusplus)
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 15e89d51f..86bbaf8bf 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -25,21 +25,13 @@
 #include "zdict.h"
 
 /* Direct access to internal compression functions is required */
-#include "zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
+#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
 
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
 
-#ifndef MIN
-    #define MIN(a, b) ((a) < (b) ? (a) : (b))
-#endif
-
-#ifndef MAX_PATH
-    #ifdef PATH_MAX
-        #define MAX_PATH PATH_MAX
-    #else
-        #define MAX_PATH 256
-    #endif
+#if !(defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */))
+# define inline  /* disable */
 #endif
 
 /*-************************************
@@ -71,6 +63,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
         }                                                                      \
     } while (0)
 
+
 /*-*******************************************************
 *  Random function
 *********************************************************/
@@ -176,6 +169,14 @@ const char* BLOCK_TYPES[] = {"raw", "rle", "compressed"};
 #define MIN_SEQ_LEN (3)
 #define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
 
+#ifndef MAX_PATH
+    #ifdef PATH_MAX
+        #define MAX_PATH PATH_MAX
+    #else
+        #define MAX_PATH 256
+    #endif
+#endif
+
 BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
 BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
 BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];
@@ -241,6 +242,10 @@ typedef enum {
   gt_block,      /* generate compressed blocks without block/frame headers */
 } genType_e;
 
+#ifndef MIN
+    #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
 /*-*******************************************************
 *  Global variables (set from command line)
 *********************************************************/
diff --git a/tests/external_matchfinder.h b/tests/external_matchfinder.h
index 041f73e4d..e4f7c95f0 100644
--- a/tests/external_matchfinder.h
+++ b/tests/external_matchfinder.h
@@ -32,4 +32,4 @@ size_t zstreamExternalMatchFinder(
   size_t windowSize
 );
 
-#endif // EXTERNAL_MATCHFINDER
+#endif /* EXTERNAL_MATCHFINDER */
diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c
index ec8fef488..0985f4cc8 100644
--- a/tests/fuzz/sequence_compression_api.c
+++ b/tests/fuzz/sequence_compression_api.c
@@ -22,6 +22,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <time.h>
+#include <sys/mman.h>
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
@@ -32,11 +33,17 @@ static void* literalsBuffer = NULL;
 static void* generatedSrc = NULL;
 static ZSTD_Sequence* generatedSequences = NULL;
 
+static void* dictBuffer = NULL;
+static ZSTD_CDict* cdict = NULL;
+static ZSTD_DDict* ddict = NULL;
+
 #define ZSTD_FUZZ_GENERATED_SRC_MAXSIZE (1 << 20) /* Allow up to 1MB generated data */
+#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 20) /* Fixed size 1MB literals buffer */
 #define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */
-#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << 18) /* Allow up to a 256KB dict */
-#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 18) /* Fixed size 256KB literals buffer */
+#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << ZSTD_WINDOWLOG_MAX_32) /* Allow up to 1 << ZSTD_WINDOWLOG_MAX_32 dictionary */
 #define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */
+#define ZSTD_FUZZ_DICT_FILE "sequence_fuzz_dictionary"
+
 
 /* Deterministic random number generator */
 #define FUZZ_RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
@@ -55,9 +62,9 @@ static uint32_t FUZZ_RDG_rand(uint32_t* src)
 /* Make a pseudorandom string - this simple function exists to avoid
  * taking a dependency on datagen.h to have RDG_genBuffer().
  */
-static char* generatePseudoRandomString(char* str, size_t size) {
+static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProducer_t* producer) {
     const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
-    uint32_t seed = 0;
+    uint32_t seed = FUZZ_dataProducer_uint32(producer);
     if (size) {
         for (size_t n = 0; n < size; n++) {
             int key = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
@@ -67,6 +74,26 @@ static char* generatePseudoRandomString(char* str, size_t size) {
     return str;
 }
 
+/*
+ * Create large dictionary file
+ */
+static void generateDictFile(size_t size, FUZZ_dataProducer_t* producer) {
+    char c;
+    FILE *dictFile;
+    const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
+    uint32_t seed = FUZZ_dataProducer_uint32(producer);
+
+    dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "w");
+    FUZZ_ASSERT(dictFile);
+
+    while (size) {
+      c = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
+      fputc(c, dictFile);
+      size--;
+    }
+    fclose(dictFile);
+}
+
 /* Returns size of source buffer */
 static size_t decodeSequences(void* dst, size_t nbSequences,
                               size_t literalsSize,
@@ -100,14 +127,14 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
             size_t j = 0;
             size_t k = 0;
             if (dictSize != 0) {
-                if (generatedSequences[i].offset > bytesWritten) {
-                    /* Offset goes into the dictionary */
-                    size_t offsetFromEndOfDict = generatedSequences[i].offset - bytesWritten;
-                    for (; k < offsetFromEndOfDict && k < matchLength; ++k) {
-                        op[k] = dictPtr[dictSize - offsetFromEndOfDict + k];
+                if (generatedSequences[i].offset > bytesWritten) { /* Offset goes into the dictionary */
+                    size_t dictOffset = generatedSequences[i].offset - bytesWritten;
+                    size_t matchInDict = MIN(matchLength, dictOffset);
+                    for (; k < matchInDict; ++k) {
+                        op[k] = dictPtr[dictSize - dictOffset + k];
                     }
-                    matchLength -= k;
-                    op += k;
+                    matchLength -= matchInDict;
+                    op += matchInDict;
                 }
             }
             for (; j < matchLength; ++j) {
@@ -138,9 +165,9 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
                                       size_t literalsSizeLimit, size_t dictSize,
                                       size_t windowLog, ZSTD_sequenceFormat_e mode)
 {
-    const uint32_t repCode = 0;  /* not used by sequence ingestion api */
-    const uint32_t windowSize = 1 << windowLog;
-    const uint32_t blockSizeMax = MIN(128 << 10, 1 << windowLog);
+    const uint32_t repCode = 0;  /* Not used by sequence ingestion api */
+    size_t windowSize = 1ULL << windowLog;
+    size_t blockSizeMax = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
     uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
     uint32_t bytesGenerated = 0;
     uint32_t nbSeqGenerated = 0;
@@ -148,12 +175,12 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
     uint32_t blockSize = 0;
 
     if (mode == ZSTD_sf_explicitBlockDelimiters) {
-        /* ensure that no sequence can be larger than one block */
+        /* Ensure that no sequence can be larger than one block */
         literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2);
         matchLengthMax = MIN(matchLengthMax, blockSizeMax/2);
     }
 
-    while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* extra room for explicit delimiters */
+    while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* Extra room for explicit delimiters */
          && bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE
          && !FUZZ_dataProducer_empty(producer)) {
         uint32_t matchLength;
@@ -210,38 +237,31 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
             }
             generatedSequences[nbSeqGenerated++] = seq;
             isFirstSequence = 0;
-    }   }
+        }
+    }
+
     if (mode == ZSTD_sf_explicitBlockDelimiters) {
         /* always end sequences with a block delimiter */
         const ZSTD_Sequence endBlock = {0, 0, 0, 0};
         assert(nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ);
         generatedSequences[nbSeqGenerated++] = endBlock;
     }
-
     return nbSeqGenerated;
 }
 
 static size_t roundTripTest(void* result, size_t resultCapacity,
                             void* compressed, size_t compressedCapacity,
                             const void* src, size_t srcSize,
-                            const void* dict, size_t dictSize,
                             const ZSTD_Sequence* seqs, size_t seqSize,
-                            int wLog, int cLevel, unsigned hasDict,
+                            unsigned hasDict,
                             ZSTD_sequenceFormat_e mode)
 {
     size_t cSize;
     size_t dSize;
 
-    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
     if (hasDict) {
-        FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize));
-        FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize));
+        FUZZ_ZASSERT(ZSTD_CCtx_refCDict(cctx, cdict));
+        FUZZ_ZASSERT(ZSTD_DCtx_refDDict(dctx, ddict));
     }
 
     cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity,
@@ -272,7 +292,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     size_t cBufSize;
     size_t generatedSrcSize;
     size_t nbSequences;
-    void* dictBuffer = NULL;
     size_t dictSize = 0;
     unsigned hasDict;
     unsigned wLog;
@@ -281,23 +300,66 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
 
     FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size);
     FUZZ_ASSERT(producer);
-    if (literalsBuffer == NULL) {
+
+    if (!cctx) {
+        cctx = ZSTD_createCCtx();
+        FUZZ_ASSERT(cctx);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+
+    /* Generate window log first so we don't generate offsets too large */
+    wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+    cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
+    mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
+
+    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach);
+
+    if (!literalsBuffer) {
         literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
         FUZZ_ASSERT(literalsBuffer);
-        literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
+        literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, producer);
     }
 
+    if (!dictBuffer) { /* Generate global dictionary buffer */
+        FILE* dictFile;
+        ZSTD_compressionParameters cParams;
+
+        /* Generate a large dictionary file and mmap to buffer */
+        generateDictFile(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, producer);
+        dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "r");
+        dictBuffer = mmap(NULL, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, PROT_READ, MAP_PRIVATE, fileno(dictFile), 0);
+        FUZZ_ASSERT(dictBuffer);
+        fclose(dictFile);
+
+        /* Create global cdict and ddict*/
+        cParams = ZSTD_getCParams(1, ZSTD_FUZZ_GENERATED_SRC_MAXSIZE, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
+        cParams.minMatch = ZSTD_MINMATCH_MIN;
+        cParams.hashLog = ZSTD_HASHLOG_MIN;
+        cParams.chainLog = ZSTD_CHAINLOG_MIN;
+
+        cdict = ZSTD_createCDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
+        ddict = ZSTD_createDDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
+        FUZZ_ASSERT(cdict);
+        FUZZ_ASSERT(ddict);
+    }
+
+    FUZZ_ASSERT(cdict);
+    FUZZ_ASSERT(ddict);
+
     hasDict = FUZZ_dataProducer_uint32Range(producer, 0, 1);
     if (hasDict) {
-        dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
-        dictBuffer = FUZZ_malloc(dictSize);
-        FUZZ_ASSERT(dictBuffer);
-        dictBuffer = generatePseudoRandomString(dictBuffer, dictSize);
+        dictSize = ZSTD_FUZZ_GENERATED_DICT_MAXSIZE;
     }
-    /* Generate window log first so we don't generate offsets too large */
-    wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32);
-    cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
-    mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
 
     if (!generatedSequences) {
         generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ);
@@ -305,8 +367,10 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     if (!generatedSrc) {
         generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE);
     }
+
     nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog, mode);
     generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize, mode);
+
     /* Note : in explicit block delimiters mode,
      * the fuzzer might generate a lot of small blocks.
      * In which case, the final compressed size might be > ZSTD_compressBound().
@@ -318,30 +382,17 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     rBufSize = generatedSrcSize;
     rBuf = FUZZ_malloc(rBufSize);
 
-    if (!cctx) {
-        cctx = ZSTD_createCCtx();
-        FUZZ_ASSERT(cctx);
-    }
-    if (!dctx) {
-        dctx = ZSTD_createDCtx();
-        FUZZ_ASSERT(dctx);
-    }
-
     {   const size_t result = roundTripTest(rBuf, rBufSize,
                                         cBuf, cBufSize,
                                         generatedSrc, generatedSrcSize,
-                                        dictBuffer, dictSize,
                                         generatedSequences, nbSequences,
-                                        (int)wLog, cLevel, hasDict, mode);
+                                        hasDict, mode);
         FUZZ_ASSERT(result <= generatedSrcSize);  /* can be 0 when no round-trip */
     }
 
     free(rBuf);
     free(cBuf);
     FUZZ_dataProducer_free(producer);
-    if (hasDict) {
-        free(dictBuffer);
-    }
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 6ad3d975e..2797ed726 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -115,6 +115,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
     setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
     setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
+    setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ce0bea573..4a091c897 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -25,7 +25,8 @@
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
 #include <string.h>       /* strcmp */
-#undef NDEBUG
+#include <time.h>         /* time(), time_t */
+#undef NDEBUG             /* always enable assert() */
 #include <assert.h>
 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "debug.h"        /* DEBUG_STATIC_ASSERT */
@@ -476,7 +477,7 @@ static void test_compressBound(unsigned tnb)
             CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
     }   }
 
-    // Ensure error if srcSize too big
+    /* Ensure error if srcSize too big */
     {   size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
         CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
         CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
@@ -489,7 +490,7 @@ static void test_decompressBound(unsigned tnb)
 {
     DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
 
-    // Simple compression, with size : should provide size;
+    /* Simple compression, with size : should provide size; */
     {   const char example[] = "abcd";
         char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
         size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
@@ -497,7 +498,7 @@ static void test_decompressBound(unsigned tnb)
         CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
     }
 
-    // Simple small compression without size : should provide 1 block size
+    /* Simple small compression without size : should provide 1 block size */
     {   char cBuffer[ZSTD_COMPRESSBOUND(0)];
         ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
         ZSTD_inBuffer in = { NULL, 0, 0 };
@@ -510,14 +511,14 @@ static void test_decompressBound(unsigned tnb)
         ZSTD_freeCCtx(cctx);
     }
 
-    // Attempt to overflow 32-bit intermediate multiplication result
-    // This requires dBound >= 4 GB, aka 2^32.
-    // This requires 2^32 / 2^17 = 2^15 blocks
-    // => create 2^15 blocks (can be empty, or just 1 byte).
+    /* Attempt to overflow 32-bit intermediate multiplication result
+     * This requires dBound >= 4 GB, aka 2^32.
+     * This requires 2^32 / 2^17 = 2^15 blocks
+     * => create 2^15 blocks (can be empty, or just 1 byte). */
     {   const char input[] = "a";
         size_t const nbBlocks = (1 << 15) + 1;
         size_t blockNb;
-        size_t const outCapacity = 1 << 18; // large margin
+        size_t const outCapacity = 1 << 18; /* large margin */
         char* const outBuffer = malloc (outCapacity);
         ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
         ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -1779,6 +1780,94 @@ static int basicUnitTests(U32 const seed, double compressibility)
                 if (!ZSTD_isError(r)) goto _output_error;
             }
             DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
+            {
+                // Test ZSTD_estimateCCtxSize_usingCCtxParams
+                {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3));
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+
+                // Test ZSTD_estimateCStreamSize_usingCCtxParams
+                  {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+            }
+            DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
+            {
+                // Test ZSTD_estimateCCtxSize_usingCCtxParams
+                {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t cctxSizeDefault;
+                    CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+                    cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+
+                // Test ZSTD_estimateCStreamSize_usingCCtxParams
+                  {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t cctxSizeDefault;
+                    CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+                    cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+            }
+            DISPLAYLEVEL(3, "OK \n");
         }
         free(staticCCtxBuffer);
         free(staticDCtxBuffer);
@@ -3535,7 +3624,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
     {
-        U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed)
+        U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
         U32 rand32 = FUZ_rand(&seed_copy);
         U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
         U32 lowbit_only_32 = 1;
@@ -3543,8 +3632,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         U32 highbit_only_32 = (U32)1 << 31;
         U64 highbit_only_64 = (U64)1 << 63;
         U32 i;
-        if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0
-        if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0
+        if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
+        if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
 
         /* Test ZSTD_countTrailingZeros32 */
         CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index 7117c1952..88b0578eb 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -29,8 +29,19 @@ test_dat_src = 'README.md'
 test_dat = 'test_dat'
 head = 'vdevel'
 dict_source = 'dict_source'
-dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
-dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+dict_globs = [
+    'programs/*.c',
+    'lib/common/*.c',
+    'lib/compress/*.c',
+    'lib/decompress/*.c',
+    'lib/dictBuilder/*.c',
+    'lib/legacy/*.c',
+    'programs/*.h',
+    'lib/common/*.h',
+    'lib/compress/*.h',
+    'lib/dictBuilder/*.h',
+    'lib/legacy/*.h'
+]
 
 
 def execute(command, print_output=False, print_error=True, param_shell=False):
@@ -85,6 +96,7 @@ def create_dict(tag, dict_source_path):
             result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
         if result == 0:
             print(dict_name + ' created')
+            assert os.path.isfile(dict_name)
         else:
             raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
     else:
@@ -103,12 +115,15 @@ def zstd(tag, args, input_file, output_file):
             print("Running: '{}', input={}, output={}" .format(
                 ' '.join(cmd), input_file, output_file
             ))
-            subprocess.check_call(cmd, stdin=i, stdout=o)
+            result = subprocess.run(cmd, stdin=i, stdout=o, stderr=subprocess.PIPE)
+            print("Stderr: {}".format(result.stderr.decode("ascii")))
+            result.check_returncode()
 
 
 def dict_compress_sample(tag, sample):
     dict_name = 'dict.' + tag
-    zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
+    verbose = ['-v', '-v', '-v']
+    zstd(tag, ['-D', dict_name, '-1'] + verbose, sample, sample + '_01_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
@@ -246,8 +261,12 @@ if __name__ == '__main__':
     # copy *.c and *.h to a temporary directory ("dict_source")
     if not os.path.isdir(dict_source_path):
         os.mkdir(dict_source_path)
-        print('cp ' + dict_files + ' ' + dict_source_path)
-        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+        for dict_glob in dict_globs:
+            files = glob.glob(dict_glob, root_dir=base_dir)
+            for file in files:
+                file = os.path.join(base_dir, file)
+                print("copying " + file + " to " + dict_source_path)
+                shutil.copy(file, dict_source_path)
 
     print('-----------------------------------------------')
     print('Compress test.dat by all released zstd')
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 10239d3f9..4a621692d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1846,15 +1846,11 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 
         CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
 
-        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
 
         /* Reference external matchfinder outside the test loop to
          * check that the reference is preserved across compressions */
-        ZSTD_registerExternalMatchFinder(
-            zc,
-            &externalMatchState,
-            zstreamExternalMatchFinder
-        );
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
 
         for (enableFallback = 0; enableFallback < 1; enableFallback++) {
             size_t testCaseId;
@@ -1916,16 +1912,160 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
         }
 
         /* Test that reset clears the external matchfinder */
-        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
         externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
         CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
         CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
 
+        /* Test that registering mFinder == NULL clears the external matchfinder */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
+        CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+        /* Test that external matchfinder doesn't interact with older APIs */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
+
+        /* Test that compression returns the correct error with LDM */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+
+#ifdef ZSTD_MULTITHREAD
+        /* Test that compression returns the correct error with nbWorkers > 0 */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+#endif
+
         free(dstBuf);
         free(checkBuf);
     }
     DISPLAYLEVEL(3, "OK \n");
 
+
+    /* Test maxBlockSize cctx param functionality */
+    DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+
+        /* Quick test to make sure maxBlockSize bounds are enforced */
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
+
+        /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
+        {
+            size_t srcSize = 2 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+            memset(src, 'x', srcSize);
+
+            /* maxBlockSize = 1KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = 3KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 - size2 == 4); /* We add another RLE block with header + character */
+            assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
+
+            /* maxBlockSize = 1KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = 3KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+
+            free(checkBuf);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+        /* Test maxBlockSize = 0 is valid */
+        {   size_t srcSize = 256 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+
+            /* maxBlockSize = 0 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+            free(checkBuf);
+        }
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);

From 1b65727e7451d6cff83b6a96364ffedfef251fb7 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 23 Jan 2023 07:59:02 -0800
Subject: [PATCH 146/197] fix nits and add new error code for invalid external
 sequences

---
 lib/common/error_private.c   |  1 +
 lib/compress/zstd_compress.c | 10 +++++-----
 lib/zstd_errors.h            |  1 +
 tests/zstreamtest.c          | 17 +++++++++++++++++
 4 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index fb4d70596..a9f01069a 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -53,6 +53,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
     case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
     case PREFIX(externalMatchFinder_failed): return "External matchfinder returned an error code";
+    case PREFIX(invalid_external_sequences): return "External sequences are not valid";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c7a573371..912cd11cd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -6166,10 +6166,10 @@ ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
      * window size. After output surpasses windowSize, we're limited to windowSize offsets again.
      */
     size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
-    RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), corruption_detected, "Offset too large!");
-    RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small");
+    size_t const matchLenLowerBound = (minMatch == 3 || useExternalMatchFinder) ? 3 : 4;
+    RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), invalid_external_sequences, "Offset too large!");
     /* Validate maxNbSeq is large enough for the given matchLength and minMatch */
-    RETURN_ERROR_IF(!useExternalMatchFinder && minMatch >= 4 && matchLength < 4, corruption_detected, "Matchlength too small for the minMatch");
+    RETURN_ERROR_IF(matchLength < matchLenLowerBound, invalid_external_sequences, "Matchlength too small for the minMatch");
     return 0;
 }
 
@@ -6226,7 +6226,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                                 cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
                                                 "Sequence validation failed");
         }
-        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, memory_allocation,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, invalid_external_sequences,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
@@ -6339,7 +6339,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
                                                    "Sequence validation failed");
         }
         DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
-        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, memory_allocation,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, invalid_external_sequences,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index bd6dbee5f..759e61ce7 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -94,6 +94,7 @@ typedef enum {
   ZSTD_error_dstBuffer_wrong     = 104,
   ZSTD_error_srcBuffer_wrong     = 105,
   ZSTD_error_externalMatchFinder_failed = 106,
+  ZSTD_error_invalid_external_sequences = 107,
   ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index cb5f7fea7..22086051b 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2097,6 +2097,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
                                    src, srcSize);
 
             CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
 
             ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
 
@@ -2110,6 +2111,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
                                    src, srcSize);
 
             CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
 
             free(sequences);
         }
@@ -2143,6 +2145,21 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
                                    src, srcSize);
 
             CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+            /* Test without sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
 
             free(sequences);
         }

From 815d1d4edaf8d69a73fdbc59dd6caf3c5f71cf43 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 23 Jan 2023 09:58:34 -0800
Subject: [PATCH 147/197] update external sequence error to fit error naming
 scheme

---
 lib/common/error_private.c   | 2 +-
 lib/compress/zstd_compress.c | 8 ++++----
 lib/zstd_errors.h            | 2 +-
 tests/zstreamtest.c          | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index a9f01069a..8b8e7a047 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -53,7 +53,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
     case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
     case PREFIX(externalMatchFinder_failed): return "External matchfinder returned an error code";
-    case PREFIX(invalid_external_sequences): return "External sequences are not valid";
+    case PREFIX(externalSequences_invalid): return "External sequences are not valid";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 912cd11cd..f74a0883e 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -6167,9 +6167,9 @@ ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
      */
     size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
     size_t const matchLenLowerBound = (minMatch == 3 || useExternalMatchFinder) ? 3 : 4;
-    RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), invalid_external_sequences, "Offset too large!");
+    RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
     /* Validate maxNbSeq is large enough for the given matchLength and minMatch */
-    RETURN_ERROR_IF(matchLength < matchLenLowerBound, invalid_external_sequences, "Matchlength too small for the minMatch");
+    RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch");
     return 0;
 }
 
@@ -6226,7 +6226,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                                 cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
                                                 "Sequence validation failed");
         }
-        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, invalid_external_sequences,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
@@ -6339,7 +6339,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
                                                    "Sequence validation failed");
         }
         DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
-        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, invalid_external_sequences,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 759e61ce7..0ad34fe2e 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -94,7 +94,7 @@ typedef enum {
   ZSTD_error_dstBuffer_wrong     = 104,
   ZSTD_error_srcBuffer_wrong     = 105,
   ZSTD_error_externalMatchFinder_failed = 106,
-  ZSTD_error_invalid_external_sequences = 107,
+  ZSTD_error_externalSequences_invalid = 107,
   ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 22086051b..3a3faa337 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2097,7 +2097,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
                                    src, srcSize);
 
             CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
-            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
 
             ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
 
@@ -2111,7 +2111,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
                                    src, srcSize);
 
             CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
-            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
 
             free(sequences);
         }
@@ -2145,7 +2145,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
                                    src, srcSize);
 
             CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
-            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
 
             ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
 
@@ -2159,7 +2159,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
                                    src, srcSize);
 
             CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
-            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_invalid_external_sequences, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
 
             free(sequences);
         }

From dc2b3e887607fed7545f3003ce755e918d75999a Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 20 Jan 2023 11:29:52 -0800
Subject: [PATCH 148/197] Fix -Wstringop-overflow warning

Backported from kernel patch [0].

I wasn't able to reproduce the warning locally, but could repro it in
the kernel.

[0] https://lore.kernel.org/lkml/20220330193352.GA119296@embeddedor/
---
 lib/decompress/huf_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 400a4e44d..3cafeba53 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -934,7 +934,7 @@ static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32
 
 static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
                            const sortedSymbol_t* sortedList,
-                           const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
+                           const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight,
                            const U32 nbBitsBaseline)
 {
     U32* const rankVal = rankValOrigin[0];

From 3bfd3be5fb0d11813a57c10bf947224b998e696c Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Mon, 23 Jan 2023 13:28:36 -0500
Subject: [PATCH 149/197] Fix ZSTD_estimate* and ZSTD_initCStream() docs

Fix the following documentation bugs:
* Note that `ZSTD_estimate*` functions are not compatible with the external matchfinder API
* Note that `ZSTD_estimateCStreamSize_usingCCtxParams()` is not compatible with `nbWorkers >= 1`
* Remove incorrect warning that the legacy streaming API is incompatible with advanced parameters and/or dictionary compression
* Note that `ZSTD_initCStream()` is incompatible with dictionary compression
* Warn that
---
 lib/zstd.h | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index a2e023452..d2c5d3ff5 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -799,8 +799,6 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
  * This following is a legacy streaming API, available since v1.0+ .
  * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
  * It is redundant, but remains fully supported.
- * Streaming in combination with advanced parameters and dictionary compression
- * can only be used through the new API.
  ******************************************************************************/
 
 /*!
@@ -809,6 +807,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
  *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ * to compress with a dictionary.
  */
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 /*!
@@ -1620,8 +1621,11 @@ ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
  *  and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
  *  Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
  *
- *  Note 2 : only single-threaded compression is supported.
+ *  Note : only single-threaded compression is supported.
  *  ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *
+ *  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the external matchfinder API at this time.
+ *  Size estimates assume that no external matchfinder is registered.
  */
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -1640,7 +1644,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
  *  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
  *         an internal ?Dict will be created, which additional size is not estimated here.
- *         In this case, get total size by adding ZSTD_estimate?DictSize */
+ *         In this case, get total size by adding ZSTD_estimate?DictSize 
+ *  Note 2 : only single-threaded compression is supported.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the external matchfinder API at this time.
+ *  Size estimates assume that no external matchfinder is registered.
+ */
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);

From 7fc00c18b8e7fc8644f46f5bc16587f5b7861cff Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 23 Jan 2023 10:42:09 -0800
Subject: [PATCH 150/197] calloc dictionary in sequence compression fuzzer
 rather than generating a random buffer

---
 .gitignore                            |  1 -
 tests/fuzz/sequence_compression_api.c | 37 +++++----------------------
 2 files changed, 6 insertions(+), 32 deletions(-)

diff --git a/.gitignore b/.gitignore
index cefdfe923..a136ea394 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,7 +26,6 @@ tmp*
 *.zstd
 dictionary.
 dictionary
-sequence_fuzz_dictionary
 NUL
 
 # Build artefacts
diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c
index 0985f4cc8..8925f197b 100644
--- a/tests/fuzz/sequence_compression_api.c
+++ b/tests/fuzz/sequence_compression_api.c
@@ -22,7 +22,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <time.h>
-#include <sys/mman.h>
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
@@ -74,26 +73,6 @@ static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProduce
     return str;
 }
 
-/*
- * Create large dictionary file
- */
-static void generateDictFile(size_t size, FUZZ_dataProducer_t* producer) {
-    char c;
-    FILE *dictFile;
-    const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
-    uint32_t seed = FUZZ_dataProducer_uint32(producer);
-
-    dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "w");
-    FUZZ_ASSERT(dictFile);
-
-    while (size) {
-      c = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
-      fputc(c, dictFile);
-      size--;
-    }
-    fclose(dictFile);
-}
-
 /* Returns size of source buffer */
 static size_t decodeSequences(void* dst, size_t nbSequences,
                               size_t literalsSize,
@@ -165,7 +144,7 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
                                       size_t literalsSizeLimit, size_t dictSize,
                                       size_t windowLog, ZSTD_sequenceFormat_e mode)
 {
-    const uint32_t repCode = 0;  /* Not used by sequence ingestion api */
+    const uint32_t repCode = 0;  /* not used by sequence ingestion api */
     size_t windowSize = 1ULL << windowLog;
     size_t blockSizeMax = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
     uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
@@ -175,12 +154,12 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
     uint32_t blockSize = 0;
 
     if (mode == ZSTD_sf_explicitBlockDelimiters) {
-        /* Ensure that no sequence can be larger than one block */
+        /* ensure that no sequence can be larger than one block */
         literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2);
         matchLengthMax = MIN(matchLengthMax, blockSizeMax/2);
     }
 
-    while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* Extra room for explicit delimiters */
+    while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* extra room for explicit delimiters */
          && bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE
          && !FUZZ_dataProducer_empty(producer)) {
         uint32_t matchLength;
@@ -331,17 +310,13 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     }
 
     if (!dictBuffer) { /* Generate global dictionary buffer */
-        FILE* dictFile;
         ZSTD_compressionParameters cParams;
 
-        /* Generate a large dictionary file and mmap to buffer */
-        generateDictFile(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, producer);
-        dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "r");
-        dictBuffer = mmap(NULL, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, PROT_READ, MAP_PRIVATE, fileno(dictFile), 0);
+        /* Generate a large dictionary buffer */
+        dictBuffer = calloc(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, 1);
         FUZZ_ASSERT(dictBuffer);
-        fclose(dictFile);
 
-        /* Create global cdict and ddict*/
+        /* Create global cdict and ddict */
         cParams = ZSTD_getCParams(1, ZSTD_FUZZ_GENERATED_SRC_MAXSIZE, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
         cParams.minMatch = ZSTD_MINMATCH_MIN;
         cParams.hashLog = ZSTD_HASHLOG_MIN;

From 7d600c628afcc59dd9768c28ffcd56b48a919f05 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 24 Jan 2023 06:40:40 -0800
Subject: [PATCH 151/197] fix bound check for
 ZSTD_copySequencesToSeqStoreNoBlockDelim()

---
 lib/compress/zstd_compress.c          |  3 ++-
 tests/fuzz/sequence_compression_api.c |  2 --
 tests/zstreamtest.c                   | 34 +++++++++++++++++++++++++++
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4a9281e3d..07e62bf5c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -6327,7 +6327,6 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
             /* Move to the next sequence */
             endPosInSequence -= currSeq.litLength + currSeq.matchLength;
             startPosInSequence = 0;
-            idx++;
         } else {
             /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
                does not reach the end of the match. So, we have to split the sequence */
@@ -6382,6 +6381,8 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
+        if (!finalMatchSplit)
+            idx++; /* Next Sequence */
     }
     DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
     assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c
index 8925f197b..9d3f0a18a 100644
--- a/tests/fuzz/sequence_compression_api.c
+++ b/tests/fuzz/sequence_compression_api.c
@@ -41,8 +41,6 @@ static ZSTD_DDict* ddict = NULL;
 #define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */
 #define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << ZSTD_WINDOWLOG_MAX_32) /* Allow up to 1 << ZSTD_WINDOWLOG_MAX_32 dictionary */
 #define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */
-#define ZSTD_FUZZ_DICT_FILE "sequence_fuzz_dictionary"
-
 
 /* Deterministic random number generator */
 #define FUZZ_RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index b79432049..a44f8004a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2139,6 +2139,40 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 
         ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
 
+
+        /* Test with no block delim */
+        {
+            size_t srcSize = 4;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 1;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+            void* const checkBuf = malloc(srcSize);
+
+            memset(src, 'x', srcSize);
+
+            sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
+
+            /* Test with sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(ZSTD_isError(cSize), "Should not throw an error");
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst, cSize));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            free(sequences);
+            free(checkBuf);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
         { /* Test case with two additional sequences */
             size_t srcSize = 19;
             void* const src = CNBuffer;

From 1d636b4ba0bd105eaf7c2427aa6b4b147bdd3035 Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <yoniko@gmail.com>
Date: Tue, 24 Jan 2023 12:54:43 -0800
Subject: [PATCH 152/197] Bug fix redzones by unpoisoning only the intended
 buffer and not the followup redzone.

---
 lib/compress/zstd_cwksp.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 70a359a27..97676693b 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -348,7 +348,9 @@ ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase
     if (alloc) {
         alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
         if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
-            __asan_unpoison_memory_region(alloc, bytes);
+            /* We need to keep the redzone poisoned while unpoisoning the bytes that
+             * are actually allocated. */
+            __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE);
         }
     }
 #endif

From 63042f1f11b72da52f137d46ad9ffca772a9ac1c Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Tue, 24 Jan 2023 15:53:59 -0800
Subject: [PATCH 153/197] fix 32bit build errors in zstd seekable

---
 contrib/seekable_format/zstdseek_decompress.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index 7128b9d22..fbb2d4fe3 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -517,9 +517,9 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
             size_t forwardProgress;
             if (zs->decompressedOffset < offset) {
                 /* dummy decompressions until we get to the target offset */
-                outTmp = (ZSTD_outBuffer){zs->outBuff, MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset), 0};
+                outTmp = (ZSTD_outBuffer){zs->outBuff, (size_t) (MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset)), 0};
             } else {
-                outTmp = (ZSTD_outBuffer){dst, len, zs->decompressedOffset - offset};
+                outTmp = (ZSTD_outBuffer){dst, len, (size_t) (zs->decompressedOffset - offset)};
             }
 
             prevOutPos = outTmp.pos;

From db18a62f8994a091a746d64001b7c767fae64545 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 25 Jan 2023 13:07:53 -0800
Subject: [PATCH 154/197] Provide more accurate error codes for busy-loop
 scenarios

fixes #3454
---
 lib/common/error_private.c       | 2 ++
 lib/decompress/zstd_decompress.c | 6 +++---
 lib/zstd_errors.h                | 2 ++
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 8b8e7a047..0cff6b80b 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -47,6 +47,8 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
     case PREFIX(srcSize_wrong): return "Src size is incorrect";
     case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
+    case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
+    case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
         /* following error codes are not stable and may be removed or changed in a future version */
     case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
     case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 7e6ecd7df..52f28f57b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2225,7 +2225,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 int const isSkipFrame = ZSTD_isSkipFrame(zds);
                 size_t loadedSize;
                 /* At this point we shouldn't be decompressing a block that we can stream. */
-                assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
+                assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)));
                 if (isSkipFrame) {
                     loadedSize = MIN(toLoad, (size_t)(iend-ip));
                 } else {
@@ -2285,8 +2285,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     if ((ip==istart) && (op==ostart)) {  /* no forward progress */
         zds->noForwardProgress ++;
         if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
-            RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
-            RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
+            RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, "");
+            RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, "");
             assert(0);
         }
     } else {
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 0ad34fe2e..6a66bedcb 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -88,6 +88,8 @@ typedef enum {
   ZSTD_error_dstSize_tooSmall = 70,
   ZSTD_error_srcSize_wrong    = 72,
   ZSTD_error_dstBuffer_null   = 74,
+  ZSTD_error_noForwardProgress_destFull = 80,
+  ZSTD_error_noForwardProgress_inputEmpty = 82,
   /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
   ZSTD_error_frameIndex_tooLarge = 100,
   ZSTD_error_seekableIO          = 102,

From 8957fef554e844ef724022075ffdf740464aa515 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 13 Jan 2023 16:34:52 -0800
Subject: [PATCH 155/197] [huf] Add generic C versions of the fast decoding
 loops

Add generic C versions of the fast decoding loops to serve architectures
that don't have an assembly implementation. Also allow selecting the C
decoding loop over the assembly decoding loop through a zstd
decompression parameter `ZSTD_d_disableHuffmanAssembly`.

I benchmarked on my Intel i9-9900K and my Macbook Air with an M1 processor.
The benchmark command forces zstd to compress without any matches, using
only literals compression, and measures only Huffman decompression speed:

```
zstd -b1e1 --compress-literals --zstd=tlen=131072 silesia.tar
```

The new fast decoding loops outperform the previous implementation uniformly,
but don't beat the x86-64 assembly. Additionally, the fast C decoding loops suffer
from the same stability problems that we've seen in the past, where the assembly
version doesn't. So even though clang gets close to assembly on x86-64, it still
has stability issues.

| Arch    | Function       | Compiler     | Default (MB/s) | Assembly (MB/s) | Fast (MB/s) |
|---------|----------------|--------------|----------------|-----------------|-------------|
| x86-64  | decompress 4X1 | gcc-12.2.0   |         1029.6 |          1308.1 |      1208.1 |
| x86-64  | decompress 4X1 | clang-14.0.6 |         1019.3 |          1305.6 |      1276.3 |
| x86-64  | decompress 4X2 | gcc-12.2.0   |         1348.5 |          1657.0 |      1374.1 |
| x86-64  | decompress 4X2 | clang-14.0.6 |         1027.6 |          1659.9 |      1468.1 |
| aarch64 | decompress 4X1 | clang-12.0.5 |         1081.0 |             N/A |      1234.9 |
| aarch64 | decompress 4X2 | clang-12.0.5 |         1270.0 |             N/A |      1516.6 |
---
 lib/common/entropy_common.c               |   8 +-
 lib/common/huf.h                          |   7 +-
 lib/common/portability_macros.h           |   8 +
 lib/decompress/huf_decompress.c           | 425 ++++++++++++++++++----
 lib/decompress/huf_decompress_amd64.S     |  26 +-
 lib/decompress/zstd_decompress.c          |  13 +
 lib/decompress/zstd_decompress_block.c    |   4 +-
 lib/decompress/zstd_decompress_internal.h |   1 +
 lib/zstd.h                                |  15 +-
 tests/fuzz/huf_decompress.c               |   3 +-
 tests/fuzz/huf_round_trip.c               |   3 +-
 tests/fuzzer.c                            |  11 +
 tests/zstreamtest.c                       |  15 +-
 13 files changed, 448 insertions(+), 91 deletions(-)

diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index e40921966..e2173afb0 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -236,7 +236,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      const void* src, size_t srcSize)
 {
     U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
-    return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
+    return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
 }
 
 FORCE_INLINE_TEMPLATE size_t
@@ -328,13 +328,13 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      U32* nbSymbolsPtr, U32* tableLogPtr,
                      const void* src, size_t srcSize,
                      void* workSpace, size_t wkspSize,
-                     int bmi2)
+                     int flags)
 {
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
         return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
     }
 #endif
-    (void)bmi2;
+    (void)flags;
     return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
 }
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 5ab79db7a..73d1ee565 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -107,7 +107,12 @@ typedef enum {
      * If set: Don't use assembly implementations
      * If unset: Allow using assembly implementations
      */
-    HUF_flags_disableAsm = (1 << 4)
+    HUF_flags_disableAsm = (1 << 4),
+    /**
+     * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
+     * If unset: Use the fast decoding loop when possible.
+     */
+    HUF_flags_disableFast = (1 << 5)
 } HUF_flags_e;
 
 
diff --git a/lib/common/portability_macros.h b/lib/common/portability_macros.h
index bf3e2b7b1..8fd6ea82d 100644
--- a/lib/common/portability_macros.h
+++ b/lib/common/portability_macros.h
@@ -137,12 +137,20 @@
 /*
  * For x86 ELF targets, add .note.gnu.property section for Intel CET in
  * assembly sources when CET is enabled.
+ *
+ * Additionally, any function that may be called indirectly must begin
+ * with ZSTD_CET_ENDBRANCH.
  */
 #if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
     && defined(__has_include)
 # if __has_include(<cet.h>)
 #  include <cet.h>
+#  define ZSTD_CET_ENDBRANCH _CET_ENDBR
 # endif
 #endif
 
+#ifndef ZSTD_CET_ENDBRANCH
+# define ZSTD_CET_ENDBRANCH
+#endif
+
 #endif /* ZSTD_PORTABILITY_MACROS_H */
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 3cafeba53..0a29f03d4 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -43,10 +43,14 @@
 #error "Cannot force the use of the X1 and X2 decoders at the same time!"
 #endif
 
-#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2
-# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
+/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is
+ * supported at runtime, so we can add the BMI2 target attribute.
+ * When it is disabled, we will still get BMI2 if it is enabled statically.
+ */
+#if DYNAMIC_BMI2
+# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
 #else
-# define HUF_ASM_X86_64_BMI2_ATTRS
+# define HUF_FAST_BMI2_ATTRS
 #endif
 
 #ifdef __cplusplus
@@ -56,7 +60,7 @@
 #endif
 #define HUF_ASM_DECL HUF_EXTERN_C
 
-#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
+#if DYNAMIC_BMI2
 # define HUF_NEED_BMI2_FUNCTION 1
 #else
 # define HUF_NEED_BMI2_FUNCTION 0
@@ -78,6 +82,11 @@
 /* **************************************************************
 *  BMI2 Variant Wrappers
 ****************************************************************/
+typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize,
+                                              const void *cSrc,
+                                              size_t cSrcSize,
+                                              const HUF_DTable *DTable);
+
 #if DYNAMIC_BMI2
 
 #define HUF_DGEN(fn)                                                        \
@@ -132,15 +141,28 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
     return dtd;
 }
 
-#if ZSTD_ENABLE_ASM_X86_64_BMI2
-
-static size_t HUF_initDStream(BYTE const* ip) {
+static size_t HUF_initFastDStream(BYTE const* ip) {
     BYTE const lastByte = ip[7];
     size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
     size_t const value = MEM_readLEST(ip) | 1;
     assert(bitsConsumed <= 8);
+    assert(sizeof(size_t) == 8);
     return value << bitsConsumed;
 }
+
+
+/**
+ * The input/output arguments to the Huffman fast decoding loop:
+ *
+ * ip [in/out] - The input pointers, must be updated to reflect what is consumed.
+ * op [in/out] - The output pointers, must be updated to reflect what is written.
+ * bits [in/out] - The bitstream containers, must be updated to reflect the current state.
+ * dt [in] - The decoding table.
+ * ilimit [in] - The input limit, stop when any input pointer is below ilimit.
+ * oend [in] - The end of the output stream. op[3] must not cross oend.
+ * iend [in] - The end of each input stream. ip[i] may cross iend[i],
+ *             as long as it is above ilimit, but that indicates corruption.
+ */
 typedef struct {
     BYTE const* ip[4];
     BYTE* op[4];
@@ -149,15 +171,17 @@ typedef struct {
     BYTE const* ilimit;
     BYTE* oend;
     BYTE const* iend[4];
-} HUF_DecompressAsmArgs;
+} HUF_DecompressFastArgs;
+
+typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*);
 
 /**
- * Initializes args for the asm decoding loop.
- * @returns 0 on success
- *          1 if the fallback implementation should be used.
+ * Initializes args for the fast decoding loop.
+ * @returns 1 on success
+ *          0 if the fallback implementation should be used.
  *          Or an error code on failure.
  */
-static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
+static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
 {
     void const* dt = DTable + 1;
     U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
@@ -166,9 +190,11 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
 
     BYTE* const oend = (BYTE*)dst + dstSize;
 
-    /* The following condition is false on x32 platform,
-     * but HUF_asm is not compatible with this ABI */
-    if (!(MEM_isLittleEndian() && !MEM_32bits())) return 1;
+    /* The fast decoding loop assumes 64-bit little-endian.
+     * This condition is false on x32.
+     */
+    if (!MEM_isLittleEndian() || MEM_32bits())
+        return 0;
 
     /* strict minimum : jump table + 1 byte per stream */
     if (srcSize < 10)
@@ -179,7 +205,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
      * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
      */
     if (dtLog != HUF_DECODER_FAST_TABLELOG)
-        return 1;
+        return 0;
 
     /* Read the jump table. */
     {
@@ -193,13 +219,13 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
         args->iend[2] = args->iend[1] + length2;
         args->iend[3] = args->iend[2] + length3;
 
-        /* HUF_initDStream() requires this, and this small of an input
+        /* HUF_initFastDStream() requires this, and this small of an input
          * won't benefit from the ASM loop anyways.
          * length1 must be >= 16 so that ip[0] >= ilimit before the loop
          * starts.
          */
         if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8)
-            return 1;
+            return 0;
         if (length4 > srcSize) return ERROR(corruption_detected);   /* overflow */
     }
     /* ip[] contains the position that is currently loaded into bits[]. */
@@ -216,7 +242,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
 
     /* No point to call the ASM loop for tiny outputs. */
     if (args->op[3] >= oend)
-        return 1;
+        return 0;
 
     /* bits[] is the bit container.
         * It is read from the MSB down to the LSB.
@@ -225,10 +251,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
         * set, so that CountTrailingZeros(bits[]) can be used
         * to count how many bits we've consumed.
         */
-    args->bits[0] = HUF_initDStream(args->ip[0]);
-    args->bits[1] = HUF_initDStream(args->ip[1]);
-    args->bits[2] = HUF_initDStream(args->ip[2]);
-    args->bits[3] = HUF_initDStream(args->ip[3]);
+    args->bits[0] = HUF_initFastDStream(args->ip[0]);
+    args->bits[1] = HUF_initFastDStream(args->ip[1]);
+    args->bits[2] = HUF_initFastDStream(args->ip[2]);
+    args->bits[3] = HUF_initFastDStream(args->ip[3]);
 
     /* If ip[] >= ilimit, it is guaranteed to be safe to
         * reload bits[]. It may be beyond its section, but is
@@ -239,10 +265,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
     args->oend = oend;
     args->dt = dt;
 
-    return 0;
+    return 1;
 }
 
-static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd)
+static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd)
 {
     /* Validate that we haven't overwritten. */
     if (args->op[stream] > segmentEnd)
@@ -257,7 +283,7 @@ static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs
 
     /* Construct the BIT_DStream_t. */
     assert(sizeof(size_t) == 8);
-    bit->bitContainer = MEM_readLE64(args->ip[stream]);
+    bit->bitContainer = MEM_readLEST(args->ip[stream]);
     bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]);
     bit->start = (const char*)args->iend[0];
     bit->limitPtr = bit->start + sizeof(size_t);
@@ -265,7 +291,6 @@ static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs
 
     return 0;
 }
-#endif
 
 
 #ifndef HUF_FORCE_DECOMPRESS_X2
@@ -655,27 +680,132 @@ size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize,
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
-HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
+HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
 
-static HUF_ASM_X86_64_BMI2_ATTRS
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    U16 const* const dtable = (U16 const*)args->dt;
+    BYTE* const oend = args->oend;
+    BYTE const* const ilimit = args->ilimit;
+
+    /* Copy the arguments to local variables */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+        int symbol;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= (stream == 3 ? oend : op[stream + 1]));
+            assert(ip[stream] >= ilimit);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each iteration produces 5 output symbols per stream */
+            size_t const oiters = (size_t)(oend - op[3]) / 5;
+            /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes
+             * per stream.
+             */
+            size_t const iiters = (size_t)(ip[0] - ilimit) / 7;
+            /* We can safely run iters iterations before running bounds checks */
+            size_t const iters = MIN(oiters, iiters);
+            size_t const symbols = iters * 5;
+
+            /* We can simply check that op[3] < olimit, instead of checking all
+             * of our bounds, since we can't hit the other bounds until we've run
+             * iters iterations, which only happens when op[3] == olimit.
+             */
+            olimit = op[3] + symbols;
+
+            /* Exit fast decoding loop once we get close to the end. */
+            if (op[3] + 20 > olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    break;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+        do {
+            /* Decode 5 symbols in each of the 4 streams */
+            for (symbol = 0; symbol < 5; ++symbol) {
+                for (stream = 0; stream < 4; ++stream) {
+                    int const index = (int)(bits[stream] >> 53);
+                    int const entry = (int)dtable[index];
+                    bits[stream] <<= (entry & 63);
+                    op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF);
+                }
+            }
+            /* Reload the bitstreams */
+            for (stream = 0; stream < 4; ++stream) {
+                int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+                int const nbBits = ctz & 7;
+                int const nbBytes = ctz >> 3;
+                op[stream] += 5;
+                ip[stream] -= nbBytes;
+                bits[stream] = MEM_read64(ip[stream]) | 1;
+                bits[stream] <<= nbBits;
+            }
+        } while (op[3] < olimit);
+    }
+
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+/**
+ * @returns @p dstSize on success (>= 6)
+ *          0 if the fallback implementation should be used
+ *          An error if an error occurred
+ */
+static HUF_FAST_BMI2_ATTRS
 size_t
-HUF_decompress4X1_usingDTable_internal_bmi2_asm(
+HUF_decompress4X1_usingDTable_internal_fast(
           void* dst,  size_t dstSize,
     const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn)
 {
     void const* dt = DTable + 1;
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
-    HUF_DecompressAsmArgs args;
-    {   size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
-        FORWARD_IF_ERROR(ret, "Failed to init asm args");
-        if (ret != 0)
-            return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+    HUF_DecompressFastArgs args;
+    {   size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
+        if (ret == 0)
+            return 0;
     }
 
     assert(args.ip[0] >= args.ilimit);
-    HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args);
+    loopFn(&args);
 
     /* Our loop guarantees that ip[] >= ilimit and that we haven't
     * overwritten any op[].
@@ -705,37 +835,43 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     }
 
     /* decoded size */
+    assert(dstSize != 0);
     return dstSize;
 }
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
-
-typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
-                                               const void *cSrc,
-                                               size_t cSrcSize,
-                                               const HUF_DTable *DTable);
 
 HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
 
 static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop;
+
 #if DYNAMIC_BMI2
     if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2;
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        if (!(flags & HUF_flags_disableAsm))
-            return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+        }
 # endif
-        return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
     }
-#else
-    (void)flags;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    if (!(flags & HUF_flags_disableAsm))
-        return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+    }
 #endif
-    return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+
+    if (!(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
 static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
@@ -1322,26 +1458,167 @@ size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize,
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
-HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
+HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
 
-static HUF_ASM_X86_64_BMI2_ATTRS size_t
-HUF_decompress4X2_usingDTable_internal_bmi2_asm(
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    BYTE* oend[4];
+    HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt;
+    BYTE const* const ilimit = args->ilimit;
+
+    /* Copy the arguments to local registers. */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    oend[0] = op[1];
+    oend[1] = op[2];
+    oend[2] = op[3];
+    oend[3] = args->oend;
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+        int symbol;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= oend[stream]);
+            assert(ip[stream] >= ilimit);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each loop does 5 table lookups for each of the 4 streams.
+             * Each table lookup consumes up to 11 bits of input, and produces
+             * up to 2 bytes of output.
+             */
+            /* We can consume up to 7 bytes of input per iteration per stream.
+             * We also know that each input pointer is >= ip[0]. So we can run
+             * iters loops before running out of input.
+             */
+            size_t iters = (size_t)(ip[0] - ilimit) / 7;
+            /* Each iteration can produce up to 10 bytes of output per stream.
+             * Each output stream my advance at different rates. So take the
+             * minimum number of safe iterations among all the output streams.
+             */
+            for (stream = 0; stream < 4; ++stream) {
+                size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10;
+                iters = MIN(iters, oiters);
+            }
+
+            /* Each iteration produces at least 5 output symbols. So until
+             * op[3] crosses olimit, we know we haven't executed iters
+             * iterations yet. This saves us maintaining an iters counter,
+             * at the expense of computing the remaining # of iterations
+             * more frequently.
+             */
+            olimit = op[3] + (iters * 5);
+
+            /* Exit the fast decoding loop if we are too close to the end. */
+            if (op[3] + 10 > olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    break;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+        do {
+            /* Do 5 table lookups for each of the first 3 streams */
+            for (symbol = 0; symbol < 5; ++symbol) {
+                for (stream = 0; stream < 3; ++stream) {
+                    int const index = (int)(bits[stream] >> 53);
+                    HUF_DEltX2 const entry = dtable[index];
+                    MEM_write16(op[stream], entry.sequence);
+                    bits[stream] <<= (entry.nbBits);
+                    op[stream] += (entry.length);
+                }
+            }
+            /* Do 1 table lookup from the final stream */
+            {
+                int const index = (int)(bits[3] >> 53);
+                HUF_DEltX2 const entry = dtable[index];
+                MEM_write16(op[3], entry.sequence);
+                bits[3] <<= (entry.nbBits);
+                op[3] += (entry.length);
+            }
+            /* Do 4 table lookups from the final stream & reload bitstreams */
+            for (stream = 0; stream < 4; ++stream) {
+                /* Do a table lookup from the final stream.
+                 * This is interleaved with the reloading to reduce register
+                 * pressure. This shouldn't be necessary, but compilers can
+                 * struggle with codegen with high register pressure.
+                 */
+                {
+                    int const index = (int)(bits[3] >> 53);
+                    HUF_DEltX2 const entry = dtable[index];
+                    MEM_write16(op[3], entry.sequence);
+                    bits[3] <<= (entry.nbBits);
+                    op[3] += (entry.length);
+                }
+                /* Reload the bistreams. The final bitstream must be reloaded
+                 * after the 5th symbol was decoded.
+                 */
+                {
+                    int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+                    int const nbBits = ctz & 7;
+                    int const nbBytes = ctz >> 3;
+                    ip[stream] -= nbBytes;
+                    bits[stream] = MEM_read64(ip[stream]) | 1;
+                    bits[stream] <<= nbBits;
+                }
+            }
+        } while (op[3] < olimit);
+    }
+
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+
+static HUF_FAST_BMI2_ATTRS size_t
+HUF_decompress4X2_usingDTable_internal_fast(
           void* dst,  size_t dstSize,
     const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable) {
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn) {
     void const* dt = DTable + 1;
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
-    HUF_DecompressAsmArgs args;
+    HUF_DecompressFastArgs args;
     {
-        size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
         FORWARD_IF_ERROR(ret, "Failed to init asm args");
-        if (ret != 0)
-            return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (ret == 0)
+            return 0;
     }
 
     assert(args.ip[0] >= args.ilimit);
-    HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args);
+    loopFn(&args);
 
     /* note : op4 already verified within main loop */
     assert(args.ip[0] >= iend);
@@ -1372,28 +1649,38 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm(
     /* decoded size */
     return dstSize;
 }
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
 
 static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop;
+
 #if DYNAMIC_BMI2
     if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2;
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        if (!(flags & HUF_flags_disableAsm))
-            return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+        }
 # endif
-        return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
     }
-#else
-    (void)flags;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    if (!(flags & HUF_flags_disableAsm))
-        return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+    }
 #endif
-    return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+
+    if (!(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
 HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
diff --git a/lib/decompress/huf_decompress_amd64.S b/lib/decompress/huf_decompress_amd64.S
index 8d7c7a644..671624fe3 100644
--- a/lib/decompress/huf_decompress_amd64.S
+++ b/lib/decompress/huf_decompress_amd64.S
@@ -30,14 +30,14 @@
  * TODO: Support Windows calling convention.
  */
 
-ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
-ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
-ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
-ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
-.global HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
-.global HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
-.global _HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
-.global _HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
+ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
+ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop)
+ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop)
+ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
+.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop
+.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop
+.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop
+.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop
 .text
 
 /* Sets up register mappings for clarity.
@@ -95,8 +95,9 @@ ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
 /* Define both _HUF_* & HUF_* symbols because MacOS
  * C symbols are prefixed with '_' & Linux symbols aren't.
  */
-_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
-HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
+_HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
+HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
+    ZSTD_CET_ENDBRANCH
     /* Save all registers - even if they are callee saved for simplicity. */
     push %rax
     push %rbx
@@ -350,8 +351,9 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
     pop %rax
     ret
 
-_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
-HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
+_HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
+HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
+    ZSTD_CET_ENDBRANCH
     /* Save all registers - even if they are callee saved for simplicity. */
     push %rax
     push %rbx
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 7e6ecd7df..d7e52a2e2 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -243,6 +243,7 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
     dctx->outBufferMode = ZSTD_bm_buffered;
     dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
     dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
+    dctx->disableHufAsm = 0;
 }
 
 static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
@@ -1811,6 +1812,11 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
             bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
             bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
             return bounds;
+        case ZSTD_d_disableHuffmanAssembly:
+            bounds.lowerBound = 0;
+            bounds.upperBound = 1;
+            return bounds;
+
         default:;
     }
     bounds.error = ERROR(parameter_unsupported);
@@ -1851,6 +1857,9 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value
         case ZSTD_d_refMultipleDDicts:
             *value = (int)dctx->refMultipleDDicts;
             return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            *value = (int)dctx->disableHufAsm;
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
@@ -1884,6 +1893,10 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             }
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
             return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value);
+            dctx->disableHufAsm = value != 0;
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index fbe4bf674..95a5e109b 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -141,7 +141,9 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 U32 const lhc = MEM_readLE32(istart);
                 size_t hufSuccess;
                 size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
-                int const flags = ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0;
+                int const flags = 0
+                    | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
+                    | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
                 switch(lhlCode)
                 {
                 case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 32685f2d3..c2ec5d9fb 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -165,6 +165,7 @@ struct ZSTD_DCtx_s
     ZSTD_dictUses_e dictUses;
     ZSTD_DDictHashSet* ddictSet;                    /* Hash set for multiple ddicts */
     ZSTD_refMultipleDDicts_e refMultipleDDicts;     /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
+    int disableHufAsm;
 
     /* streaming */
     ZSTD_dStreamStage streamStage;
diff --git a/lib/zstd.h b/lib/zstd.h
index d2c5d3ff5..d9e26ee13 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -614,13 +614,15 @@ typedef enum {
      * ZSTD_d_stableOutBuffer
      * ZSTD_d_forceIgnoreChecksum
      * ZSTD_d_refMultipleDDicts
+     * ZSTD_d_disableHuffmanAssembly
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
      ZSTD_d_experimentalParam2=1001,
      ZSTD_d_experimentalParam3=1002,
-     ZSTD_d_experimentalParam4=1003
+     ZSTD_d_experimentalParam4=1003,
+     ZSTD_d_experimentalParam5=1004
 
 } ZSTD_dParameter;
 
@@ -2345,6 +2347,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
  */
 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
 
+/* ZSTD_d_disableHuffmanAssembly
+ * Set to 1 to disable the Huffman assembly implementation.
+ * The default value is 0, which allows zstd to use the Huffman assembly
+ * implementation if available.
+ *
+ * This parameter can be used to disable Huffman assembly at runtime.
+ * If you want to disable it at compile time you can define the macro
+ * ZSTD_DISABLE_ASM.
+ */
+#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
+
 
 /*! ZSTD_DCtx_setFormat() :
  *  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
diff --git a/tests/fuzz/huf_decompress.c b/tests/fuzz/huf_decompress.c
index a9b3540ab..fcd4b1a3b 100644
--- a/tests/fuzz/huf_decompress.c
+++ b/tests/fuzz/huf_decompress.c
@@ -33,7 +33,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
         | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
         | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
-        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0);
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableFast : 0);
     /* Select a random cBufSize - it may be too small */
     size_t const dBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 8 * size + 500);
     size_t const maxTableLog = FUZZ_dataProducer_uint32Range(producer, 1, HUF_TABLELOG_MAX);
diff --git a/tests/fuzz/huf_round_trip.c b/tests/fuzz/huf_round_trip.c
index 9967c57bf..4d0f8de23 100644
--- a/tests/fuzz/huf_round_trip.c
+++ b/tests/fuzz/huf_round_trip.c
@@ -49,7 +49,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
         | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
         | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
-        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0);
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableFast : 0);
     /* Select a random cBufSize - it may be too small */
     size_t const cBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 4 * size);
     /* Select a random tableLog - we'll adjust it up later */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index bd3ca0e02..fc78c7fbc 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -684,6 +684,17 @@ static int basicUnitTests(U32 const seed, double compressibility)
       if (r != CNBuffSize) goto _output_error; }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
+    {
+        ZSTD_DCtx* dctx = ZSTD_createDCtx();
+        size_t r;
+        CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
+        r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+        if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+        ZSTD_freeDCtx(dctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
     {   size_t u;
         for (u=0; u<CNBuffSize; u++) {
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index a44f8004a..fa8a1ae37 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2889,6 +2889,9 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
         CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
 
         /* multi - fragments decompression test */
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
+        }
         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
             DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", (void const*)dict);
             CHECK_Z( ZSTD_resetDStream(zd) );
@@ -2897,6 +2900,9 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
                 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
             CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
         }
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
+        }
         {   size_t decompressionResult = 1;
             ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
@@ -2938,7 +2944,14 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
         }   }
 
         /* try decompression on noisy data */
-        CHECK_Z( ZSTD_initDStream(zd_noise) );   /* note : no dictionary */
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_and_parameters));
+        } else {
+            CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_only));
+        }
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_setParameter(zd_noise, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
+        }
         {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
             while (outBuff.pos < dstBufferSize) {

From 321490cd5b9863433b3d44816d04012874e5ecdb Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 23 Jan 2023 20:23:43 -0800
Subject: [PATCH 156/197] [version-test] Work around bugs in v0.7.3 dict
 builder

Before calling a dictionary good, make sure that it can compress an
input. If v0.7.3 rejects v0.7.3's dictionary, fall back to the v1.0
dictionary. This is not the job of the verison test to test it, because
we cannot fix this code.
---
 tests/test-zstd-versions.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index d6784d615..1bcf39e2b 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -85,6 +85,18 @@ def get_git_tags():
     return tags
 
 
+def dict_ok(tag, dict_name, sample):
+    if not os.path.isfile(dict_name):
+        return False
+    try:
+        cmd = ['./zstd.' + tag, '-D', dict_name]
+        with open(sample, "rb") as i:
+            subprocess.check_call(cmd, stdin=i, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        return True
+    except:
+        return False
+
+
 def create_dict(tag, dict_source_path, fallback_tag=None):
     dict_name = 'dict.' + tag
     if not os.path.isfile(dict_name):
@@ -96,7 +108,7 @@ def create_dict(tag, dict_source_path, fallback_tag=None):
             result = execute('./dictBuilder.' + tag + ' ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
         else:
             result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
-        if result == 0 and os.path.isfile(dict_name):
+        if result == 0 and dict_ok(tag, dict_name, files[0]):
             print(dict_name + ' created')
         elif fallback_tag is not None:
             fallback_dict_name = 'dict.' + fallback_tag

From 8c85b29e3236d0f0be00e59de004d7297c4fddf0 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 23 Jan 2023 18:55:51 -0800
Subject: [PATCH 157/197] disable --rm on -o command

make it more similar to -c (aka `stdout`) convention.
---
 programs/fileio.c                             | 97 +++++++++++--------
 programs/util.c                               |  2 +-
 programs/zstdcli.c                            | 38 +++++---
 .../compress-file-to-stdout.sh.stderr.exact   |  2 +
 .../compress-stdin-to-stdout.sh.stderr.exact  |  2 +
 .../decompress-file-to-stdout.sh.stderr.exact |  2 +
 ...decompress-stdin-to-stdout.sh.stderr.exact |  2 +
 tests/playTests.sh                            | 39 ++++++--
 8 files changed, 118 insertions(+), 66 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 2d79203de..abc90b659 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -612,7 +612,7 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
         if (!prefs->overwrite) {
             if (g_display_prefs.displayLevel <= 1) {
                 /* No interaction possible */
-                DISPLAY("zstd: %s already exists; not overwritten  \n",
+                DISPLAYLEVEL(1, "zstd: %s already exists; not overwritten  \n",
                         dstFileName);
                 return NULL;
             }
@@ -723,7 +723,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
 
     filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
     if (!filenameTableSorted) {
-        DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
+        DISPLAYLEVEL(1, "Allocation error during filename collision checking \n");
         return 1;
     }
 
@@ -740,7 +740,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
     prevElem = filenameTableSorted[0];
     for (u = 1; u < nbFiles; ++u) {
         if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
-            DISPLAY("WARNING: Two files have same filename: %s\n", prevElem);
+            DISPLAYLEVEL(2, "WARNING: Two files have same filename: %s\n", prevElem);
         }
         prevElem = filenameTableSorted[u];
     }
@@ -823,41 +823,59 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
     FIO_setMemLimit(prefs, (unsigned)maxSize);
 }
 
-/* FIO_removeMultiFilesWarning() :
+/* FIO_multiFilesConcatWarning() :
+ * This function handles logic when processing multiple files with -o or -c, displaying the appropriate warnings/prompts.
  * Returns 1 if the console should abort, 0 if console should proceed.
- * This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts.
  *
- * If -f is specified, or there is just 1 file, zstd will always proceed as usual.
- * If --rm is specified, there will be a prompt asking for user confirmation.
- *         If -f is specified with --rm, zstd will proceed as usual
- *         If -q is specified with --rm, zstd will abort pre-emptively
- *         If neither flag is specified, zstd will prompt the user for confirmation to proceed.
- * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q).
- *         Note : --rm in combination with stdout is not allowed.
+ * If output is stdout or test mode is active, check that `--rm` disabled.
+ *
+ * If there is just 1 file to process, zstd will proceed as usual.
+ * If each file get processed into its own separate destination file, proceed as usual.
+ *
+ * When multiple files are processed into a single output,
+ * display a warning message, then disable --rm if it's set.
+ *
+ * If -f is specified or if output is stdout, just proceed.
+ * If output is set with -o, prompt for confirmation.
  */
-static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff)
+static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs, const char* outFileName, int displayLevelCutoff)
 {
-    int error = 0;
-    if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) {
-        if (g_display_prefs.displayLevel <= displayLevelCutoff) {
-            if (prefs->removeSrcFile) {
-                DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s\n", outFileName);
-                error =  1;
-            }
-        } else {
-            if (!strcmp(outFileName, stdoutmark)) {
-                DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
-            } else {
-                DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
-            }
-            DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate original file names nor directory structure. \n")
-            if (prefs->removeSrcFile) {
-                assert(fCtx->hasStdoutOutput == 0); /* not possible : never erase source files when output == stdout */
-                error = (g_display_prefs.displayLevel > displayLevelCutoff) && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
-            }
-        }
+    if (fCtx->hasStdoutOutput) assert(prefs->removeSrcFile == 0);
+    if (prefs->testMode) {
+        assert(prefs->removeSrcFile == 0);
+        return 0;
     }
-    return error;
+
+    if (fCtx->nbFilesTotal == 1) return 0;
+    assert(fCtx->nbFilesTotal > 1);
+
+    if (!outFileName) return 0;
+
+    if (fCtx->hasStdoutOutput) {
+        DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
+    } else {
+        DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
+    }
+    DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate original file names nor directory structure. \n")
+
+    /* multi-input into single output : --rm is not allowed */
+    if (prefs->removeSrcFile) {
+        DISPLAYLEVEL(2, "Since it's a destructive operation, input files will not be removed. \n");
+        prefs->removeSrcFile = 0;
+    }
+
+    if (fCtx->hasStdoutOutput) return 0;
+    if (prefs->overwrite) return 0;
+
+    /* multiple files concatenated into single destination file using -o without -f */
+    if (g_display_prefs.displayLevel <= displayLevelCutoff) {
+        /* quiet mode => no prompt => fail automatically */
+        DISPLAYLEVEL(1, "Concatenating multiple processed inputs into a single output loses file metadata. \n");
+        DISPLAYLEVEL(1, "Aborting. \n");
+        return 1;
+    }
+    /* normal mode => prompt */
+    return UTIL_requireUserConfirmation("Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
 }
 
 static ZSTD_inBuffer setInBuffer(const void* buf, size_t s, size_t pos)
@@ -1767,9 +1785,9 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
             &srcFileStat, compressionLevel);
     AIO_ReadPool_closeFile(ress.readCtx);
 
-    if ( prefs->removeSrcFile   /* --rm */
-      && result == 0       /* success */
-      && strcmp(srcFileName, stdinmark)   /* exception : don't erase stdin */
+    if ( prefs->removeSrcFile  /* --rm */
+      && result == 0           /* success */
+      && strcmp(srcFileName, stdinmark)  /* exception : don't erase stdin */
       ) {
         /* We must clear the handler, since after this point calling it would
          * delete both the source and destination files.
@@ -1791,7 +1809,8 @@ checked_index(const char* options[], size_t length, size_t index) {
 
 #define INDEX(options, index) checked_index((options), sizeof(options)  / sizeof(char*), (size_t)(index))
 
-void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
+void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
+{
     static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
                                            LZMA_EXTENSION, LZ4_EXTENSION};
     static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"};
@@ -1920,7 +1939,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
     assert(outFileName != NULL || suffix != NULL);
     if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
         FILE *dstFile;
-        if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
+        if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
             FIO_freeCResources(&ress);
             return 1;
         }
@@ -2742,7 +2761,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
     dRess_t ress = FIO_createDResources(prefs, dictFileName);
 
     if (outFileName) {
-        if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
+        if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
             FIO_freeDResources(ress);
             return 1;
         }
diff --git a/programs/util.c b/programs/util.c
index bfa2abed0..e017772ef 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -121,7 +121,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
     ch = getchar();
     result = 0;
     if (strchr(acceptableLetters, ch) == NULL) {
-        UTIL_DISPLAY("%s", abortMsg);
+        UTIL_DISPLAY("%s \n", abortMsg);
         result = 1;
     }
     /* flush the rest */
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index b3386988d..3adbae739 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -339,7 +339,7 @@ static const char* lastNameFromPath(const char* path)
 
 static void errorOut(const char* msg)
 {
-    DISPLAY("%s \n", msg); exit(1);
+    DISPLAYLEVEL(1, "%s \n", msg); exit(1);
 }
 
 /*! readU32FromCharChecked() :
@@ -786,13 +786,13 @@ static unsigned init_nbThreads(void) {
     } else {                      \
         argNb++;                  \
         if (argNb >= argCount) {  \
-            DISPLAY("error: missing command argument \n"); \
+            DISPLAYLEVEL(1, "error: missing command argument \n"); \
             CLEAN_RETURN(1);      \
         }                         \
         ptr = argv[argNb];        \
         assert(ptr != NULL);      \
         if (ptr[0]=='-') {        \
-            DISPLAY("error: command cannot be separated from its argument by another command \n"); \
+            DISPLAYLEVEL(1, "error: command cannot be separated from its argument by another command \n"); \
             CLEAN_RETURN(1);      \
 }   }   }
 
@@ -859,6 +859,7 @@ int main(int argCount, const char* argv[])
 
     FIO_prefs_t* const prefs = FIO_createPreferences();
     FIO_ctx_t* const fCtx = FIO_createContext();
+    FIO_progressSetting_e progress = FIO_ps_auto;
     zstd_operation_mode operation = zom_compress;
     ZSTD_compressionParameters compressionParams;
     int cLevel = init_cLevel();
@@ -898,7 +899,7 @@ int main(int argCount, const char* argv[])
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
     (void)memLimit;
     assert(argCount >= 1);
-    if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); }
+    if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
     programName = lastNameFromPath(programName);
 #ifdef ZSTD_MULTITHREAD
     nbWorkers = init_nbThreads();
@@ -999,8 +1000,8 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
                 if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_ps_enable; continue; }
                 if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_ps_disable; continue; }
-                if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
-                if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
+                if (!strcmp(argument, "--no-progress")) { progress = FIO_ps_never; continue; }
+                if (!strcmp(argument, "--progress")) { progress = FIO_ps_always; continue; }
                 if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
                 if (!strcmp(argument, "--fake-stdin-is-console")) { UTIL_fakeStdinIsConsole(); continue; }
                 if (!strcmp(argument, "--fake-stdout-is-console")) { UTIL_fakeStdoutIsConsole(); continue; }
@@ -1057,7 +1058,7 @@ int main(int argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--output-dir-flat")) {
                     NEXT_FIELD(outDirName);
                     if (strlen(outDirName) == 0) {
-                        DISPLAY("error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
+                        DISPLAYLEVEL(1, "error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
                         CLEAN_RETURN(1);
                     }
                     continue;
@@ -1073,7 +1074,7 @@ int main(int argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--output-dir-mirror")) {
                     NEXT_FIELD(outMirroredDirName);
                     if (strlen(outMirroredDirName) == 0) {
-                        DISPLAY("error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
+                        DISPLAYLEVEL(1, "error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
                         CLEAN_RETURN(1);
                     }
                     continue;
@@ -1349,7 +1350,7 @@ int main(int argCount, const char* argv[])
         int const ret = FIO_listMultipleFiles((unsigned)filenames->tableSize, filenames->fileNames, g_displayLevel);
         CLEAN_RETURN(ret);
 #else
-        DISPLAY("file information is not supported \n");
+        DISPLAYLEVEL(1, "file information is not supported \n");
         CLEAN_RETURN(1);
 #endif
     }
@@ -1480,24 +1481,29 @@ int main(int argCount, const char* argv[])
 
     if (showDefaultCParams) {
         if (operation == zom_decompress) {
-            DISPLAY("error : can't use --show-default-cparams in decompression mode \n");
+            DISPLAYLEVEL(1, "error : can't use --show-default-cparams in decompression mode \n");
             CLEAN_RETURN(1);
         }
     }
 
     if (dictFileName != NULL && patchFromDictFileName != NULL) {
-        DISPLAY("error : can't use -D and --patch-from=# at the same time \n");
+        DISPLAYLEVEL(1, "error : can't use -D and --patch-from=# at the same time \n");
         CLEAN_RETURN(1);
     }
 
     if (patchFromDictFileName != NULL && filenames->tableSize > 1) {
-        DISPLAY("error : can't use --patch-from=# on multiple files \n");
+        DISPLAYLEVEL(1, "error : can't use --patch-from=# on multiple files \n");
         CLEAN_RETURN(1);
     }
 
-    /* No status message in pipe mode (stdin - stdout) */
+    /* No status message by default when output is stdout */
     hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
-    if ((hasStdout || !UTIL_isConsole(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
+    if (hasStdout && (g_displayLevel==2)) g_displayLevel=1;
+
+    /* when stderr is not the console, do not pollute it with status updates
+     * Note : the below code actually also silence more stuff, including completion report. */
+    if (!UTIL_isConsole(stderr) && (g_displayLevel==2)) g_displayLevel=1;
+    FIO_setProgressSetting(progress);
 
     /* don't remove source files when output is stdout */;
     if (hasStdout && removeSrcFile) {
@@ -1569,7 +1575,7 @@ int main(int argCount, const char* argv[])
             operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
         (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; (void)useRowMatchFinder; /* not used when ZSTD_NOCOMPRESS set */
-        DISPLAY("Compression not supported \n");
+        DISPLAYLEVEL(1, "Compression not supported \n");
 #endif
     } else {  /* decompression or test */
 #ifndef ZSTD_NODECOMPRESS
@@ -1579,7 +1585,7 @@ int main(int argCount, const char* argv[])
             operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName);
         }
 #else
-        DISPLAY("Decompression not supported \n");
+        DISPLAYLEVEL(1, "Decompression not supported \n");
 #endif
     }
 
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
index e86c4eae3..7c690d20b 100644
--- a/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
@@ -2,6 +2,8 @@ Trace:FileStat: > UTIL_isLink(file)
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isConsole(1)
 Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
 Trace:FileStat: > UTIL_getFileSize(file)
 Trace:FileStat:  > UTIL_stat(file)
 Trace:FileStat:  < 1
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
index 2e44511d8..8bf05e641 100644
--- a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
@@ -2,6 +2,8 @@ Trace:FileStat: > UTIL_isConsole(0)
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isConsole(1)
 Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
 Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
 Trace:FileStat:  > UTIL_stat(/*stdin*\)
 Trace:FileStat:  < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
index bbf66506b..7fe6dda15 100644
--- a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
@@ -2,6 +2,8 @@ Trace:FileStat: > UTIL_isLink(file.zst)
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isConsole(1)
 Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isDirectory(file.zst)
 Trace:FileStat:  > UTIL_stat(file.zst)
 Trace:FileStat:  < 1
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
index 61487f61e..e36cb9d05 100644
--- a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
@@ -2,6 +2,8 @@ Trace:FileStat: > UTIL_isConsole(0)
 Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isConsole(1)
 Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
 Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
 Trace:FileStat:  > UTIL_stat(/*stdin*\)
 Trace:FileStat:  < 0
diff --git a/tests/playTests.sh b/tests/playTests.sh
index d0764d4d3..e064c86df 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -387,7 +387,7 @@ println "\n===>  file removal"
 zstd -f --rm tmp
 test ! -f tmp  # tmp should no longer be present
 zstd -f -d --rm tmp.zst
-test ! -f tmp.zst   # tmp.zst should no longer be present
+test ! -f tmp.zst  # tmp.zst should no longer be present
 println "test: --rm is disabled when output is stdout"
 test -f tmp
 zstd --rm tmp -c > $INTOVOID
@@ -396,6 +396,20 @@ zstd -f --rm tmp -c > $INTOVOID
 test -f tmp # tmp shall still be there
 zstd -f tmp -c > $INTOVOID --rm
 test -f tmp # tmp shall still be there
+println "test: --rm is disabled when multiple inputs are concatenated into a single output"
+cp tmp tmp2
+zstd --rm tmp tmp2 -c > $INTOVOID
+test -f tmp
+test -f tmp2
+rm -f tmp3.zst
+echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation
+test -f tmp
+test -f tmp2
+zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt
+test -f tmp
+test -f tmp2
+zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate"
+
 println "test : should quietly not remove non-regular file"
 println hello > tmp
 zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
@@ -437,34 +451,35 @@ println hello > tmp1
 println world > tmp2
 zstd tmp1 tmp2 -o "$INTOVOID" -f
 zstd tmp1 tmp2 -c | zstd -t
-zstd tmp1 tmp2 -o tmp.zst
+echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst
 test ! -f tmp1.zst
 test ! -f tmp2.zst
 zstd tmp1 tmp2
 zstd -t tmp1.zst tmp2.zst
 zstd -dc tmp1.zst tmp2.zst
 zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f
-zstd -d tmp1.zst tmp2.zst -o tmp
+echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp
 touch tmpexists
 zstd tmp1 tmp2 -f -o tmpexists
 zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite"
 println gooder > tmp_rm1
 println boi > tmp_rm2
 println worldly > tmp_rm3
-echo 'y' | zstd tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst --rm     # tests the warning prompt for --rm with multiple inputs into once source
-test ! -f tmp_rm1
-test ! -f tmp_rm2
+echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst
+test -f tmp_rm1
+test -f tmp_rm2
 cp tmp_rm3.zst tmp_rm4.zst
-echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
-test ! -f tmp_rm3.zst
-test ! -f tmp_rm4.zst
+echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
+test -f tmp_rm3.zst
+test -f tmp_rm4.zst
 println gooder > tmpexists1
 zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID
-
 # Bug: PR #972
 if [ "$?" -eq 139 ]; then
   die "should not have segfaulted"
 fi
+test -f tmpexists1
+test -f tmpexists
 println "\n===>  multiple files and shell completion "
 datagen -s1        > tmp1 2> $INTOVOID
 datagen -s2 -g100K > tmp2 2> $INTOVOID
@@ -1172,6 +1187,10 @@ zstd -t tmp3 && die "bad file not detected !"   # detects 0-sized files as bad
 println "test --rm and --test combined "
 zstd -t --rm tmp1.zst
 test -f tmp1.zst   # check file is still present
+cp tmp1.zst tmp2.zst
+zstd -t tmp1.zst tmp2.zst --rm
+test -f tmp1.zst   # check file is still present
+test -f tmp2.zst   # check file is still present
 split -b16384 tmp1.zst tmpSplit.
 zstd -t tmpSplit.* && die "bad file not detected !"
 datagen | zstd -c | zstd -t

From 02434e0867695db62ade7a94852dfeae1b630e75 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 25 Jan 2023 16:18:20 -0800
Subject: [PATCH 158/197] enforce a hard fail when input files are set to be
 erased

in scenarios where it's supposed to not be possible.

suggested by @terrelln
---
 programs/fileio.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index abc90b659..fb71804d6 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -840,9 +840,21 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
  */
 static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs, const char* outFileName, int displayLevelCutoff)
 {
-    if (fCtx->hasStdoutOutput) assert(prefs->removeSrcFile == 0);
+    if (fCtx->hasStdoutOutput) {
+        if (prefs->removeSrcFile)
+            /* this should not happen ; hard fail, to protect user's data
+             * note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
+            EXM_THROW(43, "It's not allowed to remove input files when processed output is piped to stdout. "
+                "This scenario is not supposed to be possible. "
+                "This is a programming error. File an issue for it to be fixed.");
+    }
     if (prefs->testMode) {
-        assert(prefs->removeSrcFile == 0);
+        if (prefs->removeSrcFile)
+            /* this should not happen ; hard fail, to protect user's data
+             * note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
+            EXM_THROW(43, "Test mode shall not remove input files! "
+                 "This scenario is not supposed to be possible. "
+                 "This is a programming error. File an issue for it to be fixed.");
         return 0;
     }
 

From 6422d1d7a85182e983b361a20bbe2064611680fc Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <yoniko@gmail.com>
Date: Wed, 25 Jan 2023 17:59:35 -0800
Subject: [PATCH 159/197] Bugfix: `--[no-]row-match-finder` do the opposite of
 what they are supposed to

---
 programs/zstdcli.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index b3386988d..a64d5d53b 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -838,7 +838,6 @@ int main(int argCount, const char* argv[])
         ldmFlag = 0,
         main_pause = 0,
         adapt = 0,
-        useRowMatchFinder = 0,
         adaptMin = MINCLEVEL,
         adaptMax = MAXCLEVEL,
         rsyncable = 0,
@@ -852,6 +851,7 @@ int main(int argCount, const char* argv[])
         ultra=0,
         contentSize=1,
         removeSrcFile=0;
+    ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
     unsigned nbWorkers = 0;
     double compressibility = 0.5;
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
@@ -977,8 +977,8 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
                 if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
                 if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
-                if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = 1; continue; }
-                if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = 2; continue; }
+                if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = ZSTD_ps_disable; continue; }
+                if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = ZSTD_ps_enable; continue; }
                 if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
                 if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
                 if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }

From 3c215220e321b85763bb242d47184711919b951e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 25 Jan 2023 16:29:34 -0800
Subject: [PATCH 160/197] modify cli-test logic : ignore stderr message by
 default

Previously, cli-test would, by default, check that a stderr output is strictly identical to a saved outcome.
When there was no instructions on how to interpret stderr, it would default to requiring it to be empty.

There are many tests cases though where stderr content doesn't matter, and we are mainly interested in the return code of the cli.
For these cases, it was possible to set a .ignore document, which would instruct to ignore stderr content.

This PR update the logic, to make .ignore the default.
When willing to check that stderr content is empty, one must now add an empty .strict file.

This will allow status message to evolve without triggering many cli-tests errors.
This is especially important when some of these status include compression results, which may change as a result of compression optimizations.
It also makes it easier to add new tests which only care about the CLI's return code.
---
 tests/cli-tests/run.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/cli-tests/run.py b/tests/cli-tests/run.py
index 8b4e21a53..45af5124b 100755
--- a/tests/cli-tests/run.py
+++ b/tests/cli-tests/run.py
@@ -395,12 +395,10 @@ class TestCase:
             return self._check_output_exact(out_name, read_file(exact_name), exact_name)
         elif os.path.exists(glob_name):
             return self._check_output_glob(out_name, read_file(glob_name))
-        elif os.path.exists(ignore_name):
+        else:
             check_name = f"check_{out_name}"
             self._success[check_name] = True
             self._message[check_name] = f"{out_name} ignored!"
-        else:
-            return self._check_output_exact(out_name, bytes(), exact_name)
 
     def _check_stderr(self) -> None:
         """Checks the stderr output against the expectation."""
@@ -738,4 +736,3 @@ if __name__ == "__main__":
         sys.exit(0)
     else:
         sys.exit(1)
-

From 82ca00811aa297c1f00f549facc70787fdb59d64 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Jan 2023 11:43:27 -0800
Subject: [PATCH 161/197] change logic when stderr is not console : don't
 update progress status

but keep warnings and final operation statement.

updated tests/cli-tests/ accordingly
---
 programs/zstdcli.c                            |  5 +-
 tests/cli-tests/basic/memlimit.sh             | 14 +++---
 tests/cli-tests/compression/levels.sh         | 44 ++++++++---------
 .../compression/levels.sh.stderr.exact        | 48 +++++++++----------
 tests/cli-tests/compression/multi-threaded.sh | 16 +++----
 .../multi-threaded.sh.stderr.exact            | 10 ++++
 .../pass-through.sh.stderr.exact              |  1 +
 .../dictionaries/dictionary-mismatch.sh       |  2 +-
 .../dictionary-mismatch.sh.stderr.exact       |  3 +-
 .../progress/no-progress.sh.stderr.glob       |  6 +++
 10 files changed, 83 insertions(+), 66 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index ce2b96a8c..39f8b34fe 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1500,9 +1500,8 @@ int main(int argCount, const char* argv[])
     hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
     if (hasStdout && (g_displayLevel==2)) g_displayLevel=1;
 
-    /* when stderr is not the console, do not pollute it with status updates
-     * Note : the below code actually also silence more stuff, including completion report. */
-    if (!UTIL_isConsole(stderr) && (g_displayLevel==2)) g_displayLevel=1;
+    /* when stderr is not the console, do not pollute it with progress updates (unless requested) */
+    if (!UTIL_isConsole(stderr) && (progress!=FIO_ps_always)) progress=FIO_ps_never;
     FIO_setProgressSetting(progress);
 
     /* don't remove source files when output is stdout */;
diff --git a/tests/cli-tests/basic/memlimit.sh b/tests/cli-tests/basic/memlimit.sh
index 61a5fc084..88d734d0d 100755
--- a/tests/cli-tests/basic/memlimit.sh
+++ b/tests/cli-tests/basic/memlimit.sh
@@ -15,25 +15,25 @@ zstd --memory=32asbdf file && die "Should not allow bogus suffix"
 println "+ zstd --memory=hello file"
 zstd --memory=hello file && die "Should not allow non-numeric parameter"
 println "+ zstd --memory=1 file"
-zstd --memory=1 file && die "Should allow numeric parameter without suffix"
+zstd -q --memory=1 file && die "Should allow numeric parameter without suffix"
 rm file.zst
 println "+ zstd --memory=1K file"
-zstd --memory=1K file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1KB file"
-zstd --memory=1KB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1KiB file"
-zstd --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1M file"
-zstd --memory=1M file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1MB file"
-zstd --memory=1MB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1MiB file"
-zstd --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 
 rm file
diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh
index 47ac2029f..cc2700a30 100755
--- a/tests/cli-tests/compression/levels.sh
+++ b/tests/cli-tests/compression/levels.sh
@@ -6,10 +6,10 @@ set -v
 datagen > file
 
 # Compress with various levels and ensure that their sizes are ordered
-zstd --fast=10 file -o file-f10.zst
-zstd --fast=1 file -o file-f1.zst
-zstd -1 file -o file-1.zst
-zstd -19 file -o file-19.zst
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
 
 zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
 
@@ -18,15 +18,15 @@ cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst
 
 # Test default levels
-zstd --fast file -f
+zstd --fast file -f -q
 cmp file.zst file-f1.zst || die "--fast is not level -1"
 
-zstd -0 file -o file-0.zst
-zstd -f file
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
 cmp file.zst file-0.zst || die "Level 0 is not the default level"
 
 # Test level clamping
-zstd -99 file -o file-99.zst
+zstd -99 file -o file-99.zst -q
 cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
 zstd --fast=200000 file -c | zstd -t
 
@@ -34,10 +34,10 @@ zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 
 # Test setting a level through the environment variable
-ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
-ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
-ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
-ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
 
 cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
 cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
@@ -45,18 +45,18 @@ cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level
 cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
 
 # Test invalid environment clevel is the default level
-zstd -f file
-ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q    ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
 
 # Test environment clevel is overridden by command line
-ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
-ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
 
 cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
 cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact
index 5664f854b..c8fb79c68 100644
--- a/tests/cli-tests/compression/levels.sh.stderr.exact
+++ b/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -2,30 +2,31 @@
 datagen > file
 
 # Compress with various levels and ensure that their sizes are ordered
-zstd --fast=10 file -o file-f10.zst
-zstd --fast=1 file -o file-f1.zst
-zstd -1 file -o file-1.zst
-zstd -19 file -o file-19.zst
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
 
 zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
+4 files decompressed : 262148 bytes total 
 
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst
 
 # Test default levels
-zstd --fast file -f
+zstd --fast file -f -q
 cmp file.zst file-f1.zst || die "--fast is not level -1"
 
-zstd -0 file -o file-0.zst
-zstd -f file
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
 cmp file.zst file-0.zst || die "Level 0 is not the default level"
 
 # Test level clamping
-zstd -99 file -o file-99.zst
-Warning : compression level higher than max, reduced to 19 
+zstd -99 file -o file-99.zst -q
 cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
 zstd --fast=200000 file -c | zstd -t
+/*stdin*\           : 65537 bytes 
 
 zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
@@ -33,11 +34,10 @@ zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
 
 # Test setting a level through the environment variable
-ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
-ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
-ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
-ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
-Warning : compression level higher than max, reduced to 19 
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
 
 cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
 cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
@@ -45,25 +45,25 @@ cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level
 cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
 
 # Test invalid environment clevel is the default level
-zstd -f file
-ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value 
-ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value 
-ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value 
-ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value 
-ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value 
-ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q    ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value 
-ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large 
 
 # Test environment clevel is overridden by command line
-ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
-ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
 
 cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
 cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/tests/cli-tests/compression/multi-threaded.sh b/tests/cli-tests/compression/multi-threaded.sh
index bd01448e2..17a5eb518 100755
--- a/tests/cli-tests/compression/multi-threaded.sh
+++ b/tests/cli-tests/compression/multi-threaded.sh
@@ -3,13 +3,13 @@
 set -e
 
 # Test multi-threaded flags
-zstd --single-thread file -f            ; zstd -t file.zst
-zstd -T2 -f file                        ; zstd -t file.zst
-zstd --rsyncable -f file                ; zstd -t file.zst
-zstd -T0 -f file                        ; zstd -t file.zst
-zstd -T0 --auto-threads=logical -f file ; zstd -t file.zst
-zstd -T0 --auto-threads=physical -f file; zstd -t file.zst
+zstd --single-thread file -f -q         ; zstd -t file.zst
+zstd -T2 -f file -q                     ; zstd -t file.zst
+zstd --rsyncable -f file -q             ; zstd -t file.zst
+zstd -T0 -f file -q                     ; zstd -t file.zst
+zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
+zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
 
 # multi-thread decompression warning test
-zstd -T0 -f file                        ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
-zstd -T0 -f file                        ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
+zstd -T0 -f file -q                     ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
+zstd -T0 -f file -q                     ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
diff --git a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
index 54d47d8bd..11daff6ba 100644
--- a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
+++ b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
@@ -1 +1,11 @@
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
 Warning : decompression does not support multi-threading
+file.zst            : 65537 bytes 
diff --git a/tests/cli-tests/decompression/pass-through.sh.stderr.exact b/tests/cli-tests/decompression/pass-through.sh.stderr.exact
index f9ac13cb2..62f96ae42 100644
--- a/tests/cli-tests/decompression/pass-through.sh.stderr.exact
+++ b/tests/cli-tests/decompression/pass-through.sh.stderr.exact
@@ -1,3 +1,4 @@
+file                 :230.00%   (    10 B =>     23 B, file.zst) 
 zstd: file: unsupported format 
 zstd: file: unsupported format 
 zstd: file: unsupported format 
diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
index 8264ccca5..885cac223 100755
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
@@ -23,7 +23,7 @@ if [ false ]; then
 fi
 
 set -v
-zstd files/0 -D dicts/0
+zstd files/0 -D dicts/0 -q
 zstd -t files/0.zst -D dicts/0
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 zstd -t files/0.zst            && die "Must fail" ||:
diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
index 0afea722e..8896763c1 100644
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
@@ -1,5 +1,6 @@
-zstd files/0 -D dicts/0
+zstd files/0 -D dicts/0 -q
 zstd -t files/0.zst -D dicts/0
+files/0.zst         : 1000 bytes 
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
 zstd -t files/0.zst            && die "Must fail" ||:
diff --git a/tests/cli-tests/progress/no-progress.sh.stderr.glob b/tests/cli-tests/progress/no-progress.sh.stderr.glob
index b4262517a..d0f911227 100644
--- a/tests/cli-tests/progress/no-progress.sh.stderr.glob
+++ b/tests/cli-tests/progress/no-progress.sh.stderr.glob
@@ -1,15 +1,21 @@
 Tests cases where progress information should not be printed
 args =
 compress file to file
+hello*hello.zst*
 compress pipe to pipe
 compress pipe to file
+*stdin*hello.zst*
 compress file to pipe
 compress 2 files
+2 files compressed*
 decompress file to file
+hello.zst*
 decompress pipe to pipe
 decompress pipe to file
+*stdin*
 decompress file to pipe
 decompress 2 files
+2 files decompressed*
 
 args = --fake-stderr-is-console -q
 compress file to file

From bda947e17a61b9f7434ea878dff04a409a2ff772 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 26 Jan 2023 12:11:25 -0800
Subject: [PATCH 162/197] [huf] Fix bug in fast C decoders

The input bounds checks were buggy because they were only breaking from
the inner loop, not the outer loop. The fuzzers found this immediately.
The fix is to use `goto _out` instead of `break`.

This condition can happen on corrupted inputs.

I've benchmarked before and after on x86-64 and there were small changes
in performance, some positive, and some negative, and they end up about
balacing out.

Credit to  OSS-Fuzz
---
 lib/decompress/huf_decompress.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 0a29f03d4..c2d1f633a 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -742,7 +742,7 @@ void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
              */
             for (stream = 1; stream < 4; ++stream) {
                 if (ip[stream] < ip[stream - 1])
-                    break;
+                    goto _out;
             }
         }
 
@@ -775,6 +775,8 @@ void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
         } while (op[3] < olimit);
     }
 
+_out:
+
     /* Save the final values of each of the state variables back to args. */
     ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
     ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
@@ -1535,7 +1537,7 @@ void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
              */
             for (stream = 1; stream < 4; ++stream) {
                 if (ip[stream] < ip[stream - 1])
-                    break;
+                    goto _out;
             }
         }
 
@@ -1593,6 +1595,8 @@ void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
         } while (op[3] < olimit);
     }
 
+_out:
+
     /* Save the final values of each of the state variables back to args. */
     ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
     ZSTD_memcpy(&args->ip, &ip, sizeof(ip));

From 814f4bfb993c6a1a171b2321f69afcd0dea01f1b Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Fri, 27 Jan 2023 08:21:47 -0800
Subject: [PATCH 163/197] fix long offset resolution

---
 lib/common/zstd_internal.h   |  1 +
 lib/compress/zstd_compress.c | 17 +++++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 94670a03d..12d73e209 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -299,6 +299,7 @@ typedef struct {
     BYTE*  ofCode;
     size_t maxNbSeq;
     size_t maxNbLit;
+    BYTE* longOffsets;
 
     /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
      * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 07e62bf5c..fca22c4bb 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1610,7 +1610,8 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
     size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
-                            + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
+                            + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE))
+                            + ZSTD_cwksp_alloc_size(sizeof(BYTE)); /* longOffsets */
     size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE);
     size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
     size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
@@ -2109,6 +2110,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
         zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
         zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
+        zc->seqStore.longOffsets = ZSTD_cwksp_reserve_buffer(ws, sizeof(BYTE));
+        zc->seqStore.longOffsets[0] = 0;
         zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
 
         FORWARD_IF_ERROR(ZSTD_reset_matchState(
@@ -2565,16 +2568,21 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
     BYTE* const llCodeTable = seqStorePtr->llCode;
     BYTE* const ofCodeTable = seqStorePtr->ofCode;
     BYTE* const mlCodeTable = seqStorePtr->mlCode;
+    BYTE* const longOffsetsFlag = seqStorePtr->longOffsets;
     U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     U32 u;
+    BYTE longOffsets = 0;
     assert(nbSeq <= seqStorePtr->maxNbSeq);
     for (u=0; u<nbSeq; u++) {
         U32 const llv = sequences[u].litLength;
+        U32 const ofCode = ZSTD_highbit32(sequences[u].offBase);
         U32 const mlv = sequences[u].mlBase;
         llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
-        ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offBase);
+        ofCodeTable[u] = (BYTE)ofCode;
         mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
+        longOffsets |= (ofCode >= STREAM_ACCUMULATOR_MIN);
     }
+    longOffsetsFlag[0] = longOffsets;
     if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
         llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
     if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
@@ -2756,7 +2764,6 @@ ZSTD_entropyCompressSeqStore_internal(
                               void* entropyWorkspace, size_t entropyWkspSize,
                         const int bmi2)
 {
-    const int longOffsets = cctxParams->cParams.windowLog >= STREAM_ACCUMULATOR_MIN;
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
     unsigned* count = (unsigned*)entropyWorkspace;
     FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
@@ -2767,6 +2774,7 @@ ZSTD_entropyCompressSeqStore_internal(
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
     const BYTE* const llCodeTable = seqStorePtr->llCode;
     const BYTE* const mlCodeTable = seqStorePtr->mlCode;
+    const BYTE* const longOffsetsFlag = seqStorePtr->longOffsets;
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstCapacity;
     BYTE* op = ostart;
@@ -2834,7 +2842,8 @@ ZSTD_entropyCompressSeqStore_internal(
         op += stats.size;
     }
 
-    {   size_t const bitstreamSize = ZSTD_encodeSequences(
+    {   const BYTE longOffsets = longOffsetsFlag[0];
+        size_t const bitstreamSize = ZSTD_encodeSequences(
                                         op, (size_t)(oend - op),
                                         CTable_MatchLength, mlCodeTable,
                                         CTable_OffsetBits, ofCodeTable,

From d210628b0b01d4447de2b9e4404d2bf557b40217 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Fri, 27 Jan 2023 09:52:00 -0800
Subject: [PATCH 164/197] initialize long offsets in decodecorpus

---
 tests/decodecorpus.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 148088a76..b04015d89 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -186,6 +186,7 @@ BYTE SEQUENCE_LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; /* storeSeq expects a place to
 BYTE SEQUENCE_LLCODE[ZSTD_BLOCKSIZE_MAX];
 BYTE SEQUENCE_MLCODE[ZSTD_BLOCKSIZE_MAX];
 BYTE SEQUENCE_OFCODE[ZSTD_BLOCKSIZE_MAX];
+BYTE SEQUENCE_LONGOFFSETS[1];
 
 U64 WKSP[HUF_WORKSPACE_SIZE_U64];
 
@@ -634,6 +635,7 @@ static inline void initSeqStore(seqStore_t *seqStore) {
     seqStore->llCode = SEQUENCE_LLCODE;
     seqStore->mlCode = SEQUENCE_MLCODE;
     seqStore->ofCode = SEQUENCE_OFCODE;
+    seqStore->longOffsets = SEQUENCE_LONGOFFSETS;
 
     ZSTD_resetSeqStore(seqStore);
 }

From 9e4c66b9e92df871c8e0da61c1796d402874fed6 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Fri, 27 Jan 2023 12:04:29 -0800
Subject: [PATCH 165/197] record long offsets in ZSTD_symbolEncodingTypeStats_t
 + add test case

---
 lib/common/zstd_internal.h   |  3 +-
 lib/compress/zstd_compress.c | 23 ++++++--------
 tests/decodecorpus.c         |  2 --
 tests/zstreamtest.c          | 60 ++++++++++++++++++++++++++++++++++++
 4 files changed, 71 insertions(+), 17 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 12d73e209..8878aa100 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -299,7 +299,6 @@ typedef struct {
     BYTE*  ofCode;
     size_t maxNbSeq;
     size_t maxNbLit;
-    BYTE* longOffsets;
 
     /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
      * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
@@ -347,7 +346,7 @@ typedef struct {
 } ZSTD_frameSizeInfo;   /* decompress & legacy */
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
 
 /* custom memory allocation functions */
 void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index fca22c4bb..a83b1ccdd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1610,8 +1610,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
     size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
-                            + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE))
-                            + ZSTD_cwksp_alloc_size(sizeof(BYTE)); /* longOffsets */
+                            + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
     size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE);
     size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
     size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
@@ -2110,8 +2109,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
         zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
         zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
-        zc->seqStore.longOffsets = ZSTD_cwksp_reserve_buffer(ws, sizeof(BYTE));
-        zc->seqStore.longOffsets[0] = 0;
         zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
 
         FORWARD_IF_ERROR(ZSTD_reset_matchState(
@@ -2562,16 +2559,15 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par
 
 /* See doc/zstd_compression_format.md for detailed format description */
 
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
 {
     const seqDef* const sequences = seqStorePtr->sequencesStart;
     BYTE* const llCodeTable = seqStorePtr->llCode;
     BYTE* const ofCodeTable = seqStorePtr->ofCode;
     BYTE* const mlCodeTable = seqStorePtr->mlCode;
-    BYTE* const longOffsetsFlag = seqStorePtr->longOffsets;
     U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     U32 u;
-    BYTE longOffsets = 0;
+    int longOffsets = 0;
     assert(nbSeq <= seqStorePtr->maxNbSeq);
     for (u=0; u<nbSeq; u++) {
         U32 const llv = sequences[u].litLength;
@@ -2582,11 +2578,11 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
         mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
         longOffsets |= (ofCode >= STREAM_ACCUMULATOR_MIN);
     }
-    longOffsetsFlag[0] = longOffsets;
     if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
         llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
     if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
         mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
+    return longOffsets;
 }
 
 /* ZSTD_useTargetCBlockSize():
@@ -2620,6 +2616,7 @@ typedef struct {
     U32 MLtype;
     size_t size;
     size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
+    int longOffsets;
 } ZSTD_symbolEncodingTypeStats_t;
 
 /* ZSTD_buildSequencesStatistics():
@@ -2650,7 +2647,7 @@ ZSTD_buildSequencesStatistics(
 
     stats.lastCountSize = 0;
     /* convert length/distances into codes */
-    ZSTD_seqToCodes(seqStorePtr);
+    stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
     assert(op <= oend);
     assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
     /* build CTable for Literal Lengths */
@@ -2774,11 +2771,11 @@ ZSTD_entropyCompressSeqStore_internal(
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
     const BYTE* const llCodeTable = seqStorePtr->llCode;
     const BYTE* const mlCodeTable = seqStorePtr->mlCode;
-    const BYTE* const longOffsetsFlag = seqStorePtr->longOffsets;
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstCapacity;
     BYTE* op = ostart;
     size_t lastCountSize;
+    int longOffsets = 0;
 
     entropyWorkspace = count + (MaxSeq + 1);
     entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
@@ -2840,10 +2837,10 @@ ZSTD_entropyCompressSeqStore_internal(
         *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
         lastCountSize = stats.lastCountSize;
         op += stats.size;
+        longOffsets = stats.longOffsets;
     }
 
-    {   const BYTE longOffsets = longOffsetsFlag[0];
-        size_t const bitstreamSize = ZSTD_encodeSequences(
+    {   size_t const bitstreamSize = ZSTD_encodeSequences(
                                         op, (size_t)(oend - op),
                                         CTable_MatchLength, mlCodeTable,
                                         CTable_OffsetBits, ofCodeTable,
@@ -3485,7 +3482,7 @@ ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
 static ZSTD_symbolEncodingTypeStats_t
 ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
 {
-    ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0};
+    ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
     nextEntropy->litlength_repeatMode = FSE_repeat_none;
     nextEntropy->offcode_repeatMode = FSE_repeat_none;
     nextEntropy->matchlength_repeatMode = FSE_repeat_none;
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index b04015d89..148088a76 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -186,7 +186,6 @@ BYTE SEQUENCE_LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; /* storeSeq expects a place to
 BYTE SEQUENCE_LLCODE[ZSTD_BLOCKSIZE_MAX];
 BYTE SEQUENCE_MLCODE[ZSTD_BLOCKSIZE_MAX];
 BYTE SEQUENCE_OFCODE[ZSTD_BLOCKSIZE_MAX];
-BYTE SEQUENCE_LONGOFFSETS[1];
 
 U64 WKSP[HUF_WORKSPACE_SIZE_U64];
 
@@ -635,7 +634,6 @@ static inline void initSeqStore(seqStore_t *seqStore) {
     seqStore->llCode = SEQUENCE_LLCODE;
     seqStore->mlCode = SEQUENCE_MLCODE;
     seqStore->ofCode = SEQUENCE_OFCODE;
-    seqStore->longOffsets = SEQUENCE_LONGOFFSETS;
 
     ZSTD_resetSeqStore(seqStore);
 }
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index fa8a1ae37..3e5d854f1 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2222,6 +2222,66 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+
+    DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        ZSTD_DCtx* dctx = ZSTD_createDCtx();
+
+        /* Test large offset, small window size*/
+        {
+            size_t srcSize = 21;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 4;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+            void* const checkBuf = malloc(srcSize);
+            const size_t largeDictSize = 1 << 30;
+            ZSTD_CDict* cdict = NULL;
+            ZSTD_DDict* ddict = NULL;
+
+            /* Generate large dictionary */
+            void* dictBuffer = calloc(largeDictSize, 1);
+            ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
+            cParams.minMatch = ZSTD_MINMATCH_MIN;
+            cParams.hashLog = ZSTD_HASHLOG_MIN;
+            cParams.chainLog = ZSTD_CHAINLOG_MIN;
+
+            cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
+            ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
+
+            ZSTD_CCtx_refCDict(cctx, cdict);
+            ZSTD_DCtx_refDDict(dctx, ddict);
+
+            sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
+            sequences[1] = (ZSTD_Sequence) {1 << 29, 0, 3, 0};
+            sequences[2] = (ZSTD_Sequence) {1 << 29, 0, 9, 0};
+            sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(ZSTD_isError(cSize), "Should not throw an error");
+
+            {
+                size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
+                CHECK(ZSTD_isError(dSize), "Should not throw an error");
+                CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+            }
+
+            free(sequences);
+            free(checkBuf);
+            free(dictBuffer);
+            ZSTD_freeCDict(cdict);
+            ZSTD_freeDDict(ddict);
+        }
+        ZSTD_freeCCtx(cctx);
+        ZSTD_freeDCtx(dctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);

From 423a74986f7416ca8f54b20d0fe880f2d232b213 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 27 Jan 2023 11:59:26 -0800
Subject: [PATCH 166/197] [fse] Delete unused functions

Delete all unused FSE functions, now that we are no longer syncing
to/from upstream.

This avoids confusion about Zstd's stack usage like in Issue #3453.
It also removes dead code, which is always a plus.
---
 lib/common/fse.h            |  82 +------------------------
 lib/common/fse_decompress.c |  94 ----------------------------
 lib/compress/fse_compress.c | 118 ------------------------------------
 3 files changed, 2 insertions(+), 292 deletions(-)

diff --git a/lib/common/fse.h b/lib/common/fse.h
index 2ef8df459..02a1f0bc5 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -53,34 +53,6 @@ extern "C" {
 FSE_PUBLIC_API unsigned FSE_versionNumber(void);   /**< library version number; to be used when checking dll version */
 
 
-/*-****************************************
-*  FSE simple functions
-******************************************/
-/*! FSE_compress() :
-    Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
-    'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
-    @return : size of compressed data (<= dstCapacity).
-    Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
-                     if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
-                     if FSE_isError(return), compression failed (more details using FSE_getErrorName())
-*/
-FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
-                             const void* src, size_t srcSize);
-
-/*! FSE_decompress():
-    Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
-    into already allocated destination buffer 'dst', of size 'dstCapacity'.
-    @return : size of regenerated data (<= maxDstSize),
-              or an error code, which can be tested using FSE_isError() .
-
-    ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
-    Why ? : making this distinction requires a header.
-    Header management is intentionally delegated to the user layer, which can better manage special cases.
-*/
-FSE_PUBLIC_API size_t FSE_decompress(void* dst,  size_t dstCapacity,
-                               const void* cSrc, size_t cSrcSize);
-
-
 /*-*****************************************
 *  Tool functions
 ******************************************/
@@ -91,20 +63,6 @@ FSE_PUBLIC_API unsigned    FSE_isError(size_t code);        /* tells if a return
 FSE_PUBLIC_API const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
 
 
-/*-*****************************************
-*  FSE advanced functions
-******************************************/
-/*! FSE_compress2() :
-    Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
-    Both parameters can be defined as '0' to mean : use default value
-    @return : size of compressed data
-    Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
-                     if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
-                     if FSE_isError(return), it's an error code.
-*/
-FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-
-
 /*-*****************************************
 *  FSE detailed API
 ******************************************/
@@ -164,8 +122,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
 /*! Constructor and Destructor of FSE_CTable.
     Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
 typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
-FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
-FSE_PUBLIC_API void        FSE_freeCTable (FSE_CTable* ct);
 
 /*! FSE_buildCTable():
     Builds `ct`, which must be already allocated, using FSE_createCTable().
@@ -241,23 +197,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
                            unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
                            const void* rBuffer, size_t rBuffSize, int bmi2);
 
-/*! Constructor and Destructor of FSE_DTable.
-    Note that its size depends on 'tableLog' */
 typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be more restrictive than void* */
-FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
-FSE_PUBLIC_API void        FSE_freeDTable(FSE_DTable* dt);
-
-/*! FSE_buildDTable():
-    Builds 'dt', which must be already allocated, using FSE_createDTable().
-    return : 0, or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
-
-/*! FSE_decompress_usingDTable():
-    Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
-    into `dst` which must be already allocated.
-    @return : size of regenerated data (necessarily <= `dstCapacity`),
-              or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
 
 /*!
 Tutorial :
@@ -320,16 +260,6 @@ If there is an error, the function will return an error code, which can be teste
 unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
 /**< same as FSE_optimalTableLog(), which used `minus==2` */
 
-/* FSE_compress_wksp() :
- * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
- * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
- */
-#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)   ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
-size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
-
-size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
-/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
-
 size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
 /**< build a fake FSE_CTable, designed to compress always the same symbolValue */
 
@@ -347,19 +277,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
 FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
 
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
-/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
-
-size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
-/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
-
 #define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
 #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
-/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
-
 size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
-/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
+ * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
 
 typedef enum {
    FSE_repeat_none,  /**< Cannot use the previous table */
diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
index ca894eb14..1e1c9f92d 100644
--- a/lib/common/fse_decompress.c
+++ b/lib/common/fse_decompress.c
@@ -56,19 +56,6 @@
 #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
 #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
 
-
-/* Function templates */
-FSE_DTable* FSE_createDTable (unsigned tableLog)
-{
-    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
-    return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
-}
-
-void FSE_freeDTable (FSE_DTable* dt)
-{
-    ZSTD_free(dt);
-}
-
 static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
 {
     void* const tdPtr = dt+1;   /* because *dt is unsigned, 32-bits aligned on 32-bits */
@@ -185,49 +172,6 @@ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsi
 /*-*******************************************************
 *  Decompression (Byte symbols)
 *********************************************************/
-size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
-{
-    void* ptr = dt;
-    FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
-    void* dPtr = dt + 1;
-    FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
-
-    DTableH->tableLog = 0;
-    DTableH->fastMode = 0;
-
-    cell->newState = 0;
-    cell->symbol = symbolValue;
-    cell->nbBits = 0;
-
-    return 0;
-}
-
-
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
-{
-    void* ptr = dt;
-    FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
-    void* dPtr = dt + 1;
-    FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
-    const unsigned tableSize = 1 << nbBits;
-    const unsigned tableMask = tableSize - 1;
-    const unsigned maxSV1 = tableMask+1;
-    unsigned s;
-
-    /* Sanity checks */
-    if (nbBits < 1) return ERROR(GENERIC);         /* min size */
-
-    /* Build Decoding Table */
-    DTableH->tableLog = (U16)nbBits;
-    DTableH->fastMode = 1;
-    for (s=0; s<maxSV1; s++) {
-        dinfo[s].newState = 0;
-        dinfo[s].symbol = (BYTE)s;
-        dinfo[s].nbBits = (BYTE)nbBits;
-    }
-
-    return 0;
-}
 
 FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
           void* dst, size_t maxDstSize,
@@ -291,26 +235,6 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
     return op-ostart;
 }
 
-
-size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
-                            const void* cSrc, size_t cSrcSize,
-                            const FSE_DTable* dt)
-{
-    const void* ptr = dt;
-    const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
-    const U32 fastMode = DTableH->fastMode;
-
-    /* select fast mode (static) */
-    if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
-    return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
-}
-
-
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
-{
-    return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
 typedef struct {
     short ncount[FSE_MAX_SYMBOL_VALUE + 1];
     FSE_DTable dtable[1]; /* Dynamically sized */
@@ -384,22 +308,4 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc,
     return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
 }
 
-
-typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
-    U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
-    return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
-}
-
-size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
-{
-    /* Static analyzer seems unable to understand this table will be properly initialized later */
-    U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
-    return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
-}
-#endif
-
-
 #endif   /* FSE_COMMONDEFS_ONLY */
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 6a9aa11e5..5d3770808 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -343,16 +343,6 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize,
 *  FSE Compression Code
 ****************************************************************/
 
-FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
-{
-    size_t size;
-    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
-    size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
-    return (FSE_CTable*)ZSTD_malloc(size);
-}
-
-void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
-
 /* provides the minimum logSize to safely represent a distribution */
 static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
 {
@@ -533,40 +523,6 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
     return tableLog;
 }
 
-
-/* fake FSE_CTable, for raw (uncompressed) input */
-size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
-{
-    const unsigned tableSize = 1 << nbBits;
-    const unsigned tableMask = tableSize - 1;
-    const unsigned maxSymbolValue = tableMask;
-    void* const ptr = ct;
-    U16* const tableU16 = ( (U16*) ptr) + 2;
-    void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1);   /* assumption : tableLog >= 1 */
-    FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
-    unsigned s;
-
-    /* Sanity checks */
-    if (nbBits < 1) return ERROR(GENERIC);             /* min size */
-
-    /* header */
-    tableU16[-2] = (U16) nbBits;
-    tableU16[-1] = (U16) maxSymbolValue;
-
-    /* Build table */
-    for (s=0; s<tableSize; s++)
-        tableU16[s] = (U16)(tableSize + s);
-
-    /* Build Symbol Transformation Table */
-    {   const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
-        for (s=0; s<=maxSymbolValue; s++) {
-            symbolTT[s].deltaNbBits = deltaNbBits;
-            symbolTT[s].deltaFindState = s-1;
-    }   }
-
-    return 0;
-}
-
 /* fake FSE_CTable, for rle input (always same symbol) */
 size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
 {
@@ -665,78 +621,4 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
 
 size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
 
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-/* FSE_compress_wksp() :
- * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
- * `wkspSize` size must be `(1<<tableLog)`.
- */
-size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
-{
-    BYTE* const ostart = (BYTE*) dst;
-    BYTE* op = ostart;
-    BYTE* const oend = ostart + dstSize;
-
-    unsigned count[FSE_MAX_SYMBOL_VALUE+1];
-    S16   norm[FSE_MAX_SYMBOL_VALUE+1];
-    FSE_CTable* CTable = (FSE_CTable*)workSpace;
-    size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
-    void* scratchBuffer = (void*)(CTable + CTableSize);
-    size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
-
-    /* init conditions */
-    if (wkspSize < FSE_COMPRESS_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
-    if (srcSize <= 1) return 0;  /* Not compressible */
-    if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
-    if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
-
-    /* Scan input and build symbol stats */
-    {   CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) );
-        if (maxCount == srcSize) return 1;   /* only a single symbol in src : rle */
-        if (maxCount == 1) return 0;         /* each symbol present maximum once => not compressible */
-        if (maxCount < (srcSize >> 7)) return 0;   /* Heuristic : not compressible enough */
-    }
-
-    tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
-    CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) );
-
-    /* Write table description header */
-    {   CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
-        op += nc_err;
-    }
-
-    /* Compress */
-    CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) );
-    {   CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) );
-        if (cSize == 0) return 0;   /* not enough space for compressed data */
-        op += cSize;
-    }
-
-    /* check compressibility */
-    if ( (size_t)(op-ostart) >= srcSize-1 ) return 0;
-
-    return op-ostart;
-}
-
-typedef struct {
-    FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
-    union {
-      U32 hist_wksp[HIST_WKSP_SIZE_U32];
-      BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
-    } workspace;
-} fseWkspMax_t;
-
-size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
-{
-    fseWkspMax_t scratchBuffer;
-    DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE));   /* compilation failures here means scratchBuffer is not large enough */
-    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
-    return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
-}
-
-size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
-    return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
-}
-#endif
-
 #endif   /* FSE_COMMONDEFS_ONLY */

From 2bde9fbf85d5044b0c95938f8a976420f0022945 Mon Sep 17 00:00:00 2001
From: daniellerozenblit <48103643+daniellerozenblit@users.noreply.github.com>
Date: Fri, 27 Jan 2023 16:58:53 -0500
Subject: [PATCH 167/197] Update lib/compress/zstd_compress.c

Co-authored-by: Nick Terrell <nickrterrell@gmail.com>
---
 lib/compress/zstd_compress.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index a83b1ccdd..579044dc9 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2576,7 +2576,9 @@ int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
         llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
         ofCodeTable[u] = (BYTE)ofCode;
         mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
-        longOffsets |= (ofCode >= STREAM_ACCUMULATOR_MIN);
+        assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN));
+        if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
+            longOffsets = 1;
     }
     if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
         llCodeTable[seqStorePtr->longLengthPos] = MaxLL;

From da589a134a40a19103458a6ccb8ada7bb09eea5d Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Fri, 27 Jan 2023 14:18:29 -0800
Subject: [PATCH 168/197] update CI

---
 .github/workflows/dev-long-tests.yml | 11 +++++++++++
 tests/Makefile                       |  2 +-
 tests/zstreamtest.c                  |  9 +++++----
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index 3fa2ceea2..c6e1e0be2 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -65,6 +65,17 @@ jobs:
     - name: thread sanitizer fuzztest
       run: CC=clang make tsan-fuzztest
 
+
+  big-tests-zstreamtest32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: zstream tests in 32bit mode, with big tests
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CC=clang make -C tests test-zstream32 FUZZERFLAGS="--big-tests"
+
   # lasts ~23mn
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
diff --git a/tests/Makefile b/tests/Makefile
index 8226176cc..3eed19ea8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -373,7 +373,7 @@ test-zstream: zstreamtest
 	$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
 
 test-zstream32: zstreamtest32
-	$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest32 -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
 
 test-longmatch: longmatch
 	$(QEMU_SYS) ./longmatch
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 3e5d854f1..2442593b7 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2224,7 +2224,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 
 
     DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
-    {
+    if (bigTests) {
         ZSTD_CCtx* cctx = ZSTD_createCCtx();
         ZSTD_DCtx* dctx = ZSTD_createDCtx();
 
@@ -2237,7 +2237,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
             size_t const kNbSequences = 4;
             ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
             void* const checkBuf = malloc(srcSize);
-            const size_t largeDictSize = 1 << 30;
+            const size_t largeDictSize = 1 << 25;
             ZSTD_CDict* cdict = NULL;
             ZSTD_DDict* ddict = NULL;
 
@@ -2255,8 +2255,8 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
             ZSTD_DCtx_refDDict(dctx, ddict);
 
             sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
-            sequences[1] = (ZSTD_Sequence) {1 << 29, 0, 3, 0};
-            sequences[2] = (ZSTD_Sequence) {1 << 29, 0, 9, 0};
+            sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
+            sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
             sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
 
             cSize = ZSTD_compressSequences(cctx, dst, dstSize,
@@ -3094,6 +3094,7 @@ int main(int argc, const char** argv)
 
             if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
             if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
+            if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
 
             argument++;
             while (*argument!=0) {

From b3b43f2893fa03da3b8004b449a7ec590f0e1e5b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 27 Jan 2023 11:14:56 -0800
Subject: [PATCH 169/197] Fix invalid assert in 32-bit decoding

The assert is only correct for valid sequences, so disable it for
everything execpt round trip fuzzers.
---
 lib/decompress/zstd_decompress_block.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 95a5e109b..0d934043b 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1241,7 +1241,13 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
                     offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
                     BIT_reloadDStream(&seqState->DStream);
                     if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                    /* This assert is only valid when decoding valid sequences.
+                     * It cal fail when we consume more bits than are in the bitstream,
+                     * which can happen on corruption.
+                     */
                     assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32);   /* to avoid another reload */
+#endif
                 } else {
                     offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
                     if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);

From 295724b515c01beb6e3583404854681d4eb3869b Mon Sep 17 00:00:00 2001
From: daniellerozenblit <48103643+daniellerozenblit@users.noreply.github.com>
Date: Sat, 28 Jan 2023 12:14:48 -0500
Subject: [PATCH 170/197] Update .github/workflows/dev-long-tests.yml

Co-authored-by: Nick Terrell <nickrterrell@gmail.com>
---
 .github/workflows/dev-long-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml
index c6e1e0be2..1c8c9ec55 100644
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@@ -74,7 +74,7 @@ jobs:
       run: |
         sudo apt-get -qqq update
         make libc6install
-        CC=clang make -C tests test-zstream32 FUZZERFLAGS="--big-tests"
+        CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests"
 
   # lasts ~23mn
   gcc-8-asan-ubsan-testzstd:

From dd7fdc98c8196dfd12594fc5349d8c260b150d0b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 30 Jan 2023 05:11:42 +0000
Subject: [PATCH 171/197] Bump github/codeql-action from 2.1.39 to 2.2.1

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.1.39 to 2.2.1.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/a34ca99b4610d924e04c68db79e503e1f79f9f02...3ebbd71c74ef574dbc558c82f70e52732c8b44fe)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/scorecards.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index a2277d761..6a9e5217e 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -59,6 +59,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@a34ca99b4610d924e04c68db79e503e1f79f9f02 # tag=v2.1.39
+        uses: github/codeql-action/upload-sarif@3ebbd71c74ef574dbc558c82f70e52732c8b44fe # tag=v2.2.1
         with:
           sarif_file: results.sarif

From 66fae56c860fdbf195a45c2b2372692460f07fa7 Mon Sep 17 00:00:00 2001
From: Danielle Rozenblit <drozenblit@fb.com>
Date: Mon, 30 Jan 2023 06:26:03 -0800
Subject: [PATCH 172/197] remove big test around large offset with small window
 size

---
 tests/zstreamtest.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 2442593b7..db1e25188 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2224,7 +2224,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 
 
     DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
-    if (bigTests) {
+    {
         ZSTD_CCtx* cctx = ZSTD_createCCtx();
         ZSTD_DCtx* dctx = ZSTD_createDCtx();
 

From 2f74507bbd0e8aca907a2842a311cc24ec815cf2 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 30 Jan 2023 11:15:15 -0800
Subject: [PATCH 173/197] Simplify 32-bit long offsets decoding logic

The previous code had an issue when `bitsConsumed == 32` it would read 0
bits for the `ofBits` read, which violates the precondition of
`BIT_readBitsFast()`. This can happen when the stream is corrupted.

Fix thie issue by always reading the maximum possible number of extra
bits. I've measured neutral decoding performance, likely because this
branch is unlikely, but this should be faster anyways. And if not, it is
only 32-bit decoding, so performance isn't as critical.

Credit to OSS-Fuzz
---
 lib/common/zstd_internal.h             |  2 ++
 lib/decompress/zstd_decompress_block.c | 21 +++++++++++----------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 8878aa100..37836dc70 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -113,6 +113,8 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
 #define LLFSELog    9
 #define OffFSELog   8
 #define MaxFSELog  MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
+#define MaxMLBits 16
+#define MaxLLBits 16
 
 #define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
 /* Each table cannot take more than #symbols * FSELog bits */
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 0d934043b..a2a6eb3ed 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1220,6 +1220,10 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
         U32 const llnbBits = llDInfo->nbBits;
         U32 const mlnbBits = mlDInfo->nbBits;
         U32 const ofnbBits = ofDInfo->nbBits;
+
+        assert(llBits <= MaxLLBits);
+        assert(mlBits <= MaxMLBits);
+        assert(ofBits <= MaxOff);
         /*
          * As gcc has better branch and block analyzers, sometimes it is only
          * valuable to mark likeliness for clang, it gives around 3-4% of
@@ -1235,19 +1239,16 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
     #endif
                 ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
                 ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
-                assert(ofBits <= MaxOff);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
                 if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
-                    U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed);
+                    /* Always read extra bits, this keeps the logic simple,
+                     * avoids branches, and avoids accidentally reading 0 bits.
+                     */
+                    U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
                     offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
                     BIT_reloadDStream(&seqState->DStream);
-                    if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
-                    /* This assert is only valid when decoding valid sequences.
-                     * It cal fail when we consume more bits than are in the bitstream,
-                     * which can happen on corruption.
-                     */
-                    assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32);   /* to avoid another reload */
-#endif
+                    offset += BIT_readBitsFast(&seqState->DStream, extraBits);
                 } else {
                     offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
                     if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);

From 39ceef27f92f09fa3131e89cff083c7623d0d0a1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 30 Jan 2023 19:06:39 -0800
Subject: [PATCH 174/197] bump version number to v1.5.4

start preparation for release
---
 CHANGELOG            |   4 +-
 doc/zstd_manual.html | 266 ++++++++++++++++++++++++++++++++++---------
 lib/zstd.h           |   6 +-
 3 files changed, 218 insertions(+), 58 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index e8c40f302..6ddec4107 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,6 @@
-v1.5.3 (June, 2022)
+v1.5.4
+
+v1.5.3 (June, 2022) (unpublished)
 perf: 5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg)
 perf: 5-10% faster streaming compression at levels 1-2 (#3114, @embg)
 perf: Remove branch in ZSTD_fast_noDict (#3129, @felixhandte)
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index cca287b72..2ea5c2010 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.5.3 Manual</title>
+<title>zstd 1.5.4 Manual</title>
 </head>
 <body>
-<h1>zstd 1.5.3 Manual</h1>
+<h1>zstd 1.5.4 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -121,8 +121,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
    note 6 : This function replaces ZSTD_getDecompressedSize() 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_API
-ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+<pre><b>ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
 </b><p>  NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
   Both functions work the same way, but ZSTD_getDecompressedSize() blends
@@ -139,8 +139,30 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
         or an error code if input is invalid 
 </p></pre><BR>
 
-<h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
-size_t      ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
+<h3>Helper functions</h3><pre></pre><b><pre></b>/* ZSTD_compressBound() :<b>
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
+size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
+</b>/* ZSTD_isError() :<b>
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
 unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
 const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
 int         ZSTD_minCLevel(void);               </b>/*!< minimum negative compression level allowed, requires v1.4.0+ */<b>
@@ -362,6 +384,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableMatchFinderFallback
+     * ZSTD_c_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -381,7 +405,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_c_experimentalParam13=1010,
      ZSTD_c_experimentalParam14=1011,
      ZSTD_c_experimentalParam15=1012,
-     ZSTD_c_experimentalParam16=1013
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015
 } ZSTD_cParameter;
 </b></pre><BR>
 <pre><b>typedef struct {
@@ -444,7 +470,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                   They will be used to compress next frame.
                   Resetting session never fails.
   - The parameters : changes all parameters back to "default".
-                  This removes any reference to any dictionary too.
+                  This also removes any reference to any dictionary or external matchfinder.
                   Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
                   otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
   - Both : similar to resetting the session, followed by resetting parameters.
@@ -483,13 +509,15 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      * ZSTD_d_stableOutBuffer
      * ZSTD_d_forceIgnoreChecksum
      * ZSTD_d_refMultipleDDicts
+     * ZSTD_d_disableHuffmanAssembly
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
      ZSTD_d_experimentalParam2=1001,
      ZSTD_d_experimentalParam3=1002,
-     ZSTD_d_experimentalParam4=1003
+     ZSTD_d_experimentalParam4=1003,
+     ZSTD_d_experimentalParam5=1004
 
 } ZSTD_dParameter;
 </b></pre><BR>
@@ -658,6 +686,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
      ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+
+ Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ to compress with a dictionary.
  
 </p></pre><BR>
 
@@ -693,6 +724,33 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 </pre></b><BR>
 <h3>Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
+<pre><b>size_t ZSTD_initDStream(ZSTD_DStream* zds);
+</b><p> Initialize/reset DStream state for new decompression operation.
+ Call before new decompression operation using same DStream.
+
+ Note : This function is redundant with the advanced API and equivalent to:
+     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+     ZSTD_DCtx_refDDict(zds, NULL);
+ 
+</p></pre><BR>
+
+<pre><b>size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+</b><p> Streaming decompression function.
+ Call repetitively to consume full input updating it as necessary.
+ Function will update both input and output `pos` fields exposing current state via these fields:
+ - `input.pos < input.size`, some input remaining and caller should provide remaining input
+   on the next call.
+ - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+   call ZSTD_decompressStream() again to flush remaining data to output.
+ Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+
+ @return : 0 when a frame is completely decoded and fully flushed,
+           or an error code, which can be tested using ZSTD_isError(),
+           or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ 
+</p></pre><BR>
+
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_DStreamOutSize(void);   </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
@@ -818,8 +876,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
            meaning "return to no-dictionary mode".
-  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
-           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+           until parameters are reset, a new dictionary is loaded, or the dictionary
+           is explicitly invalidated by loading a NULL dictionary.
   Note 2 : Loading a dictionary involves building tables.
            It's also a CPU consuming operation, with non-negligible impact on latency.
            Tables are dependent on compression parameters, and for this reason,
@@ -832,7 +891,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-</b><p>  Reference a prepared dictionary, to be used for all next compressed frames.
+</b><p>  Reference a prepared dictionary, to be used for all future compressed frames.
   Note that compression parameters are enforced from within CDict,
   and supersede any compression parameter previously set within CCtx.
   The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@@ -867,9 +926,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-</b><p>  Create an internal DDict from dict buffer,
-  to be used to decompress next frames.
-  The dictionary remains valid for all future frames, until explicitly invalidated.
+</b><p>  Create an internal DDict from dict buffer, to be used to decompress all future frames.
+  The dictionary remains valid for all future frames, until explicitly invalidated, or
+  a new dictionary is loaded.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
             meaning "return to no-dictionary mode".
@@ -893,9 +952,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
   The memory for the table is allocated on the first call to refDDict, and can be
   freed with ZSTD_freeDCtx().
 
+  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+  will be managed, and referencing a dictionary effectively "discards" any previous one.
+
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Note 1 : Currently, only one dictionary can be managed.
-           Referencing a new dictionary effectively "discards" any previous one.
   Special: referencing a NULL DDict means "return to no-dictionary mode".
   Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
  
@@ -1131,11 +1191,65 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small) 
 </p></pre><BR>
 
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+</b><p> Zstd supports in-place decompression, where the input and output buffers overlap.
+ In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ and the input buffer must be at the end of the output buffer.
+
+  _______________________ Output Buffer ________________________
+ |                                                              |
+ |                                        ____ Input Buffer ____|
+ |                                       |                      |
+ v                                       v                      v
+ |---------------------------------------|-----------|----------|
+ ^                                                   ^          ^
+ |___________________ Output_Size ___________________|_ Margin _|
+
+ NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ ZSTD_decompressDCtx().
+ NOTE: This function supports multi-frame input.
+
+ @param src The compressed frame(s)
+ @param srcSize The size of the compressed frame(s)
+ @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ 
+</p></pre><BR>
+
+<pre><b>#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)(                                              \
+        ZSTD_FRAMEHEADERSIZE_MAX                                                              </b>/* Frame header */ + \<b>
+        4                                                                                         </b>/* checksum */ + \<b>
+        ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) </b>/* 3 bytes per block */ + \<b>
+        (blockSize)                                                                    </b>/* One block of margin */   \<b>
+    ))
+</b><p> Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ the compressed frame, compute it from the original size and the blockSizeLog.
+ See ZSTD_decompressionMargin() for details.
+
+ WARNING: This macro does not support multi-frame input, the input must be a single
+ zstd frame. If you need that support use the function, or implement it yourself.
+
+ @param originalSize The original uncompressed size of the data.
+ @param blockSize    The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+                     Unless you explicitly set the windowLog smaller than
+                     ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ 
+</p></pre><BR>
+
 <pre><b>typedef enum {
   ZSTD_sf_noBlockDelimiters = 0,         </b>/* Representation of ZSTD_Sequence has no block delimiters, sequences only */<b>
   ZSTD_sf_explicitBlockDelimiters = 1    </b>/* Representation of ZSTD_Sequence contains explicit block delimiters */<b>
 } ZSTD_sequenceFormat_e;
 </b></pre><BR>
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+</b><p> `srcSize` : size of the input buffer
+  @return : upper-bound for the number of sequences that can be generated
+            from a buffer of srcSize bytes
+
+  note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ 
+</p></pre><BR>
+
 <pre><b></b><p> Generate sequences using ZSTD_compress2(), given a source buffer.
 
  Each block will end with a dummy sequence
@@ -1260,8 +1374,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
   and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
   Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
 
-  Note 2 : only single-threaded compression is supported.
+  Note : only single-threaded compression is supported.
   ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+
+  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the external matchfinder API at this time.
+  Size estimates assume that no external matchfinder is registered.
  
 </p></pre><BR>
 
@@ -1281,7 +1398,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, si
   or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
   Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
          an internal ?Dict will be created, which additional size is not estimated here.
-         In this case, get total size by adding ZSTD_estimate?DictSize 
+         In this case, get total size by adding ZSTD_estimate?DictSize
+  Note 2 : only single-threaded compression is supported.
+  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the external matchfinder API at this time.
+  Size estimates assume that no external matchfinder is registered.
+ 
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
@@ -1381,20 +1503,27 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPo
   This function never fails (wide contract) 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_compress2")
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+</b><p>  Set all parameters provided within @cparams into the working @cctx.
+  Note : if modifying parameters during compression (MT mode only),
+         note that changes to the .windowLog parameter will be ignored.
+ @return 0 on success, or an error code (can be checked with ZSTD_isError()) 
+</p></pre><BR>
+
+<pre><b>ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
-                              void* dst, size_t dstCapacity,
-                        const void* src, size_t srcSize,
-                        const void* dict,size_t dictSize,
-                              ZSTD_parameters params);
+                  void* dst, size_t dstCapacity,
+            const void* src, size_t srcSize,
+            const void* dict,size_t dictSize,
+                  ZSTD_parameters params);
 </b><p>  Note : this function is now DEPRECATED.
          It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
   This prototype will generate compilation warnings. 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                   void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize,
@@ -1555,8 +1684,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/*
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
 size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 </b><p>  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
   Instruct the decoder context about what kind of data to decode next.
@@ -1582,8 +1711,8 @@ size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 <BR></pre>
 
 <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
              int compressionLevel,
              unsigned long long pledgedSrcSize);
@@ -1600,8 +1729,8 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
          const void* dict, size_t dictSize,
                int compressionLevel);
@@ -1618,8 +1747,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         const void* dict, size_t dictSize,
               ZSTD_parameters params,
@@ -1640,8 +1769,8 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
 </b><p> This function is DEPRECATED, and equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1652,8 +1781,8 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                    const ZSTD_CDict* cdict,
                          ZSTD_frameParameters fParams,
@@ -1674,8 +1803,8 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 </b><p> This function is DEPRECATED, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1721,32 +1850,32 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 </p></pre><BR>
 
 <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+<pre><b>ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
 
  note: no dictionary will be used if dict == NULL or dictSize < 8
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+<pre><b>ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_refDDict(zds, ddict);
 
  note : ddict is referenced, it must outlive decompression session
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+<pre><b>ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
  re-use decompression parameters from previous init; saves dictionary loading
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
@@ -1764,7 +1893,6 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
   Start by initializing a context.
   Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
-  It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
 
   Then, consume your input using ZSTD_compressContinue().
   There are some important considerations to keep in mind when using this advanced function :
@@ -1788,8 +1916,9 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 <h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
-ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </pre></b><BR>
+<pre><b>size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
+</b></pre><BR>
 <pre><b>size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </b></pre><BR>
 <a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
@@ -1801,8 +1930,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepar
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
   Data fragment must be large enough to ensure successful decoding.
  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
-  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
-           >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
   It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@@ -1821,7 +1950,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepar
 
   The most memory efficient way is to use a round buffer of sufficient size.
   Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
-  which can @return an error code if required value is too large for current system (in 32-bits mode).
+  which can return an error code if required value is too large for current system (in 32-bits mode).
   In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
   up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
   which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@@ -1841,7 +1970,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepar
   ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
 
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
   It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
   It can also be an error code, which can be tested with ZSTD_isError().
 
@@ -1872,6 +2001,8 @@ typedef struct {
     unsigned headerSize;
     unsigned dictID;
     unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
 } ZSTD_frameHeader;
 </pre></b><BR>
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input */<b>
@@ -1899,7 +2030,6 @@ ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowS
     - It is necessary to init context before starting
       + compression : any ZSTD_compressBegin*() variant, including with dictionary
       + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
-      + copyCCtx() and copyDCtx() can be used too
     - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
       + If input is larger than a block size, it's necessary to split input data into multiple blocks
       + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
@@ -1920,5 +2050,33 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_
 ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */<b>
 </pre></b><BR>
+<pre><b>ZSTDLIB_STATIC_API void
+ZSTD_registerExternalMatchFinder(
+  ZSTD_CCtx* cctx,
+  void* externalMatchState,
+  ZSTD_externalMatchFinder_F* externalMatchFinder
+);
+</b><p> Instruct zstd to use an external matchfinder function.
+
+ The externalMatchState must be initialized by the caller, and the caller is
+ responsible for managing its lifetime. This parameter is sticky across
+ compressions. It will remain set until the user explicitly resets compression
+ parameters.
+
+ External matchfinder registration is considered to be an "advanced parameter",
+ part of the "advanced API". This means it will only have an effect on
+ compression APIs which respect advanced parameters, such as compress2() and
+ compressStream(). Older compression APIs such as compressCCtx(), which predate
+ the introduction of "advanced parameters", will ignore any external matchfinder
+ setting.
+
+ The external matchfinder can be "cleared" by registering a NULL external
+ matchfinder function pointer. This removes all limitations described above in
+ the "LIMITATIONS" section of the API docs.
+
+ The user is strongly encouraged to read the full API documentation (above)
+ before calling this function. 
+</p></pre><BR>
+
 </html>
 </body>
diff --git a/lib/zstd.h b/lib/zstd.h
index d9e26ee13..237142289 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -14,7 +14,7 @@ extern "C" {
 #ifndef ZSTD_H_235446
 #define ZSTD_H_235446
 
-/* ======   Dependency   ======*/
+/* ======   Dependencies   ======*/
 #include <limits.h>   /* INT_MAX */
 #include <stddef.h>   /* size_t */
 
@@ -106,7 +106,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    5
-#define ZSTD_VERSION_RELEASE  3
+#define ZSTD_VERSION_RELEASE  4
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 
 /*! ZSTD_versionNumber() :
@@ -1646,7 +1646,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
  *  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
  *         an internal ?Dict will be created, which additional size is not estimated here.
- *         In this case, get total size by adding ZSTD_estimate?DictSize 
+ *         In this case, get total size by adding ZSTD_estimate?DictSize
  *  Note 2 : only single-threaded compression is supported.
  *  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
  *  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the external matchfinder API at this time.

From af09777b2488ec3af191e541ebe4fc220316ffa8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 31 Jan 2023 08:59:27 -0800
Subject: [PATCH 175/197] ensure that benchmark mode can only be invoked with
 zstd format

fix #3463
---
 programs/zstdcli.c | 38 ++++++++++++++++++++++++--------------
 tests/playTests.sh |  6 ++++++
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 39f8b34fe..660e66bb6 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -852,6 +852,7 @@ int main(int argCount, const char* argv[])
         contentSize=1,
         removeSrcFile=0;
     ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
+    FIO_compressionType_t cType = FIO_zstdCompression;
     unsigned nbWorkers = 0;
     double compressibility = 0.5;
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
@@ -911,17 +912,17 @@ int main(int argCount, const char* argv[])
     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
     if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZ)) {   /* behave like gzip */
-        suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); removeSrcFile=1;
+        suffix = GZ_EXTENSION; cType = FIO_gzipCompression; removeSrcFile=1;
         dictCLevel = cLevel = 6;  /* gzip default is -6 */
     }
     if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; }                                                     /* behave like gunzip, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); removeSrcFile=1; }    /* behave like lzma */
-    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); removeSrcFile=1; }          /* behave like xz */
-    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); removeSrcFile=1; }     /* behave like unxz, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); }                        /* behave like lz4 */
-    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); }                    /* behave like unlz4, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; removeSrcFile=1; }    /* behave like lzma */
+    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; cType = FIO_xzCompression; removeSrcFile=1; }          /* behave like xz */
+    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; cType = FIO_xzCompression; removeSrcFile=1; }     /* behave like unxz, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; cType = FIO_lz4Compression; }                        /* behave like lz4 */
+    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; cType = FIO_lz4Compression; }                    /* behave like unlz4, also supports multiple formats */
     memset(&compressionParams, 0, sizeof(compressionParams));
 
     /* init crash handler */
@@ -982,20 +983,20 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = ZSTD_ps_enable; continue; }
                 if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
                 if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
-                if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
+                if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; }
 #ifdef ZSTD_GZCOMPRESS
-                if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; }
+                if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; }
                 if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
                     if (!strcmp(argument, "--best")) { dictCLevel = cLevel = 9; continue; }
                     if (!strcmp(argument, "--no-name")) { /* ignore for now */; continue; }
                 }
 #endif
 #ifdef ZSTD_LZMACOMPRESS
-                if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression);  continue; }
-                if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression);  continue; }
+                if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; continue; }
+                if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; cType = FIO_xzCompression; continue; }
 #endif
 #ifdef ZSTD_LZ4COMPRESS
-                if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression);  continue; }
+                if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; cType = FIO_lz4Compression; continue; }
 #endif
                 if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
                 if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_ps_enable; continue; }
@@ -1051,7 +1052,7 @@ int main(int argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
                 if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
                 if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
-                if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
+                if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
                 if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
                 if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
                 if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
@@ -1358,6 +1359,10 @@ int main(int argCount, const char* argv[])
     /* Check if benchmark is selected */
     if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
+        if (cType != FIO_zstdCompression) {
+            DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
+            CLEAN_RETURN(1);
+        }
         benchParams.blockSize = blockSize;
         benchParams.nbWorkers = (int)nbWorkers;
         benchParams.realTime = (unsigned)setRealTimePrio;
@@ -1529,6 +1534,7 @@ int main(int argCount, const char* argv[])
     FIO_setMemLimit(prefs, memLimit);
     if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
+        FIO_setCompressionType(prefs, cType);
         FIO_setContentSize(prefs, contentSize);
         FIO_setNbWorkers(prefs, (int)nbWorkers);
         FIO_setBlockSize(prefs, (int)blockSize);
@@ -1573,7 +1579,11 @@ int main(int argCount, const char* argv[])
         else
             operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
-        (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; (void)useRowMatchFinder; /* not used when ZSTD_NOCOMPRESS set */
+        /* these variables are only used when compression mode is enabled */
+        (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable;
+        (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode;
+        (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint;
+        (void)ZSTD_strategyMap; (void)useRowMatchFinder; (void)cType;
         DISPLAYLEVEL(1, "Compression not supported \n");
 #endif
     } else {  /* decompression or test */
diff --git a/tests/playTests.sh b/tests/playTests.sh
index e064c86df..5d78e9e7d 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -1218,6 +1218,12 @@ println "benchmark decompression only"
 zstd -f tmp1
 zstd -b -d -i0 tmp1.zst
 
+GZIPMODE=1
+zstd --format=gzip -V || GZIPMODE=0
+if [ $GZIPMODE -eq 1 ]; then
+    println "benchmark mode is only compatible with zstd"
+    zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!"
+fi
 
 println "\n===>  zstd compatibility tests "
 

From 64052ef57d09a9117ccf1c535d3387f0d5270ca9 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Tue, 31 Jan 2023 13:55:48 -0500
Subject: [PATCH 176/197] Guard against invalid sequences from external
 matchfinders (#3465)

---
 lib/compress/zstd_compress.c | 44 +++++++++++++++++++--------
 tests/external_matchfinder.c | 23 ++++++++++++++
 tests/external_matchfinder.h |  6 +++-
 tests/fuzz/zstd_helpers.c    |  1 +
 tests/zstreamtest.c          | 59 ++++++++++++++++++++++--------------
 5 files changed, 96 insertions(+), 37 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 579044dc9..de2db8221 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -277,14 +277,8 @@ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
     return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
 }
 
-/* Enables validation for external sequences in debug builds. */
 static int ZSTD_resolveExternalSequenceValidation(int mode) {
-#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2)
-    (void)mode;
-    return 1;
-#else
     return mode;
-#endif
 }
 
 /* Resolves maxBlockSize to the default if no value is present. */
@@ -3050,6 +3044,23 @@ static size_t ZSTD_postProcessExternalMatchFinderResult(
     }
 }
 
+/* ZSTD_fastSequenceLengthSum() :
+ * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*.
+ * Similar to another function in zstd_compress.c (determine_blockSize),
+ * except it doesn't check for a block delimiter to end summation.
+ * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
+ * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */
+static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) {
+    size_t matchLenSum, litLenSum, i;
+    matchLenSum = 0;
+    litLenSum = 0;
+    for (i = 0; i < seqBufSize; i++) {
+        litLenSum += seqBuf[i].litLength;
+        matchLenSum += seqBuf[i].matchLength;
+    }
+    return litLenSum + matchLenSum;
+}
+
 typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
 
 static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
@@ -3167,8 +3178,15 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                 /* Return early if there is no error, since we don't need to worry about last literals */
                 if (!ZSTD_isError(nbPostProcessedSeqs)) {
                     ZSTD_sequencePosition seqPos = {0,0,0};
-                    ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
-                        zc, &seqPos, zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs, src, srcSize
+                    size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs);
+                    RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!");
+                    FORWARD_IF_ERROR(
+                        ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
+                            zc, &seqPos,
+                            zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs,
+                            src, srcSize
+                        ),
+                        "Failed to copy external sequences to seqStore!"
                     );
                     ms->ldmSeqStore = NULL;
                     DEBUGLOG(5, "Copied %lu sequences from external matchfinder to internal seqStore.", (unsigned long)nbExternalSeqs);
@@ -6286,7 +6304,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
         ip += inSeqs[idx].litLength;
         seqPos->posInSrc += inSeqs[idx].litLength;
     }
-    RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!");
+    RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
     seqPos->idx = idx+1;
     return 0;
 }
@@ -6444,13 +6462,13 @@ blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD
         blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength;
         if (end) {
             if (inSeqs[spos].matchLength != 0)
-                RETURN_ERROR(corruption_detected, "delimiter format error : both matchlength and offset must be == 0");
+                RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0");
             break;
         }
         spos++;
     }
     if (!end)
-        RETURN_ERROR(corruption_detected, "Reached end of sequences without finding a block delimiter");
+        RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter");
     return blockSize;
 }
 
@@ -6471,9 +6489,9 @@ static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
     {   size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos);
         FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters");
         if (explicitBlockSize > blockSize)
-            RETURN_ERROR(corruption_detected, "sequences incorrectly define a too large block");
+            RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block");
         if (explicitBlockSize > remaining)
-            RETURN_ERROR(srcSize_wrong, "sequences define a frame longer than source");
+            RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source");
         return explicitBlockSize;
     }
 }
diff --git a/tests/external_matchfinder.c b/tests/external_matchfinder.c
index 8ae76d519..97c47caff 100644
--- a/tests/external_matchfinder.c
+++ b/tests/external_matchfinder.c
@@ -108,6 +108,29 @@ size_t zstreamExternalMatchFinder(
                 compressionLevel,
                 windowSize
             );
+        case EMF_INVALID_OFFSET:
+            outSeqs[0].offset = 1 << 20;
+            outSeqs[0].matchLength = 4;
+            outSeqs[0].litLength = (U32)(srcSize - 4);
+            return 1;
+        case EMF_INVALID_MATCHLEN:
+            outSeqs[0].offset = 1;
+            outSeqs[0].matchLength = (U32)(srcSize);
+            outSeqs[0].litLength = 1;
+            return 1;
+        case EMF_INVALID_LITLEN:
+            outSeqs[0].offset = 0;
+            outSeqs[0].matchLength = 0;
+            outSeqs[0].litLength = (U32)(srcSize + 1);
+            return 1;
+        case EMF_INVALID_LAST_LITS:
+            outSeqs[0].offset = 1;
+            outSeqs[0].matchLength = 1;
+            outSeqs[0].litLength = 1;
+            outSeqs[1].offset = 0;
+            outSeqs[1].matchLength = 0;
+            outSeqs[1].litLength = (U32)(srcSize - 1);
+            return 2;
         case EMF_SMALL_ERROR:
             return outSeqsCapacity + 1;
         case EMF_BIG_ERROR:
diff --git a/tests/external_matchfinder.h b/tests/external_matchfinder.h
index e4f7c95f0..7550bbceb 100644
--- a/tests/external_matchfinder.h
+++ b/tests/external_matchfinder.h
@@ -20,7 +20,11 @@ typedef enum {
     EMF_ONE_BIG_SEQ = 1,
     EMF_LOTS_OF_SEQS = 2,
     EMF_BIG_ERROR = 3,
-    EMF_SMALL_ERROR = 4
+    EMF_SMALL_ERROR = 4,
+    EMF_INVALID_OFFSET = 5,
+    EMF_INVALID_MATCHLEN = 6,
+    EMF_INVALID_LITLEN = 7,
+    EMF_INVALID_LAST_LITS = 8
 } EMF_testCase;
 
 size_t zstreamExternalMatchFinder(
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 43a26405f..310012cb5 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -128,6 +128,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
     setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
     setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
+    setRand(cctx, ZSTD_c_validateSequences, 0, 1, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index db1e25188..aff847b4d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1873,48 +1873,61 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
          * check that the reference is preserved across compressions */
         ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
 
-        for (enableFallback = 0; enableFallback < 1; enableFallback++) {
+        for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
             size_t testCaseId;
+            size_t const numTestCases = 9;
 
-            EMF_testCase const EMF_successCases[] = {
+            EMF_testCase const testCases[] = {
                 EMF_ONE_BIG_SEQ,
                 EMF_LOTS_OF_SEQS,
-            };
-            size_t const EMF_numSuccessCases = 2;
-
-            EMF_testCase const EMF_failureCases[] = {
                 EMF_ZERO_SEQS,
                 EMF_BIG_ERROR,
                 EMF_SMALL_ERROR,
+                EMF_INVALID_OFFSET,
+                EMF_INVALID_MATCHLEN,
+                EMF_INVALID_LITLEN,
+                EMF_INVALID_LAST_LITS
             };
-            size_t const EMF_numFailureCases = 3;
 
-            /* Test external matchfinder success scenarios */
-            for (testCaseId = 0; testCaseId < EMF_numSuccessCases; testCaseId++) {
+            ZSTD_ErrorCode const errorCodes[] = {
+                ZSTD_error_no_error,
+                ZSTD_error_no_error,
+                ZSTD_error_externalMatchFinder_failed,
+                ZSTD_error_externalMatchFinder_failed,
+                ZSTD_error_externalMatchFinder_failed,
+                ZSTD_error_externalSequences_invalid,
+                ZSTD_error_externalSequences_invalid,
+                ZSTD_error_externalSequences_invalid,
+                ZSTD_error_externalSequences_invalid
+            };
+
+            for (testCaseId = 0; testCaseId < numTestCases; testCaseId++) {
                 size_t res;
-                externalMatchState = EMF_successCases[testCaseId];
+
+                int const compressionShouldSucceed = (
+                    (errorCodes[testCaseId] == ZSTD_error_no_error) ||
+                    (enableFallback && errorCodes[testCaseId] == ZSTD_error_externalMatchFinder_failed)
+                );
+
+                int const testWithSequenceValidation = (
+                    testCases[testCaseId] == EMF_INVALID_OFFSET
+                );
+
+                externalMatchState = testCases[testCaseId];
+
                 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
                 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
                 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
-                CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
-                CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
-                CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
-            }
 
-            /* Test external matchfinder failure scenarios */
-            for (testCaseId = 0; testCaseId < EMF_numFailureCases; testCaseId++) {
-                size_t res;
-                externalMatchState = EMF_failureCases[testCaseId];
-                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
-                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
-                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
-                if (enableFallback) {
+                if (compressionShouldSucceed) {
+                    CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
                     CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
                     CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
                 } else {
                     CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
                     CHECK(
-                        ZSTD_getErrorCode(res) != ZSTD_error_externalMatchFinder_failed,
+                        ZSTD_getErrorCode(res) != errorCodes[testCaseId],
                         "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
                     );
                 }

From 7f8189ca57741f89b98b06fabfeffb2c8b8683b5 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Tue, 31 Jan 2023 13:51:32 -0800
Subject: [PATCH 177/197] add ZSTD_c_fastExternalSequenceParsing cctxParam

---
 lib/compress/zstd_compress.c          | 79 ++++++++++++++++++++++++---
 lib/compress/zstd_compress_internal.h |  7 ++-
 lib/zstd.h                            | 40 +++++++++++---
 tests/fuzz/zstd_helpers.c             |  1 +
 4 files changed, 109 insertions(+), 18 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index de2db8221..3ad6f05a2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -290,6 +290,15 @@ static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
     }
 }
 
+static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) {
+    if (value != ZSTD_ps_auto) return value;
+    if (cLevel < 10) {
+        return ZSTD_ps_disable;
+    } else {
+        return ZSTD_ps_enable;
+    }
+}
+
 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
  * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
@@ -315,6 +324,8 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
     cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
     cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
+    cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes,
+                                                                             cctxParams.compressionLevel);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -381,6 +392,7 @@ ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
     cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
     cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
+    cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -613,6 +625,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
         return bounds;
 
+    case ZSTD_c_searchForExternalRepcodes:
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
+        return bounds;
+
     default:
         bounds.error = ERROR(parameter_unsupported);
         return bounds;
@@ -680,6 +697,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_prefetchCDictTables:
     case ZSTD_c_enableMatchFinderFallback:
     case ZSTD_c_maxBlockSize:
+    case ZSTD_c_searchForExternalRepcodes:
     default:
         return 0;
     }
@@ -738,6 +756,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_prefetchCDictTables:
     case ZSTD_c_enableMatchFinderFallback:
     case ZSTD_c_maxBlockSize:
+    case ZSTD_c_searchForExternalRepcodes:
         break;
 
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -981,6 +1000,11 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->maxBlockSize = value;
         return CCtxParams->maxBlockSize;
 
+    case ZSTD_c_searchForExternalRepcodes:
+        BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value);
+        CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value;
+        return CCtxParams->searchForExternalRepcodes;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -1122,6 +1146,9 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_maxBlockSize:
         *value = (int)CCtxParams->maxBlockSize;
         break;
+    case ZSTD_c_searchForExternalRepcodes:
+        *value = (int)CCtxParams->searchForExternalRepcodes;
+        break;
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
@@ -3184,7 +3211,8 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                         ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
                             zc, &seqPos,
                             zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs,
-                            src, srcSize
+                            src, srcSize,
+                            zc->appliedParams.searchForExternalRepcodes
                         ),
                         "Failed to copy external sequences to seqStore!"
                     );
@@ -6000,6 +6028,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
     params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
     params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
+    params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel);
 
 #ifdef ZSTD_MULTITHREAD
     /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
@@ -6259,9 +6288,11 @@ size_t
 ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                               ZSTD_sequencePosition* seqPos,
                                         const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                        const void* src, size_t blockSize)
+                                        const void* src, size_t blockSize,
+                                        ZSTD_paramSwitch_e externalRepSearch)
 {
     U32 idx = seqPos->idx;
+    U32 const startIdx = idx;
     BYTE const* ip = (BYTE const*)(src);
     const BYTE* const iend = ip + blockSize;
     repcodes_t updatedRepcodes;
@@ -6279,10 +6310,16 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
     ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
     for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
         U32 const litLength = inSeqs[idx].litLength;
-        U32 const ll0 = (litLength == 0);
         U32 const matchLength = inSeqs[idx].matchLength;
-        U32 const offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
-        ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+        U32 offBase;
+
+        if (externalRepSearch == ZSTD_ps_disable) {
+            offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset);
+        } else {
+            U32 const ll0 = (litLength == 0);
+            offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
+            ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+        }
 
         DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
         if (cctx->appliedParams.validateSequences) {
@@ -6296,6 +6333,29 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
     }
+
+    /* If we skipped repcode search while parsing, we need to update repcodes now */
+    assert(idx >= startIdx);
+    if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
+        U32* const rep = updatedRepcodes.rep;
+        U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */
+
+        if (lastSeqIdx >= startIdx + 2) {
+            rep[2] = inSeqs[lastSeqIdx - 2].offset;
+            rep[1] = inSeqs[lastSeqIdx - 1].offset;
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        } else if (lastSeqIdx == startIdx + 1) {
+            rep[2] = rep[0];
+            rep[1] = inSeqs[lastSeqIdx - 1].offset;
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        } else {
+            assert(lastSeqIdx == startIdx);
+            rep[2] = rep[1];
+            rep[1] = rep[0];
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        }
+    }
+
     ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
 
     if (inSeqs[idx].litLength) {
@@ -6312,7 +6372,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
 size_t
 ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                    const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                   const void* src, size_t blockSize)
+                                   const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch)
 {
     U32 idx = seqPos->idx;
     U32 startPosInSequence = seqPos->posInSequence;
@@ -6324,6 +6384,9 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
     U32 bytesAdjustment = 0;
     U32 finalMatchSplit = 0;
 
+    /* TODO(embg) support fast parsing mode in noBlockDelim mode */
+    (void)externalRepSearch;
+
     if (cctx->cdict) {
         dictSize = cctx->cdict->dictContentSize;
     } else if (cctx->prefixDict.dict) {
@@ -6431,7 +6494,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
 
 typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                        const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                       const void* src, size_t blockSize);
+                                       const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
 static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
 {
     ZSTD_sequenceCopier sequenceCopier = NULL;
@@ -6539,7 +6602,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
         ZSTD_resetSeqStore(&cctx->seqStore);
         DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize);
 
-        additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize);
+        additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes);
         FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");
         blockSize -= additionalByteAdjustment;
 
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 79068bcb8..dac7a0aa2 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -358,6 +358,9 @@ struct ZSTD_CCtx_params_s {
 
     /* Adjust the max block size*/
     size_t maxBlockSize;
+
+    /* Controls repcode search in external sequence parsing */
+    ZSTD_paramSwitch_e searchForExternalRepcodes;
 };  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
 
 #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
@@ -1453,7 +1456,7 @@ size_t
 ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                               ZSTD_sequencePosition* seqPos,
                                         const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                        const void* src, size_t blockSize);
+                                        const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
 
 /* Returns the number of bytes to move the current read position back by.
  * Only non-zero if we ended up splitting a sequence.
@@ -1470,6 +1473,6 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
 size_t
 ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                    const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                   const void* src, size_t blockSize);
+                                   const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
 
 #endif /* ZSTD_COMPRESS_H */
diff --git a/lib/zstd.h b/lib/zstd.h
index 237142289..91a3679a1 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -501,7 +501,8 @@ typedef enum {
      ZSTD_c_experimentalParam15=1012,
      ZSTD_c_experimentalParam16=1013,
      ZSTD_c_experimentalParam17=1014,
-     ZSTD_c_experimentalParam18=1015
+     ZSTD_c_experimentalParam18=1015,
+     ZSTD_c_experimentalParam19=1016
 } ZSTD_cParameter;
 
 typedef struct {
@@ -2126,18 +2127,41 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  * documentation (below) before setting this parameter. */
 #define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
 
-/*  ZSTD_c_maxBlockSize
- *  Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
- *  The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
  *
- *  This parameter can be used to set an upper bound on the blocksize
- *  that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
- *  bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
- *  compressBound() innacurate). Only currently meant to be used for testing.
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() innacurate). Only currently meant to be used for testing.
  *
  */
 #define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
 
+/* ZSTD_c_searchForExternalRepcodes
+ * This parameter affects how zstd parses external sequences, such as sequences
+ * provided through the compressSequences() API or from an external matchfinder.
+ *
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
+ * external sequences, even if those repcodes are not explicitly indicated in
+ * the "rep" field. Note that this is the only way to exploit repcode matches
+ * while using compressSequences() or an external matchfinder, since zstd
+ * currently ignores the "rep" field of external sequences.
+ *
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
+ * external sequences, regardless of whether the "rep" field has been set. This
+ * reduces sequence compression overhead by about 25% while sacrificing some
+ * compression ratio.
+ *
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
+ * based on compression level.
+ *
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
+ */
+#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
+
 /*! ZSTD_CCtx_getParameter() :
  *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  *  and store it into int* value.
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 310012cb5..6b833471e 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -129,6 +129,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
     setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
     setRand(cctx, ZSTD_c_validateSequences, 0, 1, producer);
+    setRand(cctx, ZSTD_c_searchForExternalRepcodes, 0, 2, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }

From cc3e3acd3415cfb14aad5fbe4e819fcf20760e8f Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 1 Feb 2023 16:38:36 -0800
Subject: [PATCH 178/197] Fix 32-bit decoding with large dictionary

The 32-bit decoder could corrupt the regenerated data by using regular
offset mode when there were actually long offsets. This is because we
were only considering the window size in the calculation, not the
dictionary size. So a large dictionary could allow longer offsets.

Fix this in two ways:
1. Instead of looking at the window size, look at the total referencable
   bytes in the history buffer. Use this in the comparison instead of
   the window size. Additionally, we were comparing against the wrong
   value, it was too low. Fix that by computing exactly the maximum
   offset for regular sequence decoding.
2. If it is possible that we have long offsets due to (1), then check
   the offset code decoding table, and if the decoding table's maximum
   number of additional bits is no more than STREAM_ACCUMULATOR_MIN,
   then we can't have long offsets.

This gates us to be using the long offsets decoder only when we are very
likely to actually have long offsets.

Note that this bug only affects the decoding of the data, and the
original compressed data, if re-read with a patched decoder, will
correctly regenerate the orginal data. Except that the encoder also had
the same issue previously.

This fixes both the open OSS-Fuzz issues.

Credit to OSS-Fuzz
---
 lib/decompress/zstd_decompress_block.c | 125 +++++++++++++++++++------
 1 file changed, 96 insertions(+), 29 deletions(-)

diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index a2a6eb3ed..f8a7963a9 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1170,7 +1170,7 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16
 }
 
 /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
- * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
+ * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
  * bits before reloading. This value is the maximum number of bytes we read
  * after reloading when we are decoding long offsets.
  */
@@ -1986,34 +1986,74 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
 #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
 
 
+/**
+ * @returns The total size of the history referencable by zstd, including
+ * both the prefix and the extDict. At @p op any offset larger than this
+ * is invalid.
+ */
+static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
+{
+    return (size_t)(op - virtualStart);
+}
 
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
-    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-/* ZSTD_getLongOffsetsShare() :
+typedef struct {
+    unsigned longOffsetShare;
+    unsigned maxNbAdditionalBits;
+} ZSTD_OffsetInfo;
+
+/* ZSTD_getOffsetInfo() :
  * condition : offTable must be valid
  * @return : "share" of long offsets (arbitrarily defined as > (1<<23))
- *           compared to maximum possible of (1<<OffFSELog) */
-static unsigned
-ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable)
+ *           compared to maximum possible of (1<<OffFSELog),
+ *           as well as the maximum number additional bits required.
+ */
+static ZSTD_OffsetInfo
+ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable)
 {
     const void* ptr = offTable;
     U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
     const ZSTD_seqSymbol* table = offTable + 1;
     U32 const max = 1 << tableLog;
-    U32 u, total = 0;
+    U32 u;
+    ZSTD_OffsetInfo info = {0, 0};
     DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
 
     assert(max <= (1 << OffFSELog));  /* max not too large */
     for (u=0; u<max; u++) {
-        if (table[u].nbAdditionalBits > 22) total += 1;
+        info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
+        if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
     }
 
     assert(tableLog <= OffFSELog);
-    total <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
+    info.longOffsetShare <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
 
-    return total;
+    return info;
+}
+
+/**
+ * @returns The maximum offset we can decode in one read of our bitstream, without
+ * reloading more bits in the middle of the offset bits read. Any offsets larger
+ * than this must use the long offset decoder.
+ */
+static size_t ZSTD_maxShortOffset(void)
+{
+    if (MEM_64bits()) {
+        /* We can decode any offset without reloading bits.
+         * This might change if the max window size grows.
+         */
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+        return (size_t)-1;
+    } else {
+        /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
+         * This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
+         * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
+         */
+        size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
+        size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
+        assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
+        return maxOffset;
+    }
 }
-#endif
 
 size_t
 ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
@@ -2021,13 +2061,6 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
                         const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
 {   /* blockType == blockCompressed */
     const BYTE* ip = (const BYTE*)src;
-    /* isLongOffset must be true if there are long offsets.
-     * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
-     * We don't expect that to be the case in 64-bit mode.
-     * In block mode, window size is not known, so we have to be conservative.
-     * (note: it could possibly be evaluated from current-lowLimit)
-     */
-    ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
     DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
 
     /* Note : the wording of the specification
@@ -2050,6 +2083,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
     /* Build Decoding Tables */
     {
+        /* Compute the maximum block size, which must also work when !frame and fParams are unset.
+         * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
+         */
+        size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX));
+        size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart);
+        /* isLongOffset must be true if there are long offsets.
+         * Offsets are long if they are larger than ZSTD_maxShortOffset().
+         * We don't expect that to be the case in 64-bit mode.
+         *
+         * We check here to see if our history is large enough to allow long offsets.
+         * If it isn't, then we can't possible have (valid) long offsets. If the offset
+         * is invalid, then it is okay to read it incorrectly.
+         *
+         * If isLongOffsets is true, then we will later check our decoding table to see
+         * if it is even possible to generate long offsets.
+         */
+        ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
         /* These macros control at build-time which decompressor implementation
          * we use. If neither is defined, we do some inspection and dispatch at
          * runtime.
@@ -2057,6 +2107,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
     !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
         int usePrefetchDecoder = dctx->ddictIsCold;
+#else
+        /* Set to 1 to avoid computing offset info if we don't need to.
+         * Otherwise this value is ignored.
+         */
+        int usePrefetchDecoder = 1;
 #endif
         int nbSeq;
         size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
@@ -2066,26 +2121,38 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
         RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
 
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
-    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-        if ( !usePrefetchDecoder
-          && (!frame || (dctx->fParams.windowSize > (1<<24)))
-          && (nbSeq>ADVANCED_SEQS) ) {  /* could probably use a larger nbSeq limit */
-            U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr);
-            U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
-            usePrefetchDecoder = (shareLongOffsets >= minShare);
+        /* If we could potentially have long offsets, or we might want to use the prefetch decoder,
+         * compute information about the share of long offsets, and the maximum nbAdditionalBits.
+         * NOTE: could probably use a larger nbSeq limit
+         */
+        if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
+            ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr);
+            if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
+                /* If isLongOffset, but the maximum number of additional bits that we see in our table is small
+                 * enough, then we know it is impossible to have too long an offset in this block, so we can
+                 * use the regular offset decoder.
+                 */
+                isLongOffset = ZSTD_lo_isRegularOffset;
+            }
+            if (!usePrefetchDecoder) {
+                U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
+                usePrefetchDecoder = (info.longOffsetShare >= minShare);
+            }
         }
-#endif
 
         dctx->ddictIsCold = 0;
 
 #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
     !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-        if (usePrefetchDecoder)
+        if (usePrefetchDecoder) {
+#else
+        (void)usePrefetchDecoder;
+        {
 #endif
 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
             return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
 #endif
+        }
 
 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
         /* else */

From 3fe5f1fbb9fde42d50cbe35f89775d284663da99 Mon Sep 17 00:00:00 2001
From: Elliot Gorokhovsky <embg@fb.com>
Date: Wed, 1 Feb 2023 18:24:46 -0800
Subject: [PATCH 179/197] assert externalRepSearch != ZSTD_ps_auto

---
 lib/compress/zstd_compress.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3ad6f05a2..5b89ca6d2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -6335,6 +6335,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
     }
 
     /* If we skipped repcode search while parsing, we need to update repcodes now */
+    assert(externalRepSearch != ZSTD_ps_auto);
     assert(idx >= startIdx);
     if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
         U32* const rep = updatedRepcodes.rep;

From 71a0259247829546cd639e379a8c47384ed96a26 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 2 Feb 2023 10:53:08 -0800
Subject: [PATCH 180/197] Fix ZSTD_getOffsetInfo() when nbSeq == 0

In 32-bit mode, ZSTD_getOffsetInfo() can be called when nbSeq == 0, and
in this case the offset table is uninitialized. The function should just
return 0 for both values, because there are no sequences.

Credit to OSS-Fuzz
---
 lib/decompress/zstd_decompress_block.c | 35 +++++++++++++++-----------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index f8a7963a9..0a06a021e 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -2008,25 +2008,30 @@ typedef struct {
  *           as well as the maximum number additional bits required.
  */
 static ZSTD_OffsetInfo
-ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable)
+ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq)
 {
-    const void* ptr = offTable;
-    U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
-    const ZSTD_seqSymbol* table = offTable + 1;
-    U32 const max = 1 << tableLog;
-    U32 u;
     ZSTD_OffsetInfo info = {0, 0};
-    DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
+    /* If nbSeq == 0, then the offTable is uninitialized, but we have
+     * no sequences, so both values should be 0.
+     */
+    if (nbSeq != 0) {
+        const void* ptr = offTable;
+        U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
+        const ZSTD_seqSymbol* table = offTable + 1;
+        U32 const max = 1 << tableLog;
+        U32 u;
+        DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
 
-    assert(max <= (1 << OffFSELog));  /* max not too large */
-    for (u=0; u<max; u++) {
-        info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
-        if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
+        assert(max <= (1 << OffFSELog));  /* max not too large */
+        for (u=0; u<max; u++) {
+            info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
+            if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
+        }
+
+        assert(tableLog <= OffFSELog);
+        info.longOffsetShare <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
     }
 
-    assert(tableLog <= OffFSELog);
-    info.longOffsetShare <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
-
     return info;
 }
 
@@ -2126,7 +2131,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
          * NOTE: could probably use a larger nbSeq limit
          */
         if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
-            ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr);
+            ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
             if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
                 /* If isLongOffset, but the maximum number of additional bits that we see in our table is small
                  * enough, then we know it is impossible to have too long an offset in this block, so we can

From 79bdb8cbb6232370336c2da0d1cc7610b67bb35d Mon Sep 17 00:00:00 2001
From: Yonatan Komornik <11005061+yoniko@users.noreply.github.com>
Date: Thu, 2 Feb 2023 15:19:22 -0800
Subject: [PATCH 181/197] AsyncIO performance regression for small files fix
 (#3474)

- Do not use threaded AsyncIO when handling small files.
- Some typo / doc fixes
---
 programs/fileio.c         | 25 ++++++++++
 programs/fileio_asyncio.c | 97 ++++++++++++++++++++++++++++-----------
 programs/fileio_asyncio.h | 22 +++++++++
 3 files changed, 116 insertions(+), 28 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index fb71804d6..9a8300cdd 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1758,6 +1758,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
     int result;
     FILE* srcFile;
     stat_t srcFileStat;
+    U64 fileSize = UTIL_FILESIZE_UNKNOWN;
     DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
 
     if (strcmp(srcFileName, stdinmark)) {
@@ -1790,6 +1791,17 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
     srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
     if (srcFile == NULL) return 1;   /* srcFile could not be opened */
 
+    /* Don't use AsyncIO for small files */
+    if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
+        fileSize = UTIL_getFileSizeStat(&srcFileStat);
+    if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
+        AIO_ReadPool_setAsync(ress.readCtx, 0);
+        AIO_WritePool_setAsync(ress.writeCtx, 0);
+    } else {
+        AIO_ReadPool_setAsync(ress.readCtx, 1);
+        AIO_WritePool_setAsync(ress.writeCtx, 1);
+    }
+
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
     result = FIO_compressFilename_dstFile(
             fCtx, prefs, ress,
@@ -2586,6 +2598,7 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs
     FILE* srcFile;
     stat_t srcFileStat;
     int result;
+    U64 fileSize = UTIL_FILESIZE_UNKNOWN;
 
     if (UTIL_isDirectory(srcFileName)) {
         DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
@@ -2594,6 +2607,18 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs
 
     srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
     if (srcFile==NULL) return 1;
+
+    /* Don't use AsyncIO for small files */
+    if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
+        fileSize = UTIL_getFileSizeStat(&srcFileStat);
+    if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
+        AIO_ReadPool_setAsync(ress.readCtx, 0);
+        AIO_WritePool_setAsync(ress.writeCtx, 0);
+    } else {
+        AIO_ReadPool_setAsync(ress.readCtx, 1);
+        AIO_WritePool_setAsync(ress.writeCtx, 1);
+    }
+
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
 
     result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName, &srcFileStat);
diff --git a/programs/fileio_asyncio.c b/programs/fileio_asyncio.c
index 8f12fe1f9..fe9cca95d 100644
--- a/programs/fileio_asyncio.c
+++ b/programs/fileio_asyncio.c
@@ -140,7 +140,7 @@ int AIO_supported(void) {
 }
 
 /* ***********************************
- *  General IoPool implementation
+ *  Generic IoPool implementation
  *************************************/
 
 static IOJob_t *AIO_IOPool_createIoJob(IOPoolCtx_t *ctx, size_t bufferSize) {
@@ -163,20 +163,22 @@ static IOJob_t *AIO_IOPool_createIoJob(IOPoolCtx_t *ctx, size_t bufferSize) {
  * Displays warning if asyncio is requested but MT isn't available. */
 static void AIO_IOPool_createThreadPool(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs) {
     ctx->threadPool = NULL;
+    ctx->threadPoolActive = 0;
     if(prefs->asyncIO) {
         if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL))
-            EXM_THROW(102,"Failed creating write availableJobs mutex");
+            EXM_THROW(102,"Failed creating ioJobsMutex mutex");
         /* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to
          * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */
         assert(MAX_IO_JOBS >= 2);
         ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2);
+        ctx->threadPoolActive = 1;
         if (!ctx->threadPool)
-            EXM_THROW(104, "Failed creating writer thread pool");
+            EXM_THROW(104, "Failed creating I/O thread pool");
     }
 }
 
 /* AIO_IOPool_init:
- * Allocates and sets and a new write pool including its included availableJobs. */
+ * Allocates and sets and a new I/O thread pool including its included availableJobs. */
 static void AIO_IOPool_init(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs, POOL_function poolFunction, size_t bufferSize) {
     int i;
     AIO_IOPool_createThreadPool(ctx, prefs);
@@ -192,27 +194,59 @@ static void AIO_IOPool_init(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs, POOL_fun
 }
 
 
+/* AIO_IOPool_threadPoolActive:
+ * Check if current operation uses thread pool.
+ * Note that in some cases we have a thread pool initialized but choose not to use it. */
+static int AIO_IOPool_threadPoolActive(IOPoolCtx_t* ctx) {
+    return ctx->threadPool && ctx->threadPoolActive;
+}
+
+
+/* AIO_IOPool_lockJobsMutex:
+ * Locks the IO jobs mutex if threading is active */
+static void AIO_IOPool_lockJobsMutex(IOPoolCtx_t* ctx) {
+    if(AIO_IOPool_threadPoolActive(ctx))
+        ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
+}
+
+/* AIO_IOPool_unlockJobsMutex:
+ * Unlocks the IO jobs mutex if threading is active */
+static void AIO_IOPool_unlockJobsMutex(IOPoolCtx_t* ctx) {
+    if(AIO_IOPool_threadPoolActive(ctx))
+        ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
+}
+
 /* AIO_IOPool_releaseIoJob:
  * Releases an acquired job back to the pool. Doesn't execute the job. */
 static void AIO_IOPool_releaseIoJob(IOJob_t* job) {
     IOPoolCtx_t* const ctx = (IOPoolCtx_t *) job->ctx;
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(ctx);
     assert(ctx->availableJobsCount < ctx->totalIoJobs);
     ctx->availableJobs[ctx->availableJobsCount++] = job;
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
+    AIO_IOPool_unlockJobsMutex(ctx);
 }
 
 /* AIO_IOPool_join:
  * Waits for all tasks in the pool to finish executing. */
 static void AIO_IOPool_join(IOPoolCtx_t* ctx) {
-    if(ctx->threadPool)
+    if(AIO_IOPool_threadPoolActive(ctx))
         POOL_joinJobs(ctx->threadPool);
 }
 
+/* AIO_IOPool_setThreaded:
+ * Allows (de)activating threaded mode, to be used when the expected overhead
+ * of threading costs more than the expected gains. */
+static void AIO_IOPool_setThreaded(IOPoolCtx_t* ctx, int threaded) {
+    assert(threaded == 0 || threaded == 1);
+    assert(ctx != NULL);
+    if(ctx->threadPoolActive != threaded) {
+        AIO_IOPool_join(ctx);
+        ctx->threadPoolActive = threaded;
+    }
+}
+
 /* AIO_IOPool_free:
- * Release a previously allocated write thread pool. Makes sure all takss are done and released. */
+ * Release a previously allocated IO thread pool. Makes sure all tasks are done and released. */
 static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
     int i;
     if(ctx->threadPool) {
@@ -236,12 +270,10 @@ static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
 static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
     IOJob_t *job;
     assert(ctx->file != NULL || ctx->prefs->testMode);
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(ctx);
     assert(ctx->availableJobsCount > 0);
     job = (IOJob_t*) ctx->availableJobs[--ctx->availableJobsCount];
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
+    AIO_IOPool_unlockJobsMutex(ctx);
     job->usedBufferSize = 0;
     job->file = ctx->file;
     job->offset = 0;
@@ -251,8 +283,7 @@ static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
 
 /* AIO_IOPool_setFile:
  * Sets the destination file for future files in the pool.
- * Requires completion of all queues write jobs and release of all otherwise acquired jobs.
- * Also requires ending of sparse write if a previous file was used in sparse mode. */
+ * Requires completion of all queued jobs and release of all otherwise acquired jobs. */
 static void AIO_IOPool_setFile(IOPoolCtx_t* ctx, FILE* file) {
     assert(ctx!=NULL);
     AIO_IOPool_join(ctx);
@@ -269,7 +300,7 @@ static FILE* AIO_IOPool_getFile(const IOPoolCtx_t* ctx) {
  * The queued job shouldn't be used directly after queueing it. */
 static void AIO_IOPool_enqueueJob(IOJob_t* job) {
     IOPoolCtx_t* const ctx = (IOPoolCtx_t *)job->ctx;
-    if(ctx->threadPool)
+    if(AIO_IOPool_threadPoolActive(ctx))
         POOL_add(ctx->threadPool, ctx->poolFunction, job);
     else
         ctx->poolFunction(job);
@@ -300,8 +331,7 @@ void AIO_WritePool_enqueueAndReacquireWriteJob(IOJob_t **job) {
  * Blocks on completion of all current write jobs before executing. */
 void AIO_WritePool_sparseWriteEnd(WritePoolCtx_t* ctx) {
     assert(ctx != NULL);
-    if(ctx->base.threadPool)
-        POOL_joinJobs(ctx->base.threadPool);
+    AIO_IOPool_join(&ctx->base);
     AIO_fwriteSparseEnd(ctx->base.prefs, ctx->base.file, ctx->storedSkips);
     ctx->storedSkips = 0;
 }
@@ -368,6 +398,13 @@ void AIO_WritePool_free(WritePoolCtx_t* ctx) {
     free(ctx);
 }
 
+/* AIO_WritePool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_WritePool_setAsync(WritePoolCtx_t* ctx, int async) {
+    AIO_IOPool_setThreaded(&ctx->base, async);
+}
+
 
 /* ***********************************
  *  ReadPool implementation
@@ -383,14 +420,13 @@ static void AIO_ReadPool_releaseAllCompletedJobs(ReadPoolCtx_t* ctx) {
 
 static void AIO_ReadPool_addJobToCompleted(IOJob_t* job) {
     ReadPoolCtx_t* const ctx = (ReadPoolCtx_t *)job->ctx;
-    if(ctx->base.threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(&ctx->base);
     assert(ctx->completedJobsCount < MAX_IO_JOBS);
     ctx->completedJobs[ctx->completedJobsCount++] = job;
-    if(ctx->base.threadPool) {
+    if(AIO_IOPool_threadPoolActive(&ctx->base)) {
         ZSTD_pthread_cond_signal(&ctx->jobCompletedCond);
-        ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex);
     }
+    AIO_IOPool_unlockJobsMutex(&ctx->base);
 }
 
 /* AIO_ReadPool_findNextWaitingOffsetCompletedJob_locked:
@@ -426,8 +462,7 @@ static size_t AIO_ReadPool_numReadsInFlight(ReadPoolCtx_t* ctx) {
  * Would block. */
 static IOJob_t* AIO_ReadPool_getNextCompletedJob(ReadPoolCtx_t* ctx) {
     IOJob_t *job = NULL;
-    if (ctx->base.threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(&ctx->base);
 
     job = AIO_ReadPool_findNextWaitingOffsetCompletedJob_locked(ctx);
 
@@ -443,8 +478,7 @@ static IOJob_t* AIO_ReadPool_getNextCompletedJob(ReadPoolCtx_t* ctx) {
         ctx->waitingOnOffset += job->usedBufferSize;
     }
 
-    if (ctx->base.threadPool)
-        ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex);
+    AIO_IOPool_unlockJobsMutex(&ctx->base);
     return job;
 }
 
@@ -524,7 +558,7 @@ ReadPoolCtx_t* AIO_ReadPool_create(const FIO_prefs_t* prefs, size_t bufferSize)
 
     if(ctx->base.threadPool)
         if (ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL))
-            EXM_THROW(103,"Failed creating write jobCompletedCond mutex");
+            EXM_THROW(103,"Failed creating jobCompletedCond cond");
 
     return ctx;
 }
@@ -620,3 +654,10 @@ int AIO_ReadPool_closeFile(ReadPoolCtx_t* ctx) {
     AIO_ReadPool_setFile(ctx, NULL);
     return fclose(file);
 }
+
+/* AIO_ReadPool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_ReadPool_setAsync(ReadPoolCtx_t* ctx, int async) {
+    AIO_IOPool_setThreaded(&ctx->base, async);
+}
diff --git a/programs/fileio_asyncio.h b/programs/fileio_asyncio.h
index 34dad6f4d..feb25a3f9 100644
--- a/programs/fileio_asyncio.h
+++ b/programs/fileio_asyncio.h
@@ -8,6 +8,17 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
+ /*
+  * FileIO AsyncIO exposes read/write IO pools that allow doing IO asynchronously.
+  * Current implementation relies on having one thread that reads and one that
+  * writes.
+  * Each IO pool supports up to `MAX_IO_JOBS` that can be enqueued for work, but
+  * are performed serially by the appropriate worker thread.
+  * Most systems exposes better primitives to perform asynchronous IO, such as
+  * io_uring on newer linux systems. The API is built in such a way that in the
+  * future we could replace the threads with better solutions when available.
+  */
+
 #ifndef ZSTD_FILEIO_ASYNCIO_H
 #define ZSTD_FILEIO_ASYNCIO_H
 
@@ -27,6 +38,7 @@ extern "C" {
 typedef struct {
     /* These struct fields should be set only on creation and not changed afterwards */
     POOL_ctx* threadPool;
+    int threadPoolActive;
     int totalIoJobs;
     const FIO_prefs_t* prefs;
     POOL_function poolFunction;
@@ -136,6 +148,11 @@ WritePoolCtx_t* AIO_WritePool_create(const FIO_prefs_t* prefs, size_t bufferSize
  * Frees and releases a writePool and its resources. Closes destination file. */
 void AIO_WritePool_free(WritePoolCtx_t* ctx);
 
+/* AIO_WritePool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_WritePool_setAsync(WritePoolCtx_t* ctx, int async);
+
 /* AIO_ReadPool_create:
  * Allocates and sets and a new readPool including its included jobs.
  * bufferSize should be set to the maximal buffer we want to read at a time, will also be used
@@ -146,6 +163,11 @@ ReadPoolCtx_t* AIO_ReadPool_create(const FIO_prefs_t* prefs, size_t bufferSize);
  * Frees and releases a readPool and its resources. Closes source file. */
 void AIO_ReadPool_free(ReadPoolCtx_t* ctx);
 
+/* AIO_ReadPool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_ReadPool_setAsync(ReadPoolCtx_t* ctx, int async);
+
 /* AIO_ReadPool_consumeBytes:
  * Consumes byes from srcBuffer's beginning and updates srcBufferLoaded accordingly. */
 void AIO_ReadPool_consumeBytes(ReadPoolCtx_t *ctx, size_t n);

From 35835f4126270a413ac2a28fb4981cc842ce62c0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Feb 2023 05:12:10 +0000
Subject: [PATCH 182/197] Bump ilammy/msvc-dev-cmd from 1.12.0 to 1.12.1

Bumps [ilammy/msvc-dev-cmd](https://github.com/ilammy/msvc-dev-cmd) from 1.12.0 to 1.12.1.
- [Release notes](https://github.com/ilammy/msvc-dev-cmd/releases)
- [Commits](https://github.com/ilammy/msvc-dev-cmd/compare/7315a94840631165970262a99c72cfb48a65d25d...cec98b9d092141f74527d0afa6feb2af698cfe89)

---
updated-dependencies:
- dependency-name: ilammy/msvc-dev-cmd
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/dev-short-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 2b2eb11b2..132dbb996 100644
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -218,7 +218,7 @@ jobs:
       - name: Install packages
         run: pip install --pre meson
       - name: Initialize the MSVC dev command prompt
-        uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d
+        uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89
       - name: Configure with Meson
         run: |
           meson setup build/meson/ builddir -Dbin_tests=true -Dbin_programs=true -Dbin_contrib=true

From 9cabd155fdf14b9ba8d240b42b2403e1da178a92 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 7 Feb 2023 00:35:51 -0800
Subject: [PATCH 183/197] return error code when benchmark fails

such scenario can happen, for example,
when trying a decompression-only benchmark on invalid data.
Other possibilities include an allocation error in an intermediate step.

So far, the benchmark would return immediately, but still return 0.
On command line, this would be confusing, as the program appears successful (though it does not display any successful message).

Now it returns !0, which can be interpreted as an error by command line.
---
 programs/zstdcli.c | 13 ++++++++-----
 tests/playTests.sh |  2 ++
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 660e66bb6..93f75e21d 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1370,7 +1370,7 @@ int main(int argCount, const char* argv[])
         benchParams.ldmFlag = ldmFlag;
         benchParams.ldmMinMatch = (int)g_ldmMinMatch;
         benchParams.ldmHashLog = (int)g_ldmHashLog;
-        benchParams.useRowMatchFinder = useRowMatchFinder;
+        benchParams.useRowMatchFinder = (int)useRowMatchFinder;
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
             benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
         }
@@ -1391,15 +1391,18 @@ int main(int argCount, const char* argv[])
                     int c;
                     DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
                     for(c = cLevel; c <= cLevelLast; c++) {
-                        BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
+                        BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
+                        if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
                 }   }
             } else {
                 for(; cLevel <= cLevelLast; cLevel++) {
-                    BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
+                    BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
+                    if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
             }   }
         } else {
             for(; cLevel <= cLevelLast; cLevel++) {
-                BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
+                BMK_benchOutcome_t const bo = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
+                if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
         }   }
 
 #else
@@ -1545,7 +1548,7 @@ int main(int argCount, const char* argv[])
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
         if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
         FIO_setAdaptiveMode(prefs, adapt);
-        FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
+        FIO_setUseRowMatchFinder(prefs, (int)useRowMatchFinder);
         FIO_setAdaptMin(prefs, adaptMin);
         FIO_setAdaptMax(prefs, adaptMax);
         FIO_setRsyncable(prefs, rsyncable);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 5d78e9e7d..5f595f611 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -1217,6 +1217,8 @@ zstd -rqi0b1e2 tmp1
 println "benchmark decompression only"
 zstd -f tmp1
 zstd -b -d -i0 tmp1.zst
+println "benchmark can fail - decompression on invalid data"
+zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed"
 
 GZIPMODE=1
 zstd --format=gzip -V || GZIPMODE=0

From 58e7067c7d500537bfdccc5ee4707ac0154656d7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 7 Feb 2023 08:47:39 -0800
Subject: [PATCH 184/197] added more accurate error messages

for the decompression-only benchmark mode.
---
 programs/benchzstd.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 7770dd2a5..e9deeb6fa 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -327,26 +327,31 @@ BMK_benchMemAdvancedNoAlloc(
     /* init */
     memset(&benchResult, 0, sizeof(benchResult));
     if (strlen(displayName)>17) displayName += strlen(displayName) - 17;   /* display last 17 characters */
-    if (adv->mode == BMK_decodeOnly) {  /* benchmark only decompression : source must be already compressed */
+    if (adv->mode == BMK_decodeOnly) {
+        /* benchmark only decompression : source must be already compressed */
         const char* srcPtr = (const char*)srcBuffer;
         U64 totalDSize64 = 0;
         U32 fileNb;
         for (fileNb=0; fileNb<nbFiles; fileNb++) {
             U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
-            if (fSize64==0) RETURN_ERROR(32, BMK_benchOutcome_t, "Impossible to determine original size ");
+            if (fSize64 == ZSTD_CONTENTSIZE_UNKNOWN) {
+                RETURN_ERROR(32, BMK_benchOutcome_t, "Decompressed size cannot be determined: cannot benchmark");
+            }
+            if (fSize64 == ZSTD_CONTENTSIZE_ERROR) {
+                RETURN_ERROR(32, BMK_benchOutcome_t, "Error while trying to assess decompressed size: data may be invalid");
+            }
             totalDSize64 += fSize64;
             srcPtr += fileSizes[fileNb];
         }
         {   size_t const decodedSize = (size_t)totalDSize64;
             assert((U64)decodedSize == totalDSize64);   /* check overflow */
             free(*resultBufferPtr);
+            if (totalDSize64 > decodedSize) {  /* size_t overflow */
+                RETURN_ERROR(32, BMK_benchOutcome_t, "decompressed size is too large for local system");
+            }
             *resultBufferPtr = malloc(decodedSize);
             if (!(*resultBufferPtr)) {
-                RETURN_ERROR(33, BMK_benchOutcome_t, "not enough memory");
-            }
-            if (totalDSize64 > decodedSize) {  /* size_t overflow */
-                free(*resultBufferPtr);
-                RETURN_ERROR(32, BMK_benchOutcome_t, "original size is too large");
+                RETURN_ERROR(33, BMK_benchOutcome_t, "allocation error: not enough memory");
             }
             cSize = srcSize;
             srcSize = decodedSize;
@@ -474,7 +479,7 @@ BMK_benchMemAdvancedNoAlloc(
                 BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
 
                 if(!BMK_isSuccessful_runOutcome(dOutcome)) {
-                    return BMK_benchOutcome_error();
+                    RETURN_ERROR(30, BMK_benchOutcome_t, "decompression error");
                 }
 
                 {   BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
@@ -598,7 +603,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
 
     void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
 
-    int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
+    int const allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
         !cSizes || !cCapacities || !resPtrs || !resSizes ||
         !timeStateCompress || !timeStateDecompress ||
         !cctx || !dctx ||

From 6740f8f0b8720ded3e8fa35f546115ac29c60e2e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 7 Feb 2023 10:02:09 -0800
Subject: [PATCH 185/197] add error message for the (rare) compression error
 scenario

---
 programs/benchzstd.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index e9deeb6fa..a76db5f37 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -451,7 +451,7 @@ BMK_benchMemAdvancedNoAlloc(
                 BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
 
                 if (!BMK_isSuccessful_runOutcome(cOutcome)) {
-                    return BMK_benchOutcome_error();
+                    RETURN_ERROR(30, BMK_benchOutcome_t, "compression error");
                 }
 
                 {   BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);

From e04706c58cc462cc771292c01abc66caaebe57a6 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 6 Feb 2023 20:20:01 -0800
Subject: [PATCH 186/197] fix oss-fuzz case 55714

impacts legacy decoder v0.3 in 32-bit mode
---
 lib/legacy/zstd_v03.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 5b1fd7175..d247d5d55 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -2706,18 +2706,24 @@ static size_t ZSTD_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
 
     /* checks */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use the pointer check */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 
     /* copy Match */
-    {
-        const BYTE* match = op - sequence.offset;
+    {   const BYTE* match = op - sequence.offset;
 
         /* check */
         if (sequence.offset > (size_t)op) return ERROR(corruption_detected);   /* address space overflow test (this test seems kept by clang optimizer) */

From cfec005efd13b45433210ad8db543de3440fe2ad Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 6 Feb 2023 20:26:25 -0800
Subject: [PATCH 187/197] fix for v0.3 blindly ported to v0.2

in case it would be applicable here too.
---
 lib/legacy/zstd_v02.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index dfaed7bb8..e09bb4a24 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -3066,12 +3066,19 @@ static size_t ZSTD_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
 
     /* checks */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use the pointer check */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 

From 7eb4471fec8807b67a8475ad13cdf0310f2fad81 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 6 Feb 2023 20:33:38 -0800
Subject: [PATCH 188/197] adapt v0.3 fix to v0.1

slightly different constraints on end of buffer conditions
---
 lib/legacy/zstd_v01.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 8c504c792..7f24cf685 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1720,20 +1720,25 @@ static size_t ZSTD_execSequence(BYTE* op,
     static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};   /* added */
     static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* subtracted */
     const BYTE* const ostart = op;
+    BYTE* const oLitEnd = op + sequence.litLength;
     const size_t litLength = sequence.litLength;
     BYTE* const endMatch = op + litLength + sequence.matchLength;    /* risk : address space overflow (32-bits) */
     const BYTE* const litEnd = *litPtr + litLength;
 
-    /* check */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (endMatch > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (litEnd > litLimit) return ERROR(corruption_detected);
-    if (sequence.matchLength > (size_t)(*litPtr-op))  return ERROR(dstSize_tooSmall);    /* overwrite literal segment */
+    if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
-        memmove(op, *litPtr, litLength);   /* overwrite risk */
-    else
-        ZSTD_wildcopy(op, *litPtr, litLength);
+    ZSTD_memmove(op, *litPtr, sequence.litLength);   /* note : v0.1 seems to allow scenarios where output or input are close to end of buffer */
+
     op += litLength;
     *litPtr = litEnd;   /* update for next sequence */
 

From b20e4e95f2a9295adf82c4fdb13bcb1864983fd9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 6 Feb 2023 20:36:37 -0800
Subject: [PATCH 189/197] copy fix for v0.3 to v0.4

in case it would be applicable for this legacy version too.
---
 lib/legacy/zstd_v03.c |  2 +-
 lib/legacy/zstd_v04.c | 15 +++++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index d247d5d55..b0d7f521e 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -2710,7 +2710,7 @@ static size_t ZSTD_execSequence(BYTE* op,
 
     if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
     if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
-    /* Now we know there are no overflow in literal nor match lengths, can use the pointer check */
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
     if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
     if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
 
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 23735443a..f820134b7 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -2828,13 +2828,20 @@ static size_t ZSTD_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
     const BYTE* match = oLitEnd - sequence.offset;
 
-    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (litEnd > litLimit) return ERROR(corruption_detected);   /* risk read beyond lit buffer */
+    if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 

From 7a1a1716587c7e600e7c1759daccf5c679b6d961 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 6 Feb 2023 20:40:29 -0800
Subject: [PATCH 190/197] port fix for v0.3 to v0.5

in case it would be applicable for this version too
---
 lib/legacy/zstd_v05.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 5d2ebb2b5..85c4ce79f 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -3182,13 +3182,20 @@ static size_t ZSTDv05_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
     const BYTE* match = oLitEnd - sequence.offset;
 
-    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (litEnd > litLimit) return ERROR(corruption_detected);   /* risk read beyond lit buffer */
+    if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTDv05_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTDv05_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 

From 67d7a659f871a1bfbd99e82ad5036e619b7ca758 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 6 Feb 2023 20:43:09 -0800
Subject: [PATCH 191/197] port fix for v0.3 to v0.6

in case it would applicable for this version
---
 lib/legacy/zstd_v06.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 53aaec468..097182a4c 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3322,13 +3322,20 @@ static size_t ZSTDv06_execSequence(BYTE* op,
     const BYTE* const iLitEnd = *litPtr + sequence.litLength;
     const BYTE* match = oLitEnd - sequence.offset;
 
-    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTDv06_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTDv06_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = iLitEnd;   /* update for next sequence */
 

From 94197471719757d9dcf7aca10705d829a45817c8 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Tue, 7 Feb 2023 14:02:12 -0800
Subject: [PATCH 192/197] fix legacy decoders v0.4, v0.5 and v0.6

---
 lib/legacy/zstd_v04.c | 1 -
 lib/legacy/zstd_v05.c | 1 -
 lib/legacy/zstd_v06.c | 1 -
 3 files changed, 3 deletions(-)

diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index f820134b7..57be832bd 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -2835,7 +2835,6 @@ static size_t ZSTD_execSequence(BYTE* op,
     if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
     /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
     if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
-    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
 
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 85c4ce79f..93a1169f3 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -3189,7 +3189,6 @@ static size_t ZSTDv05_execSequence(BYTE* op,
     if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
     /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
     if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
-    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
 
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 097182a4c..175f7cc42 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3329,7 +3329,6 @@ static size_t ZSTDv06_execSequence(BYTE* op,
     if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
     /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
     if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
-    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
 
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */

From c5bf6b8b88456804b480176c6183a893ca01bf15 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Tue, 7 Feb 2023 14:47:16 -0800
Subject: [PATCH 193/197] add requested check for legacy decoder v0.1

which uses a different technique to store literals,
and therefore must check for potential overwrites.
---
 lib/legacy/zstd_v01.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 7f24cf685..1a3aad07e 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1735,6 +1735,7 @@ static size_t ZSTD_execSequence(BYTE* op,
 
     if (endMatch > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
+    if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall);  /* overwrite literal segment */
 
     /* copy Literals */
     ZSTD_memmove(op, *litPtr, sequence.litLength);   /* note : v0.1 seems to allow scenarios where output or input are close to end of buffer */

From c689310b2544c1eee6002bb4fa9a7ed8c7609dd1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 7 Feb 2023 17:11:07 -0800
Subject: [PATCH 194/197] rewrite legacy v0.7 bound checks to be independent of
 address space overflow

---
 lib/legacy/zstd_v07.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index c2b88eb3e..15dc3ef79 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -3552,11 +3552,14 @@ size_t ZSTDv07_execSequence(BYTE* op,
     const BYTE* match = oLitEnd - sequence.offset;
 
     /* check */
-    if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    assert(oend >= op);
+    if (sequence.litLength + WILDCOPY_OVERLENGTH > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequenceLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    assert(litLimit >= *litPtr);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);;
 
     /* copy Literals */
-    ZSTDv07_wildcopy(op, *litPtr, sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+    ZSTDv07_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = iLitEnd;   /* update for next sequence */
 
@@ -3570,7 +3573,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
             return sequenceLength;
         }
         /* span extDict & currentPrefixSegment */
-        {   size_t const length1 = dictEnd - match;
+        {   size_t const length1 = (size_t)(dictEnd - match);
             memmove(oLitEnd, match, length1);
             op = oLitEnd + length1;
             sequence.matchLength -= length1;

From 83f8a05f87cc5ee94aca47d250980bde22bb7bf1 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 8 Feb 2023 11:42:46 -0800
Subject: [PATCH 195/197] Fix empty-block.zst golden decompression file

This frame is invalid because the `Window_Size = 0`, and the
`Block_Maximum_Size = min(128 KB, Window_Size) = 0`. But the empty
compressed block has a `Block_Content` size of 2, which is invalid.

The fix is to switch to using a `Window_Descriptor` instead of the
`Single_Segment_Flag`. This sets the `Window_Size = 1024`.

Hexdump before this PR: `28b5 2ffd 2000 1500 0000 00`

Hexdump after this PR: `28b5 2ffd 0000 1500 0000 00`

For issue #3482.
---
 tests/golden-decompression/empty-block.zst | Bin 11 -> 11 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/tests/golden-decompression/empty-block.zst b/tests/golden-decompression/empty-block.zst
index 2a3782aff0d1a6bab62dcdc1af29b448322a84bd..fbfb893e11eb677f1e6444ead8a5829a3a23e53e 100644
GIT binary patch
literal 11
ScmdPcs{faPL6m`kfdK##%>o|)

literal 11
ScmdPcs{dDkL6m`kfdK#$tpYy)


From 4de9d637e86ea7963225d1a87d2d26f0e1e039fe Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 8 Feb 2023 15:56:49 -0800
Subject: [PATCH 196/197] minor: fix missing newline character in help page

---
 contrib/largeNbDicts/largeNbDicts.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index 7b2af0bae..eeaaf7182 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -985,7 +985,7 @@ int usage(const char* exeName)
     DISPLAY ("-#          : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
     DISPLAY ("-D #        : use # as a dictionary (default: create one) \n");
     DISPLAY ("-i#         : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S);
-    DISPLAY ("-p#         : print speed for all rounds 0=fastest 1=median (default: 0)");
+    DISPLAY ("-p#         : print speed for all rounds 0=fastest 1=median (default: 0) \n");
     DISPLAY ("--nbBlocks=#: use # blocks for bench (default: one per file) \n");
     DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n");
     DISPLAY ("-h          : help (this text) \n");

From 4aa3bc49da8608c0cc921fbc49b703addb6cffa8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 8 Feb 2023 16:35:55 -0800
Subject: [PATCH 197/197] updated CHANGELOG in preparation for v1.5.4 release

---
 CHANGELOG | 75 ++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 49 insertions(+), 26 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 6ddec4107..11cc0d3fb 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,39 +1,62 @@
-v1.5.4
-
-v1.5.3 (June, 2022) (unpublished)
-perf: 5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg)
-perf: 5-10% faster streaming compression at levels 1-2 (#3114, @embg)
-perf: Remove branch in ZSTD_fast_noDict (#3129, @felixhandte)
-perf: Add option to prefetch CDict tables (#3177, @embg)
-perf: Minor compression ratio improvement (#2983, @Cyan4973)
-perf: Minor speed improvement for Huffman decoding (#3013, @WojciechMula)
-perf: Enable STATIC_BMI2 for gcc/clang (#3080, @TocarIP)
-perf: Optimize ZSTD_row_getMatchMask for ARM levels 8-10 (#3139, #3160, @danlark1)
-perf: aarch64 performance improvements (#3145, #3141, @JunHe77)
-perf: Lazy parameters adaptation (#2974, @Cyan4973)
-cli: Async write for decompression (#2975, @yoniko)
-cli: Use buffered output (#2985, @yoniko)
-cli: Change zstdless behavior to align with zless (#2909, @binhdvo)
-cli: AsyncIO compression (#3021, #3022, @yoniko)
-cli: Print zlib/lz4/lzma library versions in verbose version output (#3030, @terrelln)
-cli: Fix for -r on empty directory (#3027, @brailovich)
-cli: Fix required decompression memory usage reported by -vv + --long (#3042, @u1f35c)
+v1.5.4 (Feb 2023)
+perf: +20% faster huffman decompression for targets that can't compile x64 assembly (#3449, @terrelln)
+perf: up to +10% faster streaming compression at levels 1-2 (#3114, @embg)
+perf: +4-13% for levels 5-12 by optimizing function generation (#3295, @terrelln)
+pref: +3-11% compression speed for `arm` target (#3199, #3164, #3145, #3141, #3138, @JunHe77 and #3139, #3160, @danlark1)
+perf: +5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg)
+perf: +10-20% cold dict compression speed by prefetching CDict tables (#3177, @embg)
+perf: +1% faster compression by removing a branch in ZSTD_fast_noDict (#3129, @felixhandte)
+perf: Small compression ratio improvements in high compression mode (#2983, #3391, @Cyan4973 and #3285, #3302, @daniellerozenblit)
+perf: small speed improvement by better detecting `STATIC_BMI2` for `clang` (#3080, @TocarIP)
+perf: Improved streaming performance when `ZSTD_c_stableInBuffer` is set (#2974, @Cyan4973)
+cli: Asynchronous I/O for improved cli speed (#2975, #2985, #3021, #3022, @yoniko)
+cli: Change `zstdless` behavior to align with `zless` (#2909, @binhdvo)
+cli: Keep original file if `-c` or `--stdout` is given (#3052, @dirkmueller)
+cli: Keep original files when result is concatenated into a single output with `-o` (#3450, @Cyan4973)
+cli: Preserve Permissions and Ownership of regular files (#3432, @felixhandte)
+cli: Print zlib/lz4/lzma library versions with `-vv` (#3030, @terrelln)
+cli: Print checksum value for single frame files with `-lv`  (#3332, @Cyan4973)
+cli: Print `dictID` when present with `-lv` (#3184, @htnhan)
+cli: when `stderr` is *not* the console, disable status updates, but preserve final summary (#3458, @Cyan4973)
+cli: support `--best` and `--no-name` in `gzip` compatibility mode (#3059, @dirkmueller)
+cli: support for `posix` high resolution timer `clock_gettime()`, for improved benchmark accuracy (#3423, @Cyan4973)
+cli: improved help/usage (`-h`,  `-H`) formatting (#3094, @dirkmueller and #3385, @jonpalmisc)
+cli: Fix better handling of bogus numeric values (#3268, @ctkhanhly)
+cli: Fix input consists of multiple files _and_ `stdin` (#3222, @yoniko)
+cli: Fix tiny files passthrough (#3215, @cgbur)
+cli: Fix for `-r` on empty directory (#3027, @brailovich)
+cli: Fix empty string as argument for `--output-dir-*` (#3220, @embg)
+cli: Fix decompression memory usage reported by `-vv --long` (#3042, @u1f35c, and #3232, @zengyijing)
 cli: Fix infinite loop when empty input is passed to trainer (#3081, @terrelln)
-cli: Implement more gzip compatibility (#3059, @dirkmueller)
-cli: Keep original file if -c or --stdout is given (#3052, @dirkmueller)
+cli: Fix `--adapt` doesn't work when `--no-progress` is also set (#3354, @terrelln)
+api: Support for External matchfinder (#3333, @embg)
+api: Support for in-place decompression (#3432, @terrelln)
+api: New  `ZSTD_CCtx_setCParams()`  function, set all parameters defined in a  `ZSTD_compressionParameters`  structure (#3403, @Cyan4973)
+api: Streaming decompression detects incorrect header ID sooner (#3175, @Cyan4973)
+api: Window size resizing optimization for edge case (#3345, @daniellerozenblit)
+api: More accurate error codes for busy-loop scenarios (#3413, #3455, @Cyan4973)
+api: Fix limit overflow in `compressBound` and `decompressBound` (#3362, #3373, Cyan4973) reported by @nigeltao
+api: Deprecate several advanced experimental functions: streaming (#3408, @embg), copy (#3196, @mileshu)
+bug: Fix corruption that rarely occurs in 32-bit mode with wlog=25 (#3361, @terrelln)
 bug: Fix for block-splitter (#3033, @Cyan4973)
 bug: Fixes for Sequence Compression API (#3023, #3040, @Cyan4973)
 bug: Fix leaking thread handles on Windows (#3147, @animalize)
 bug: Fix timing issues with cmake/meson builds (#3166, #3167, #3170, @Cyan4973)
 build: Allow user to select legacy level for cmake (#3050, @shadchin)
 build: Enable legacy support by default in cmake (#3079, @niamster)
-build: Meson improvements (#3039, #3122, @eli-schwartz)
+build: Meson build script improvements (#3039, #3120, #3122, #3327, #3357, @eli-schwartz and #3276, @neheb)
 build: Add aarch64 to supported architectures for zstd_trace (#3054, @ooosssososos)
-doc: Split help in long and short version, cleanup formatting (#3094, @dirkmueller)
-doc: Updated man page, providing more details for --train mode (#3112, @Cyan4973)
+build: support AIX architecture (#3219, @qiongsiwu)
+build: Fix `ZSTD_LIB_MINIFY` build macro, which now reduces static library size by half (#3366, @terrelln)
+build: Fix Windows issues with Multithreading translation layer (#3364, #3380, @yoniko) and ARM64 target (#3320, @cwoffenden)
+build: Fix `cmake` script (#3382, #3392, @terrelln and #3252 @Tachi107 and #3167 @Cyan4973)
+doc: Updated man page, providing more details for `--train` mode (#3112, @Cyan4973)
 doc: Add decompressor errata document (#3092, @terrelln)
 misc: Enable Intel CET (#2992, #2994, @hjl-tools)
-misc: Streaming decompression can detect incorrect header ID sooner (#3175, @Cyan4973)
+misc: Fix `contrib/` seekable format (#3058, @yhoogstrate and #3346, @daniellerozenblit)
+misc: Improve speed of the one-file library generator (#3241, @wahern and #3005, @cwoffenden)
+
+v1.5.3 (dev version, unpublished)
 
 v1.5.2 (Jan, 2022)
 perf: Regain Minimal memset()-ing During Reuse of Compression Contexts (@Cyan4973, #2969)