From f78917cdd3b0b4b835f3a42326dbc6d16fd533b9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 09:23:07 -0800
Subject: [PATCH 01/80] streamline zlibwrapper makefile

making better usage of default build rules
---
 zlibWrapper/Makefile | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index d74c41bc4..6fd5ac3bb 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -6,7 +6,7 @@
 
 
 # Paths to static and dynamic zlib and zstd libraries
-# Use "make ZLIB_PATH=path/to/zlib ZLIB_LIBRARY=path/to/libz.a" to select a path to library
+# Use "make ZLIB_PATH=path/to/zlib ZLIB_LIBRARY=path/to/libz.so" to select a path to library
 ZLIB_LIBRARY ?= -lz
 ZLIB_PATH ?= .
 
@@ -18,7 +18,8 @@ EXAMPLE_PATH = examples
 PROGRAMS_PATH = ../programs
 TEST_FILE = ../doc/zstd_compression_format.md
 
-VPATH = $(PROGRAMS_PATH)
+vpath %.c $(PROGRAMS_PATH) $(EXAMPLE_PATH) $(ZLIBWRAPPER_PATH)
+
 
 CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH)       \
             -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
@@ -28,7 +29,9 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
             -Wdeclaration-after-statement -Wstrict-prototypes -Wundef     \
             -Wstrict-aliasing=1
 CFLAGS   ?= -O3
-CFLAGS   += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
+CFLAGS   += $(STDFLAGS) $(DEBUGFLAGS)
+CPPFLAGS += $(MOREFLAGS)
+LDLIBS   += $(ZLIB_LIBRARY)
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
@@ -61,7 +64,7 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench minigzip minigzip_zstd
 	./minigzip_zstd -d example$(EXT).gz
 	@echo ---- minigzip end ----
 	./zwrapbench -qi1b3B1K $(TEST_FILE)
-	./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
+	./zwrapbench -rqi1b1e3 ../lib
 
 #valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
 valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
@@ -79,35 +82,32 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
 #.c.o:
 #	$(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
 
-minigzip: $(EXAMPLE_PATH)/minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
+minigzip: minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
 
-minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
+minigzip_zstd: minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
+	$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
 
-example: $(EXAMPLE_PATH)/example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
+example: example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
 
-example_zstd: $(EXAMPLE_PATH)/example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
+example_zstd: example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
+	$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
 
-fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
+fitblk: fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
 
-fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
+fitblk_zstd: fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
+	$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
 
-zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o util.o timefn.o datagen.o $(ZSTDLIBRARY)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
+zwrapbench: zwrapbench.o zstd_zlibwrapper.o util.o timefn.o datagen.o $(ZSTDLIBRARY)
 
 
-zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
+zstd_zlibwrapper.o: zstd_zlibwrapper.h
 
 zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
-zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
-	$(CC) $(CPPFLAGS) $(CFLAGS) $< -c -o $@
+zstdTurnedOn_zlibwrapper.o: zstd_zlibwrapper.c zstd_zlibwrapper.h
+	$(COMPILE.c) $< $(OUTPUT_OPTION)
 
-$(ZSTDLIBDIR)/libzstd.a:
+
+$(ZSTDLIBRARY):
 	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
 
 $(ZSTDLIBDIR)/libzstd.so:

From eacf1b3bc394dea4e335813065e6086561a6d62c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 15:03:59 -0800
Subject: [PATCH 02/80] streamlined github action test

---
 .github/workflows/generic-release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/generic-release.yml b/.github/workflows/generic-release.yml
index 167ba3848..632a6b7d2 100644
--- a/.github/workflows/generic-release.yml
+++ b/.github/workflows/generic-release.yml
@@ -51,7 +51,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: zlib wrapper test
       run: |
-        make gpp6install valgrindinstall
+        make valgrindinstall
         make -C zlibWrapper test
         make -C zlibWrapper valgrindTest
 

From 9648bf027bee71db8f2cad1250c6c65730b0f2a1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 17:10:57 -0800
Subject: [PATCH 03/80] try to keep libzstd.a "as is" once created

to be compatible with scenarios such as
`make -j allmost`
---
 Makefile          |  7 +++----
 lib/Makefile      | 10 +++++-----
 programs/Makefile |  2 +-
 tests/Makefile    |  9 ++++-----
 4 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 2832fb475..f5bc70440 100644
--- a/Makefile
+++ b/Makefile
@@ -48,7 +48,7 @@ allmost: allzstd zlibwrapper
 
 # skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
-allzstd: lib-all
+allzstd: lib
 	$(Q)$(MAKE) -C $(PRGDIR) all
 	$(Q)$(MAKE) -C $(TESTDIR) all
 
@@ -57,9 +57,8 @@ all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32
 
-.PHONY: lib lib-release libzstd.a
-lib-all : lib
-lib lib-release lib-all :
+.PHONY: lib lib-release
+lib lib-release :
 	$(Q)$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
diff --git a/lib/Makefile b/lib/Makefile
index 869d76630..c9bd42449 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -215,13 +215,10 @@ SET_CACHE_DIRECTORY = \
     LDFLAGS="$(LDFLAGS)"
 
 
-.PHONY: lib-all all clean install uninstall
-
-# alias
-lib-all: all
-
+.PHONY: all
 all: lib
 
+
 .PHONY: libzstd.a  # must be run every time
 
 ifndef BUILD_DIR
@@ -339,6 +336,7 @@ libzstd-nomt: $(ZSTD_NOMT_FILES)
 	@echo files : $(ZSTD_NOMT_FILES)
 	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 
+.PHONY: clean
 clean:
 	$(RM) -r *.dSYM   # macOS-specific
 	$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
@@ -407,6 +405,7 @@ libzstd.pc: libzstd.pc.in
           -e 's|@VERSION@|$(VERSION)|' \
           $< >$@
 
+.PHONY: install
 install: install-pc install-static install-shared install-includes
 	@echo zstd static and shared library installed
 
@@ -437,6 +436,7 @@ install-includes:
 	$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
 	$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
 
+.PHONY: uninstall
 uninstall:
 	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
 	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
diff --git a/programs/Makefile b/programs/Makefile
index 8641d0ee4..e932c1293 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -229,7 +229,7 @@ $(BUILD_DIR)/zstd : $(ZSTD_OBJ)
 	@echo "$(LZMA_MSG)"
 	@echo "$(LZ4_MSG)"
 	@echo LINK $@
-	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
+	$(LINK.o) $^ -o $(LDLIBS) $@$(EXT)
 
 ifeq ($(HAVE_HASH),1)
 SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ")
diff --git a/tests/Makefile b/tests/Makefile
index 42bc353c0..1c667cf86 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -38,8 +38,8 @@ CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
               -Wstrict-prototypes -Wundef                                     \
               -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
               -Wredundant-decls -Wmissing-prototypes
-CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CFLAGS     += $(DEBUGFLAGS)
+CPPFLAGS   += $(MOREFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
@@ -107,7 +107,6 @@ libzstd :
 %-dll : libzstd
 %-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd
 
-.PHONY: $(ZSTDDIR)/libzstd.a
 $(ZSTDDIR)/libzstd.a :
 	$(MAKE) -C $(ZSTDDIR) libzstd.a
 
@@ -146,7 +145,7 @@ fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR
 # note : broken : requires symbols unavailable from dynamic library
 fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c
 #	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
-	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
+	$(LINK.c) $^ $(LDLIBS) -o $@$(EXT)
 
 fuzzer : CPPFLAGS += $(MULTITHREAD_CPP)
 fuzzer : LDFLAGS += $(MULTITHREAD_LD)
@@ -165,7 +164,7 @@ zbufftest zbufftest32 zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest zbufftest32 zbufftest-dll : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
 zbufftest32 : CFLAGS +=  -m32
 zbufftest zbufftest32 : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
+	$(LINK.c) $^ -o $@$(EXT)
 
 zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)

From f9884036c2ce6cca05d5d253ba9695968a5c8da2 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 17:19:23 -0800
Subject: [PATCH 04/80] fixed zstd recipe

---
 programs/Makefile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index e932c1293..86c4610cc 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -196,7 +196,8 @@ SET_CACHE_DIRECTORY = \
     BUILD_DIR=obj/$(HASH_DIR) \
     CPPFLAGS="$(CPPFLAGS)" \
     CFLAGS="$(CFLAGS)" \
-    LDFLAGS="$(LDFLAGS)"
+    LDFLAGS="$(LDFLAGS)" \
+    LDLIBS="$(LDLIBS)"
 
 
 .PHONY: all
@@ -207,7 +208,8 @@ allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-di
 
 .PHONY: zstd  # must always be run
 zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
-zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
+zstd : LDFLAGS += $(THREAD_LD) $(DEBUGFLAGS_LD)
+zstd : LDLIBS += $(ZLIBLD) $(LZMALD) $(LZ4LD)
 zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 ifneq (,$(filter Windows%,$(OS)))
 zstd : $(RES_FILE)
@@ -229,7 +231,7 @@ $(BUILD_DIR)/zstd : $(ZSTD_OBJ)
 	@echo "$(LZMA_MSG)"
 	@echo "$(LZ4_MSG)"
 	@echo LINK $@
-	$(LINK.o) $^ -o $(LDLIBS) $@$(EXT)
+	$(LINK.o) $^ $(LDLIBS) -o $@$(EXT)
 
 ifeq ($(HAVE_HASH),1)
 SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ")

From 9a9d3f76c48f8223cf4a5cd1279ad29615a5c2e0 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 17:53:04 -0800
Subject: [PATCH 05/80] fixed zstd+sanitizer build

---
 .github/workflows/generic-dev.yml | 2 +-
 programs/Makefile                 | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml
index 8882fcc0b..85987ef3a 100644
--- a/.github/workflows/generic-dev.yml
+++ b/.github/workflows/generic-dev.yml
@@ -71,7 +71,7 @@ jobs:
         make gcc8install
         CC=gcc-8 CFLAGS="-Werror" make -j all
         make clean
-        CC=gcc-8 make -j uasan-test-zstd </dev/null
+        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
 
   gcc-6-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
diff --git a/programs/Makefile b/programs/Makefile
index 86c4610cc..b6f3a916c 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -61,8 +61,10 @@ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-prototypes -Wundef -Wpointer-arith \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
             -Wredundant-decls -Wmissing-prototypes -Wc++-compat
-CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CFLAGS   += $(DEBUGFLAGS)
+CPPFLAGS += $(MOREFLAGS)
+LDFLAGS  += $(MOREFLAGS)
+FLAGS     = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 ZSTDLIB_COMMON := $(ZSTDDIR)/common
 ZSTDLIB_COMPRESS := $(ZSTDDIR)/compress

From 52aa7f47ddae40b24915f2275e07f26dd7dc0bc0 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 18:29:36 -0800
Subject: [PATCH 06/80] updated clang+msan test

---
 .github/workflows/generic-dev.yml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml
index 85987ef3a..1cda7ddc5 100644
--- a/.github/workflows/generic-dev.yml
+++ b/.github/workflows/generic-dev.yml
@@ -84,17 +84,15 @@ jobs:
         make clean
         CC=gcc-6 make -j uasan-test-zstd32
 
-  clang-38-msan-testzstd:
-    runs-on: ubuntu-16.04 # fails on 18.04
+  clang-msan-testzstd:
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: clang-3.8 + MSan + Test Zstd
+    - name: clang + MSan + Test Zstd
       run: |
-        # make clang38install (doesn't work)
-        sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
         sudo apt-get update
-        sudo apt-get install clang-3.8
-        CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
+        sudo apt-get install clang
+        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
 
     # Note : external libraries must be turned off when using MSAN tests,
     # because they are not msan-instrumented,

From 8233f55df8435b2d5d429da5b0fabc2e43533d32 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 21:31:58 -0800
Subject: [PATCH 07/80] verbose cross-compile tests on circleci

for better diagnosis
---
 .circleci/config.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9738c1bd4..4529743ea 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -31,9 +31,9 @@ jobs:
           command: |
             make gnu90build; make clean
             make gnu99build; make clean
-            make ppc64build; make clean
-            make ppcbuild  ; make clean
-            make armbuild  ; make clean
+            make ppc64build V=1; make clean
+            make ppcbuild   V=1; make clean
+            make armbuild   V=1; make clean
             make -C tests test-legacy test-longmatch; make clean
             make -C lib libzstd-nomt; make clean
   # This step is only run on release tags.

From 585196353d8d243c5da76318db06cfbe7c4e194b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 20 Dec 2020 21:44:40 -0800
Subject: [PATCH 08/80] fix ppc64 build on circleci

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index f5bc70440..c7d33b286 100644
--- a/Makefile
+++ b/Makefile
@@ -224,10 +224,10 @@ aarch64build: clean
 	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
 
 ppcbuild: clean
-	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
 
 ppc64build: clean
-	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS=-Werror TARGET_ARCH=-m64 $(MAKE) -j allzstd
 
 armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest

From 7e11bd012b4e25e1b55b43e97bbef16d2c886e14 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 21 Dec 2020 11:13:22 -0500
Subject: [PATCH 09/80] Implement skippable frame function

---
 lib/compress/zstd_compress.c | 20 ++++++++++++++++++++
 lib/zstd.h                   | 16 ++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 386b051df..3e6bf7464 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2954,6 +2954,26 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     return pos;
 }
 
+/* ZSTD_generateSkippableFrame_advanced() :
+ * Writes out a skippable frame with the specified magic number variant (16 are supported), 
+ * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data.
+ * 
+ * Returns the total number of bytes written, or a ZSTD error code.
+ */
+size_t ZSTD_generateSkippableFrame_advanced(void* dst, size_t dstCapacity,
+                                            const void* src, size_t srcSize, U32 magicVariant) {
+    BYTE* op = dst;
+    RETURN_ERROR_IF(dstCapacity < srcSize + 8 /* Skippable frame overhead */,
+                    dstSize_tooSmall, "Not enough room for skippable frame");
+    RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame");
+    RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported");
+
+    MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant));
+    MEM_writeLE32(op+4, (U32)srcSize);
+    ZSTD_memcpy(op+8, src, srcSize);
+    return srcSize + 8;
+}
+
 /* ZSTD_writeLastEmptyBlock() :
  * output an empty Block with end-of-frame mark to complete a frame
  * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
diff --git a/lib/zstd.h b/lib/zstd.h
index b0ecdf553..c34b43976 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1372,6 +1372,22 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size
                                   const void* src, size_t srcSize);
 
 
+/*! ZSTD_generateSkippableFrame() :
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ * 
+ * Skippable frames have a 4-byte magic number, which can range from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ * the final magic number will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ * 
+ * Returns an error if destination buffer is not large enough, if the source size is not representable
+ * with a 4-byte unsigned int, or if the magicVariant is greater than 15.
+ * 
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_API size_t ZSTD_generateSkippableFrame(void* dst, size_t dstCapacity,
+                                               const void* src, size_t srcSize, unsigned magicVariant);
+
+
 /***************************************
 *  Memory management
 ***************************************/

From 339d8ba103b3dda92801bd8a8ea511f14a087716 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 21 Dec 2020 11:33:27 -0500
Subject: [PATCH 10/80] Add unit test

---
 tests/fuzzer.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2e5d70ef2..eb84dbfa2 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1570,6 +1570,11 @@ static int basicUnitTests(U32 const seed, double compressibility)
         int const segs = 4;
         /* only use the first half so we don't push against size limit of compressedBuffer */
         size_t const segSize = (CNBuffSize / 2) / segs;
+
+        U32 skippableSize;
+        const U32 skipLen = 129 KB;
+        char* skipBuff = (char*)malloc(skipLen);
+        memset(skipBuff, 0, skipLen);
         for (i = 0; i < segs; i++) {
             CHECK_NEWV(r, ZSTD_compress(
                             (BYTE*)compressedBuffer + off, CNBuffSize - off,
@@ -1578,13 +1583,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
             off += r;
             if (i == segs/2) {
                 /* insert skippable frame */
-                const U32 skipLen = 129 KB;
-                MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
-                MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
-                off += skipLen + ZSTD_SKIPPABLEHEADERSIZE;
+                skippableSize = ZSTD_generateSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15);
+                CHECK_Z(skippableSize);
+                off += skippableSize;
             }
         }
         cSize = off;
+        free(skipBuff);
     }
     DISPLAYLEVEL(3, "OK \n");
 

From 5c41490bfe8149933213a1245acff586fb65e587 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 21 Dec 2020 11:33:41 -0500
Subject: [PATCH 11/80] Use pre-defined constants

---
 lib/compress/zstd_compress.c | 12 ++++++------
 lib/zstd.h                   | 13 +++++++------
 tests/fuzzer.c               |  2 +-
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3e6bf7464..2e0edaf99 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2954,16 +2954,16 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     return pos;
 }
 
-/* ZSTD_generateSkippableFrame_advanced() :
+/* ZSTD_writeSkippableFrame_advanced() :
  * Writes out a skippable frame with the specified magic number variant (16 are supported), 
  * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data.
  * 
  * Returns the total number of bytes written, or a ZSTD error code.
  */
-size_t ZSTD_generateSkippableFrame_advanced(void* dst, size_t dstCapacity,
-                                            const void* src, size_t srcSize, U32 magicVariant) {
-    BYTE* op = dst;
-    RETURN_ERROR_IF(dstCapacity < srcSize + 8 /* Skippable frame overhead */,
+size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+                                const void* src, size_t srcSize, unsigned magicVariant) {
+    BYTE* op = (BYTE*)dst;
+    RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */,
                     dstSize_tooSmall, "Not enough room for skippable frame");
     RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame");
     RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported");
@@ -2971,7 +2971,7 @@ size_t ZSTD_generateSkippableFrame_advanced(void* dst, size_t dstCapacity,
     MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant));
     MEM_writeLE32(op+4, (U32)srcSize);
     ZSTD_memcpy(op+8, src, srcSize);
-    return srcSize + 8;
+    return srcSize + ZSTD_SKIPPABLEHEADERSIZE;
 }
 
 /* ZSTD_writeLastEmptyBlock() :
diff --git a/lib/zstd.h b/lib/zstd.h
index c34b43976..61e118647 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1372,20 +1372,21 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size
                                   const void* src, size_t srcSize);
 
 
-/*! ZSTD_generateSkippableFrame() :
+/*! ZSTD_writeSkippableFrame() :
  * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
  * 
- * Skippable frames have a 4-byte magic number, which can range from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
  * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
- * the final magic number will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
  * 
  * Returns an error if destination buffer is not large enough, if the source size is not representable
- * with a 4-byte unsigned int, or if the magicVariant is greater than 15.
+ * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
  * 
  * @return : number of bytes written or a ZSTD error.
  */
-ZSTDLIB_API size_t ZSTD_generateSkippableFrame(void* dst, size_t dstCapacity,
-                                               const void* src, size_t srcSize, unsigned magicVariant);
+ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+                                            const void* src, size_t srcSize, unsigned magicVariant);
 
 
 /***************************************
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index eb84dbfa2..71ed3e0b8 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1583,7 +1583,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             off += r;
             if (i == segs/2) {
                 /* insert skippable frame */
-                skippableSize = ZSTD_generateSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15);
+                skippableSize = ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15);
                 CHECK_Z(skippableSize);
                 off += skippableSize;
             }

From f2ac2b7bcf6294f1c3a8d62b5f2ca78cfc6fb980 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 21 Dec 2020 15:43:14 -0800
Subject: [PATCH 12/80] try to fix cross-compiler tests

---
 Makefile          | 2 +-
 appveyor.yml      | 4 ++--
 programs/Makefile | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index c7d33b286..bf9cf3e34 100644
--- a/Makefile
+++ b/Makefile
@@ -227,7 +227,7 @@ ppcbuild: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
 
 ppc64build: clean
-	CC=powerpc-linux-gnu-gcc CFLAGS=-Werror TARGET_ARCH=-m64 $(MAKE) -j allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
 
 armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
diff --git a/appveyor.yml b/appveyor.yml
index 0e8725575..172159315 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -26,7 +26,7 @@
     - COMPILER: "clang"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
       BUILD:    "true"
 
     - COMPILER: "gcc"
@@ -204,7 +204,7 @@
     - COMPILER: "clang"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
+      SCRIPT:   "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
 
     - COMPILER: "visual"
       HOST:     "visual"
diff --git a/programs/Makefile b/programs/Makefile
index b6f3a916c..e66d182fe 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -233,7 +233,7 @@ $(BUILD_DIR)/zstd : $(ZSTD_OBJ)
 	@echo "$(LZMA_MSG)"
 	@echo "$(LZ4_MSG)"
 	@echo LINK $@
-	$(LINK.o) $^ $(LDLIBS) -o $@$(EXT)
+	$(CC) $(FLAGS) $^ $(LDLIBS) -o $@$(EXT)
 
 ifeq ($(HAVE_HASH),1)
 SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ")

From f1225b186ea716cff502110bbecc238d60abc25d Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 21 Dec 2020 15:47:20 -0800
Subject: [PATCH 13/80] try to fix 32-bit test on github actions

for some reasons, this test fails at _installing_ 32-bit dependencies
using the exact same command that actually works in other tests !!?

It's unclear why it fails repeateadly for this test only.
Try another way to install dependencies to fix that.
---
 .github/workflows/generic-dev.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml
index 8882fcc0b..621aea32c 100644
--- a/.github/workflows/generic-dev.yml
+++ b/.github/workflows/generic-dev.yml
@@ -41,7 +41,8 @@ jobs:
     - uses: actions/checkout@v2
     - name: make check on 32-bit
       run: |
-        make libc6install
+        sudo apt update
+        APT_PACKAGES="gcc-multilib" make apt-install
         CFLAGS="-m32 -O1 -fstack-protector" make check V=1
 
   gcc-6-7-libzstd:

From dd1a7e41ee6619a8d01373259979008db4f13a06 Mon Sep 17 00:00:00 2001
From: Gregory Szorc <gregory.szorc@gmail.com>
Date: Sat, 26 Dec 2020 10:00:53 -0700
Subject: [PATCH 14/80] Add ifndef guards for _LARGEFILE_SOURCE and
 _LARGEFILE64_SOURCE

This ensures the symbols aren't redefined, which would result in a compiler
error.

I was getting redefined symbols for _LARGEFILE64_SOURCE when building for
32-bit x86 Linux on an older CentOS release in a CI environment. With this
change, I'm able to compile the single file library in this environment.

Closes #2443.
---
 lib/dictBuilder/zdict.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 79c522ef3..baaa57153 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -23,9 +23,13 @@
 /* Unix Large Files support (>4GB) */
 #define _FILE_OFFSET_BITS 64
 #if (defined(__sun__) && (!defined(__LP64__)))   /* Sun Solaris 32-bits requires specific definitions */
+#  ifndef _LARGEFILE_SOURCE
 #  define _LARGEFILE_SOURCE
+#  endif
 #elif ! defined(__LP64__)                        /* No point defining Large file for 64 bit */
+#  ifndef _LARGEFILE64_SOURCE
 #  define _LARGEFILE64_SOURCE
+#  endif
 #endif
 
 

From f1585fefa20558f73b1c24819850454707ba6b2f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 28 Dec 2020 11:18:58 -0800
Subject: [PATCH 15/80] remove flackey msan ossfuzz test

while waiting for it to be fixed
---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0e14345dc..5e5aae112 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -6,7 +6,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        sanitizer: [address, undefined, memory]
+        sanitizer: [address, undefined]
     steps:
     - name: Build Fuzzers (${{ matrix.sanitizer }})
       id: build

From 7f8be046b95a9aef96db5a6e85696144d94caeee Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 28 Dec 2020 14:07:31 -0800
Subject: [PATCH 16/80] fixed minor warnings introduced in #2439

---
 tests/fuzzer.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 71ed3e0b8..d8199300b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1571,9 +1571,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
         /* only use the first half so we don't push against size limit of compressedBuffer */
         size_t const segSize = (CNBuffSize / 2) / segs;
 
-        U32 skippableSize;
         const U32 skipLen = 129 KB;
-        char* skipBuff = (char*)malloc(skipLen);
+        char* const skipBuff = (char*)malloc(skipLen);
+        assert(skipBuff != NULL);
         memset(skipBuff, 0, skipLen);
         for (i = 0; i < segs; i++) {
             CHECK_NEWV(r, ZSTD_compress(
@@ -1583,7 +1583,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
             off += r;
             if (i == segs/2) {
                 /* insert skippable frame */
-                skippableSize = ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15);
+                size_t const skippableSize =
+                    ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize,
+                                             skipBuff, skipLen, seed % 15);
                 CHECK_Z(skippableSize);
                 off += skippableSize;
             }
@@ -2744,7 +2746,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         free(seqs);
     }
     DISPLAYLEVEL(3, "OK \n");
-    
+
     DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
     {
         size_t srcSize = 500 KB;

From ff2f888d56b540823aacfff4f89f313a64c02fb9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 29 Dec 2020 11:44:37 -0800
Subject: [PATCH 17/80] fixed one more minor cast issue

can't use address calculation with `void*`
---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index d8199300b..88c879c6c 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1584,7 +1584,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             if (i == segs/2) {
                 /* insert skippable frame */
                 size_t const skippableSize =
-                    ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize,
+                    ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
                                              skipBuff, skipLen, seed % 15);
                 CHECK_Z(skippableSize);
                 off += skippableSize;

From 66e811d782a8d6c6a9dff2a57cfe5472dc95483d Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 4 Jan 2021 17:53:52 -0500
Subject: [PATCH 18/80] [license] Update year to 2021

---
 Makefile                                           | 2 +-
 contrib/diagnose_corruption/Makefile               | 2 +-
 contrib/freestanding_lib/freestanding.py           | 2 +-
 contrib/linux-kernel/Makefile                      | 2 +-
 contrib/linux-kernel/mem.h                         | 2 +-
 contrib/linux-kernel/test/include/linux/compiler.h | 2 +-
 contrib/linux-kernel/test/include/linux/errno.h    | 2 +-
 contrib/linux-kernel/test/include/linux/kernel.h   | 2 +-
 contrib/linux-kernel/test/include/linux/limits.h   | 2 +-
 contrib/linux-kernel/test/include/linux/math64.h   | 2 +-
 contrib/linux-kernel/test/include/linux/module.h   | 2 +-
 contrib/linux-kernel/test/include/linux/printk.h   | 2 +-
 contrib/linux-kernel/test/include/linux/stddef.h   | 2 +-
 contrib/linux-kernel/test/include/linux/swab.h     | 2 +-
 contrib/linux-kernel/test/include/linux/types.h    | 2 +-
 contrib/linux-kernel/test/static_test.c            | 2 +-
 contrib/linux-kernel/test/test.c                   | 2 +-
 contrib/linux-kernel/zstd_deps.h                   | 2 +-
 contrib/single_file_libs/zstd-in.c                 | 2 +-
 contrib/single_file_libs/zstddeclib-in.c           | 2 +-
 doc/educational_decoder/Makefile                   | 2 +-
 doc/educational_decoder/harness.c                  | 2 +-
 doc/educational_decoder/zstd_decompress.c          | 2 +-
 doc/educational_decoder/zstd_decompress.h          | 2 +-
 examples/Makefile                                  | 2 +-
 examples/common.h                                  | 2 +-
 examples/dictionary_compression.c                  | 2 +-
 examples/dictionary_decompression.c                | 2 +-
 examples/multiple_simple_compression.c             | 2 +-
 examples/multiple_streaming_compression.c          | 2 +-
 examples/simple_compression.c                      | 2 +-
 examples/simple_decompression.c                    | 2 +-
 examples/streaming_compression.c                   | 2 +-
 examples/streaming_compression_thread_pool.c       | 2 +-
 examples/streaming_decompression.c                 | 2 +-
 examples/streaming_memory_usage.c                  | 2 +-
 lib/Makefile                                       | 2 +-
 lib/common/bitstream.h                             | 2 +-
 lib/common/compiler.h                              | 2 +-
 lib/common/cpu.h                                   | 2 +-
 lib/common/debug.c                                 | 2 +-
 lib/common/debug.h                                 | 2 +-
 lib/common/entropy_common.c                        | 2 +-
 lib/common/error_private.c                         | 2 +-
 lib/common/error_private.h                         | 2 +-
 lib/common/fse.h                                   | 2 +-
 lib/common/fse_decompress.c                        | 2 +-
 lib/common/huf.h                                   | 2 +-
 lib/common/mem.h                                   | 2 +-
 lib/common/pool.c                                  | 2 +-
 lib/common/pool.h                                  | 2 +-
 lib/common/xxhash.c                                | 2 +-
 lib/common/xxhash.h                                | 2 +-
 lib/common/zstd_common.c                           | 2 +-
 lib/common/zstd_deps.h                             | 2 +-
 lib/common/zstd_errors.h                           | 2 +-
 lib/common/zstd_internal.h                         | 2 +-
 lib/compress/fse_compress.c                        | 2 +-
 lib/compress/hist.c                                | 2 +-
 lib/compress/hist.h                                | 2 +-
 lib/compress/huf_compress.c                        | 2 +-
 lib/compress/zstd_compress.c                       | 2 +-
 lib/compress/zstd_compress_internal.h              | 2 +-
 lib/compress/zstd_compress_literals.c              | 2 +-
 lib/compress/zstd_compress_literals.h              | 2 +-
 lib/compress/zstd_compress_sequences.c             | 2 +-
 lib/compress/zstd_compress_sequences.h             | 2 +-
 lib/compress/zstd_compress_superblock.c            | 2 +-
 lib/compress/zstd_compress_superblock.h            | 2 +-
 lib/compress/zstd_cwksp.h                          | 2 +-
 lib/compress/zstd_double_fast.c                    | 2 +-
 lib/compress/zstd_double_fast.h                    | 2 +-
 lib/compress/zstd_fast.c                           | 2 +-
 lib/compress/zstd_fast.h                           | 2 +-
 lib/compress/zstd_lazy.c                           | 2 +-
 lib/compress/zstd_lazy.h                           | 2 +-
 lib/compress/zstd_ldm.c                            | 2 +-
 lib/compress/zstd_ldm.h                            | 2 +-
 lib/compress/zstd_opt.c                            | 2 +-
 lib/compress/zstd_opt.h                            | 2 +-
 lib/compress/zstdmt_compress.c                     | 2 +-
 lib/compress/zstdmt_compress.h                     | 2 +-
 lib/decompress/huf_decompress.c                    | 2 +-
 lib/decompress/zstd_ddict.c                        | 2 +-
 lib/decompress/zstd_ddict.h                        | 2 +-
 lib/decompress/zstd_decompress.c                   | 2 +-
 lib/decompress/zstd_decompress_block.c             | 2 +-
 lib/decompress/zstd_decompress_block.h             | 2 +-
 lib/decompress/zstd_decompress_internal.h          | 2 +-
 lib/deprecated/zbuff.h                             | 2 +-
 lib/deprecated/zbuff_common.c                      | 2 +-
 lib/deprecated/zbuff_compress.c                    | 2 +-
 lib/deprecated/zbuff_decompress.c                  | 2 +-
 lib/dictBuilder/cover.c                            | 2 +-
 lib/dictBuilder/cover.h                            | 2 +-
 lib/dictBuilder/fastcover.c                        | 2 +-
 lib/dictBuilder/zdict.c                            | 2 +-
 lib/dictBuilder/zdict.h                            | 2 +-
 lib/dll/example/Makefile                           | 2 +-
 lib/legacy/zstd_legacy.h                           | 2 +-
 lib/legacy/zstd_v01.c                              | 2 +-
 lib/legacy/zstd_v01.h                              | 2 +-
 lib/legacy/zstd_v02.c                              | 2 +-
 lib/legacy/zstd_v02.h                              | 2 +-
 lib/legacy/zstd_v03.c                              | 2 +-
 lib/legacy/zstd_v03.h                              | 2 +-
 lib/legacy/zstd_v04.c                              | 2 +-
 lib/legacy/zstd_v04.h                              | 2 +-
 lib/legacy/zstd_v05.c                              | 2 +-
 lib/legacy/zstd_v05.h                              | 2 +-
 lib/legacy/zstd_v06.c                              | 2 +-
 lib/legacy/zstd_v06.h                              | 2 +-
 lib/legacy/zstd_v07.c                              | 2 +-
 lib/legacy/zstd_v07.h                              | 2 +-
 lib/zstd.h                                         | 2 +-
 programs/Makefile                                  | 2 +-
 programs/benchfn.c                                 | 2 +-
 programs/benchfn.h                                 | 2 +-
 programs/benchzstd.c                               | 2 +-
 programs/benchzstd.h                               | 2 +-
 programs/datagen.c                                 | 2 +-
 programs/datagen.h                                 | 2 +-
 programs/dibio.c                                   | 2 +-
 programs/dibio.h                                   | 2 +-
 programs/fileio.c                                  | 2 +-
 programs/fileio.h                                  | 2 +-
 programs/platform.h                                | 2 +-
 programs/timefn.c                                  | 2 +-
 programs/timefn.h                                  | 2 +-
 programs/util.c                                    | 2 +-
 programs/util.h                                    | 2 +-
 programs/windres/verrsrc.h                         | 2 +-
 programs/zstdcli.c                                 | 2 +-
 tests/DEPRECATED-test-zstd-speed.py                | 2 +-
 tests/Makefile                                     | 2 +-
 tests/automated_benchmarking.py                    | 2 +-
 tests/bigdict.c                                    | 2 +-
 tests/checkTag.c                                   | 2 +-
 tests/datagencli.c                                 | 2 +-
 tests/decodecorpus.c                               | 2 +-
 tests/fullbench.c                                  | 2 +-
 tests/fuzz/Makefile                                | 2 +-
 tests/fuzz/block_decompress.c                      | 2 +-
 tests/fuzz/block_round_trip.c                      | 2 +-
 tests/fuzz/decompress_dstSize_tooSmall.c           | 2 +-
 tests/fuzz/dictionary_decompress.c                 | 2 +-
 tests/fuzz/dictionary_loader.c                     | 2 +-
 tests/fuzz/dictionary_round_trip.c                 | 2 +-
 tests/fuzz/dictionary_stream_round_trip.c          | 2 +-
 tests/fuzz/fse_read_ncount.c                       | 2 +-
 tests/fuzz/fuzz.h                                  | 2 +-
 tests/fuzz/fuzz.py                                 | 2 +-
 tests/fuzz/fuzz_data_producer.c                    | 2 +-
 tests/fuzz/fuzz_data_producer.h                    | 2 +-
 tests/fuzz/fuzz_helpers.c                          | 4 ++--
 tests/fuzz/fuzz_helpers.h                          | 2 +-
 tests/fuzz/raw_dictionary_round_trip.c             | 2 +-
 tests/fuzz/regression_driver.c                     | 2 +-
 tests/fuzz/sequence_compression_api.c              | 2 +-
 tests/fuzz/simple_compress.c                       | 2 +-
 tests/fuzz/simple_decompress.c                     | 2 +-
 tests/fuzz/simple_round_trip.c                     | 2 +-
 tests/fuzz/stream_decompress.c                     | 2 +-
 tests/fuzz/stream_round_trip.c                     | 2 +-
 tests/fuzz/zstd_frame_info.c                       | 2 +-
 tests/fuzz/zstd_helpers.c                          | 2 +-
 tests/fuzz/zstd_helpers.h                          | 2 +-
 tests/fuzzer.c                                     | 2 +-
 tests/gzip/Makefile                                | 2 +-
 tests/invalidDictionaries.c                        | 2 +-
 tests/legacy.c                                     | 2 +-
 tests/longmatch.c                                  | 2 +-
 tests/paramgrill.c                                 | 2 +-
 tests/poolTests.c                                  | 2 +-
 tests/rateLimiter.py                               | 2 +-
 tests/regression/Makefile                          | 2 +-
 tests/regression/config.c                          | 2 +-
 tests/regression/config.h                          | 2 +-
 tests/regression/data.c                            | 2 +-
 tests/regression/data.h                            | 2 +-
 tests/regression/levels.h                          | 2 +-
 tests/regression/method.c                          | 2 +-
 tests/regression/method.h                          | 2 +-
 tests/regression/result.c                          | 2 +-
 tests/regression/result.h                          | 2 +-
 tests/regression/test.c                            | 2 +-
 tests/roundTripCrash.c                             | 2 +-
 tests/seqgen.c                                     | 2 +-
 tests/seqgen.h                                     | 2 +-
 tests/test-license.py                              | 4 ++--
 tests/test-zstd-versions.py                        | 2 +-
 tests/zbufftest.c                                  | 2 +-
 tests/zstreamtest.c                                | 2 +-
 zlibWrapper/gzcompatibility.h                      | 2 +-
 zlibWrapper/zstd_zlibwrapper.c                     | 2 +-
 zlibWrapper/zstd_zlibwrapper.h                     | 2 +-
 196 files changed, 198 insertions(+), 198 deletions(-)

diff --git a/Makefile b/Makefile
index bf9cf3e34..9f94baaa1 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/diagnose_corruption/Makefile b/contrib/diagnose_corruption/Makefile
index dc6fd2906..a21a00212 100644
--- a/contrib/diagnose_corruption/Makefile
+++ b/contrib/diagnose_corruption/Makefile
@@ -32,4 +32,4 @@ check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a
 
 .PHONY: clean
 clean:
-	rm -f check_flipped_bits
\ No newline at end of file
+	rm -f check_flipped_bits
diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index 9e91a489a..f8f206944 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) 2020-2020, Facebook, Inc.
+# Copyright (c) 2021-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index b8a65e910..a0cc67ac4 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2020, Facebook, Inc.
+# Copyright (c) 2015-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/mem.h b/contrib/linux-kernel/mem.h
index 54832a6da..63f1a68f2 100644
--- a/contrib/linux-kernel/mem.h
+++ b/contrib/linux-kernel/mem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h
index b614b27e5..58c711816 100644
--- a/contrib/linux-kernel/test/include/linux/compiler.h
+++ b/contrib/linux-kernel/test/include/linux/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h
index 11c54b93c..b24752257 100644
--- a/contrib/linux-kernel/test/include/linux/errno.h
+++ b/contrib/linux-kernel/test/include/linux/errno.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h
index 1124f02b0..9b481ef47 100644
--- a/contrib/linux-kernel/test/include/linux/kernel.h
+++ b/contrib/linux-kernel/test/include/linux/kernel.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h
index 7f8d18d40..db9c09904 100644
--- a/contrib/linux-kernel/test/include/linux/limits.h
+++ b/contrib/linux-kernel/test/include/linux/limits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h
index 4bc7f4b56..8eefa2d5c 100644
--- a/contrib/linux-kernel/test/include/linux/math64.h
+++ b/contrib/linux-kernel/test/include/linux/math64.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h
index 8fd66930d..be6d20dae 100644
--- a/contrib/linux-kernel/test/include/linux/module.h
+++ b/contrib/linux-kernel/test/include/linux/module.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h
index 2ca2fb3a2..eab08e0c4 100644
--- a/contrib/linux-kernel/test/include/linux/printk.h
+++ b/contrib/linux-kernel/test/include/linux/printk.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h
index c00d06547..8538eb3e4 100644
--- a/contrib/linux-kernel/test/include/linux/stddef.h
+++ b/contrib/linux-kernel/test/include/linux/stddef.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h
index 693b797e9..783046b42 100644
--- a/contrib/linux-kernel/test/include/linux/swab.h
+++ b/contrib/linux-kernel/test/include/linux/swab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h
index 6db834b18..459a45700 100644
--- a/contrib/linux-kernel/test/include/linux/types.h
+++ b/contrib/linux-kernel/test/include/linux/types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/static_test.c b/contrib/linux-kernel/test/static_test.c
index 53c559c02..6ac534dab 100644
--- a/contrib/linux-kernel/test/static_test.c
+++ b/contrib/linux-kernel/test/static_test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Facebook, Inc.
+ * Copyright (c) 2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c
index 9579976af..47af82f19 100644
--- a/contrib/linux-kernel/test/test.c
+++ b/contrib/linux-kernel/test/test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 7-2020, Facebook, Inc.
+ * Copyright (c) 7-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/linux-kernel/zstd_deps.h b/contrib/linux-kernel/zstd_deps.h
index 4a6d35f1d..f9fd4c0f9 100644
--- a/contrib/linux-kernel/zstd_deps.h
+++ b/contrib/linux-kernel/zstd_deps.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/single_file_libs/zstd-in.c b/contrib/single_file_libs/zstd-in.c
index 4f0fb56a2..0addbab3a 100644
--- a/contrib/single_file_libs/zstd-in.c
+++ b/contrib/single_file_libs/zstd-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/single_file_libs/zstddeclib-in.c b/contrib/single_file_libs/zstddeclib-in.c
index f461b555b..f94467e7e 100644
--- a/contrib/single_file_libs/zstddeclib-in.c
+++ b/contrib/single_file_libs/zstddeclib-in.c
@@ -8,7 +8,7 @@
  * \endcode
  */
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index 316c6eadc..b57b6080b 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index 1403a6ed6..0bf7579f7 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Facebook, Inc.
+ * Copyright (c) 2017-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index 605918b39..b4b34b501 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Facebook, Inc.
+ * Copyright (c) 2017-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/doc/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h
index 2b44eee95..eeeb61beb 100644
--- a/doc/educational_decoder/zstd_decompress.h
+++ b/doc/educational_decoder/zstd_decompress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/Makefile b/examples/Makefile
index f5e3274b1..1787bf291 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/common.h b/examples/common.h
index 4492c7e4e..a3f743ba3 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c
index d9aad45a7..559977045 100644
--- a/examples/dictionary_compression.c
+++ b/examples/dictionary_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020 Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c
index 7e50986e3..6bf77050e 100644
--- a/examples/dictionary_decompression.c
+++ b/examples/dictionary_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index e409467b2..e03ce8680 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index 8a4dc96c1..5a92a3120 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_compression.c b/examples/simple_compression.c
index 618080b33..3d6c3ae9b 100644
--- a/examples/simple_compression.c
+++ b/examples/simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index e108987c6..4a179d185 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index 045437f28..6a039dc64 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression_thread_pool.c b/examples/streaming_compression_thread_pool.c
index 22c3b2efa..7489eae93 100644
--- a/examples/streaming_compression_thread_pool.c
+++ b/examples/streaming_compression_thread_pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Martin Liska, SUSE, Facebook, Inc.
+ * Copyright (c) 2021, Martin Liska, SUSE, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 26eda3441..3f8e46002 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c
index 37dd660e4..8c1e9be61 100644
--- a/examples/streaming_memory_usage.c
+++ b/examples/streaming_memory_usage.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/Makefile b/lib/Makefile
index c9bd42449..a0c0ca10b 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index d9a273010..aa526b429 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * bitstream
  * Part of FSE library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 3e454f38c..2697f6051 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/cpu.h b/lib/common/cpu.h
index cb210593e..1b4d26dda 100644
--- a/lib/common/cpu.h
+++ b/lib/common/cpu.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2020, Facebook, Inc.
+ * Copyright (c) 2018-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/debug.c b/lib/common/debug.c
index f303f4a2e..cd1742c74 100644
--- a/lib/common/debug.c
+++ b/lib/common/debug.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/debug.h b/lib/common/debug.h
index 8b5734366..b45cda8f0 100644
--- a/lib/common/debug.h
+++ b/lib/common/debug.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index f9fcb1acf..033c07554 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Common functions of New Generation Entropy library
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 45bba5305..b6db38018 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index 71b37b8df..9d36e891c 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/fse.h b/lib/common/fse.h
index dd5fc44e8..52924469c 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
index c164430f9..c575db0de 100644
--- a/lib/common/fse_decompress.c
+++ b/lib/common/fse_decompress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 1afef90c7..43213e574 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/mem.h b/lib/common/mem.h
index 4728ef767..9813bfc42 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/pool.c b/lib/common/pool.c
index 4c1b83376..5b092ccd2 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/pool.h b/lib/common/pool.h
index 63954ca6c..d66942a0a 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index e708df3c3..f0ac38135 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -1,6 +1,6 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) 2012-2020, Yann Collet, Facebook, Inc.
+ *  Copyright (c) 2012-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - xxHash homepage: http://www.xxhash.com
diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index eceb55d5e..31c488bf5 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -1,7 +1,7 @@
 /*
  * xxHash - Extremely Fast Hash algorithm
  * Header File
- * Copyright (c) 2012-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc.
  *
  * You can contact the author at :
  * - xxHash source repository : https://github.com/Cyan4973/xxHash
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 939e9f08f..009b46600 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_deps.h b/lib/common/zstd_deps.h
index 0fb8b7818..eebc4fdfb 100644
--- a/lib/common/zstd_deps.h
+++ b/lib/common/zstd_deps.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h
index 6d0d00300..3ba57e152 100644
--- a/lib/common/zstd_errors.h
+++ b/lib/common/zstd_errors.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 0991f20a0..53a982c73 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 304a82b3c..887bdc858 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.c b/lib/compress/hist.c
index a9659d11a..9af1aa1a3 100644
--- a/lib/compress/hist.c
+++ b/lib/compress/hist.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.h b/lib/compress/hist.h
index fb9ead683..43c578799 100644
--- a/lib/compress/hist.h
+++ b/lib/compress/hist.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 302e08864..71438d7a8 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 2e0edaf99..3cebbe173 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index c04998b8b..10846ae50 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index 6dd1c1447..8945fd596 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h
index 8b0870574..c8ebe2fc4 100644
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c
index be30c08c6..4bccf0152 100644
--- a/lib/compress/zstd_compress_sequences.c
+++ b/lib/compress/zstd_compress_sequences.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.h b/lib/compress/zstd_compress_sequences.h
index 68c6f9a5a..3629b3f31 100644
--- a/lib/compress/zstd_compress_sequences.h
+++ b/lib/compress/zstd_compress_sequences.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index e23e619ee..593ac416f 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.h b/lib/compress/zstd_compress_superblock.h
index 07f4cb1dc..8138afa4c 100644
--- a/lib/compress/zstd_compress_superblock.h
+++ b/lib/compress/zstd_compress_superblock.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index d65170b39..9050cfa8d 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index ef12a524f..9ed8fa6ef 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h
index 14d944d69..b17cf3ee5 100644
--- a/lib/compress/zstd_double_fast.h
+++ b/lib/compress/zstd_double_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index db7ce83d0..9ac8ab5c9 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h
index cf6aaa8e6..1a5fd33a0 100644
--- a/lib/compress/zstd_fast.h
+++ b/lib/compress/zstd_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 49ec1b09e..3d35ee430 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index d0214d5e7..87a397196 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 3f3d7c46a..e7fead31b 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 6561024e4..7102e4227 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index e55c459de..6ec368b85 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 9aba8a901..bf31cccf5 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 50454a50b..1a642a456 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 0a9e551c9..c69031b21 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 141820671..7699c9203 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index f5cc23b38..443fe6b48 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h
index 8906a71c9..2f1744572 100644
--- a/lib/decompress/zstd_ddict.h
+++ b/lib/decompress/zstd_ddict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 21f846bc7..172d46c04 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 19cbdc5c1..774f1da71 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_block.h b/lib/decompress/zstd_decompress_block.h
index b5715c168..891cf9835 100644
--- a/lib/decompress/zstd_decompress_block.h
+++ b/lib/decompress/zstd_decompress_block.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index f80b471e9..453487de5 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index 03cb14a03..ed98b4600 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_common.c b/lib/deprecated/zbuff_common.c
index 579bc4df1..cb370cbc5 100644
--- a/lib/deprecated/zbuff_common.c
+++ b/lib/deprecated/zbuff_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_compress.c b/lib/deprecated/zbuff_compress.c
index 2d20b1377..972afd873 100644
--- a/lib/deprecated/zbuff_compress.c
+++ b/lib/deprecated/zbuff_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
index d3c49e84b..baf18294e 100644
--- a/lib/deprecated/zbuff_decompress.c
+++ b/lib/deprecated/zbuff_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index c78af133a..62eb67166 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index 9f1cb5fb9..7bbeaac15 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Facebook, Inc.
+ * Copyright (c) 2017-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 5e60f24c5..303704ec6 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2020, Facebook, Inc.
+ * Copyright (c) 2018-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index baaa57153..2b7659aaa 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h
index b782993f9..d1ac3d41f 100644
--- a/lib/dictBuilder/zdict.h
+++ b/lib/dictBuilder/zdict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dll/example/Makefile b/lib/dll/example/Makefile
index 8f1919559..a1cc189d5 100644
--- a/lib/dll/example/Makefile
+++ b/lib/dll/example/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 6bea6a519..3f21d2267 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 13115bec5..e2b93edb7 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index 791035172..f85d65de1 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 9abb6d033..0368cb643 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 5f8f6cd60..4dc2c7f84 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index a19cb205a..e2a35b4d0 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 5fc72730c..89fa3b1bd 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 77d52555b..9c73d4a53 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 15fce0d48..8d9321534 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index ca8d5c9bf..991991e9b 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index 167d892e6..074ff1bc0 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index c4ac7dba8..ccc2336b5 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index 2fd99e629..1fa8f9dc5 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 049ba4749..fe970a651 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index 9da50c4e6..3982a04e3 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd.h b/lib/zstd.h
index 61e118647..4ff03e1e9 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/Makefile b/programs/Makefile
index e66d182fe..65fcb0d98 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.c b/programs/benchfn.c
index ed7273afb..ce39f41d3 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.h b/programs/benchfn.h
index e555bbe6a..8c36831e1 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 77056203d..314a3430a 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 8c55b3c4f..769268532 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.c b/programs/datagen.c
index 4353b7ff9..835e2c027 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.h b/programs/datagen.h
index 5a2682d8f..97e0c2b06 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.c b/programs/dibio.c
index cb3829e3e..6673301ec 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.h b/programs/dibio.h
index 682723d6a..98ba9110b 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.c b/programs/fileio.c
index 65f2d531a..055fee09f 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.h b/programs/fileio.h
index 05e6d0681..007440bd2 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/platform.h b/programs/platform.h
index 68be70bb3..b4c6ee906 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.c b/programs/timefn.c
index 95460d0d9..f04b8cc26 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2019-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/timefn.h b/programs/timefn.h
index 5d2818e8a..911800433 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/util.c b/programs/util.c
index 5386d005c..460d9bf11 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/util.h b/programs/util.h
index 25fa3f53a..d2077c9ac 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/windres/verrsrc.h b/programs/windres/verrsrc.h
index 98156480f..6afb48ac4 100644
--- a/programs/windres/verrsrc.h
+++ b/programs/windres/verrsrc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 9b6f91533..1b0b6761d 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/DEPRECATED-test-zstd-speed.py b/tests/DEPRECATED-test-zstd-speed.py
index b3f807459..ab699cf5f 100755
--- a/tests/DEPRECATED-test-zstd-speed.py
+++ b/tests/DEPRECATED-test-zstd-speed.py
@@ -2,7 +2,7 @@
 # THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
 
 # ################################################################
-# Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/Makefile b/tests/Makefile
index 1c667cf86..f06050570 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py
index 77eea29de..3230821bf 100644
--- a/tests/automated_benchmarking.py
+++ b/tests/automated_benchmarking.py
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2020-2020, Facebook, Inc.
+# Copyright (c) 2021-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/bigdict.c b/tests/bigdict.c
index aeda56cb5..4d08ca198 100644
--- a/tests/bigdict.c
+++ b/tests/bigdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/checkTag.c b/tests/checkTag.c
index 90af24ab1..76664e057 100644
--- a/tests/checkTag.c
+++ b/tests/checkTag.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2018-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/datagencli.c b/tests/datagencli.c
index 713ca9963..2ca56316d 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 50935d31e..e4691919c 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 37f0e242f..5dc42ee63 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 36232a8cf..f3a561def 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2020, Facebook, Inc.
+# Copyright (c) 2016-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c
index 64d70f005..2f9878111 100644
--- a/tests/fuzz/block_decompress.c
+++ b/tests/fuzz/block_decompress.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c
index 097fc01b8..c88850fbb 100644
--- a/tests/fuzz/block_round_trip.c
+++ b/tests/fuzz/block_round_trip.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/decompress_dstSize_tooSmall.c b/tests/fuzz/decompress_dstSize_tooSmall.c
index e47b3d049..76806e304 100644
--- a/tests/fuzz/decompress_dstSize_tooSmall.c
+++ b/tests/fuzz/decompress_dstSize_tooSmall.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c
index 9944baa15..17035213c 100644
--- a/tests/fuzz/dictionary_decompress.c
+++ b/tests/fuzz/dictionary_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_loader.c b/tests/fuzz/dictionary_loader.c
index f1fdf4da9..53c252641 100644
--- a/tests/fuzz/dictionary_loader.c
+++ b/tests/fuzz/dictionary_loader.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c
index 7b7771e48..f08f0240f 100644
--- a/tests/fuzz/dictionary_round_trip.c
+++ b/tests/fuzz/dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_stream_round_trip.c b/tests/fuzz/dictionary_stream_round_trip.c
index 67e8c69ef..95e84704c 100644
--- a/tests/fuzz/dictionary_stream_round_trip.c
+++ b/tests/fuzz/dictionary_stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fse_read_ncount.c b/tests/fuzz/fse_read_ncount.c
index e20a93829..fc809ef89 100644
--- a/tests/fuzz/fse_read_ncount.c
+++ b/tests/fuzz/fse_read_ncount.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h
index 8ee964536..c9ba39ce8 100644
--- a/tests/fuzz/fuzz.h
+++ b/tests/fuzz/fuzz.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index ef94a53b4..b3aa2bdf6 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 # ################################################################
-# Copyright (c) 2016-2020, Facebook, Inc.
+# Copyright (c) 2016-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c
index f2d5a1b51..738409c44 100644
--- a/tests/fuzz/fuzz_data_producer.c
+++ b/tests/fuzz/fuzz_data_producer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h
index 25cc937fc..1cfcad9ea 100644
--- a/tests/fuzz/fuzz_data_producer.h
+++ b/tests/fuzz/fuzz_data_producer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_helpers.c b/tests/fuzz/fuzz_helpers.c
index b80dc7571..8d62ee9d8 100644
--- a/tests/fuzz/fuzz_helpers.c
+++ b/tests/fuzz/fuzz_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -29,4 +29,4 @@ int FUZZ_memcmp(void const* lhs, void const* rhs, size_t size)
         return 0;
     }
     return memcmp(lhs, rhs, size);
-}
\ No newline at end of file
+}
diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h
index cde2c4ea7..7085e14cc 100644
--- a/tests/fuzz/fuzz_helpers.h
+++ b/tests/fuzz/fuzz_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/raw_dictionary_round_trip.c b/tests/fuzz/raw_dictionary_round_trip.c
index 08e5fd9ed..c6e44db61 100644
--- a/tests/fuzz/raw_dictionary_round_trip.c
+++ b/tests/fuzz/raw_dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c
index 8180ca822..326cfdc2d 100644
--- a/tests/fuzz/regression_driver.c
+++ b/tests/fuzz/regression_driver.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c
index e838687a3..d9003d491 100644
--- a/tests/fuzz/sequence_compression_api.c
+++ b/tests/fuzz/sequence_compression_api.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c
index 620177fb0..f02223faa 100644
--- a/tests/fuzz/simple_compress.c
+++ b/tests/fuzz/simple_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c
index c3903ce8b..4ecc22b33 100644
--- a/tests/fuzz/simple_decompress.c
+++ b/tests/fuzz/simple_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 6e58fb1c3..f0e109601 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c
index 5d2bb2aaf..6117acaa9 100644
--- a/tests/fuzz/stream_decompress.c
+++ b/tests/fuzz/stream_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index 286d3871b..a09b04a9d 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c
index 876a74e9a..8b3608ab9 100644
--- a/tests/fuzz/zstd_frame_info.c
+++ b/tests/fuzz/zstd_frame_info.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 5680bd628..8ef778692 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h
index 6a4e340d3..46c6d0921 100644
--- a/tests/fuzz/zstd_helpers.h
+++ b/tests/fuzz/zstd_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 88c879c6c..c22871878 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
index 73f62f0eb..ac953bd81 100644
--- a/tests/gzip/Makefile
+++ b/tests/gzip/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-2020, Facebook, Inc.
+# Copyright (c) 2017-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c
index 23e93fd54..48ba6aebd 100644
--- a/tests/invalidDictionaries.c
+++ b/tests/invalidDictionaries.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/legacy.c b/tests/legacy.c
index 3d3ec4358..d6407a75f 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/longmatch.c b/tests/longmatch.c
index 93e78dd1f..075b152ae 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 439aebed8..896218247 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/poolTests.c b/tests/poolTests.c
index e1576ba85..c7fcfa860 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py
index 1068c4424..fbec8c2f9 100755
--- a/tests/rateLimiter.py
+++ b/tests/rateLimiter.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) 2018-2020, Facebook, Inc.
+# Copyright (c) 2018-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/Makefile b/tests/regression/Makefile
index 87c1c2b96..6874d81ba 100644
--- a/tests/regression/Makefile
+++ b/tests/regression/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2020, Facebook, Inc.
+# Copyright (c) 2015-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.c b/tests/regression/config.c
index ed6b69235..232f0fc0f 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.h b/tests/regression/config.h
index aa563b9e9..9c4562c8f 100644
--- a/tests/regression/config.h
+++ b/tests/regression/config.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.c b/tests/regression/data.c
index b75ac1192..c3ccf84f8 100644
--- a/tests/regression/data.c
+++ b/tests/regression/data.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.h b/tests/regression/data.h
index 90ed22f19..9f2fc8978 100644
--- a/tests/regression/data.h
+++ b/tests/regression/data.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/levels.h b/tests/regression/levels.h
index 5e7d40a7d..aedc1ce30 100644
--- a/tests/regression/levels.h
+++ b/tests/regression/levels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.c b/tests/regression/method.c
index 3c949a278..c4fe94a2a 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.h b/tests/regression/method.h
index 6884e5418..d89e64d6b 100644
--- a/tests/regression/method.h
+++ b/tests/regression/method.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.c b/tests/regression/result.c
index 2911722cd..5fc37c0aa 100644
--- a/tests/regression/result.c
+++ b/tests/regression/result.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.h b/tests/regression/result.h
index 0085c2adf..7b456165b 100644
--- a/tests/regression/result.h
+++ b/tests/regression/result.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/test.c b/tests/regression/test.c
index ff2cdba30..5e28d4cd9 100644
--- a/tests/regression/test.c
+++ b/tests/regression/test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Facebook, Inc.
+ * Copyright (c) 2016-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c
index c117d2c26..9e3b8614d 100644
--- a/tests/roundTripCrash.c
+++ b/tests/roundTripCrash.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.c b/tests/seqgen.c
index 29c0c4054..3461522e2 100644
--- a/tests/seqgen.c
+++ b/tests/seqgen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Facebook, Inc.
+ * Copyright (c) 2017-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.h b/tests/seqgen.h
index 808099ba9..e4948ea38 100644
--- a/tests/seqgen.h
+++ b/tests/seqgen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, Facebook, Inc.
+ * Copyright (c) 2017-2021, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/test-license.py b/tests/test-license.py
index 522884dba..a7e2ba4e3 100755
--- a/tests/test-license.py
+++ b/tests/test-license.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) 2016-2020, Facebook, Inc.
+# Copyright (c) 2016-2021, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -138,4 +138,4 @@ def main():
     return len(invalid_files)
 
 if __name__ == "__main__":
-    sys.exit(main())
\ No newline at end of file
+    sys.exit(main())
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index fa2171752..4b2dc6a3d 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -2,7 +2,7 @@
 """Test zstd interoperability between versions"""
 
 # ################################################################
-# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/zbufftest.c b/tests/zbufftest.c
index cd3706af4..557e62259 100644
--- a/tests/zbufftest.c
+++ b/tests/zbufftest.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index fa18ea4b4..688fec687 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/gzcompatibility.h b/zlibWrapper/gzcompatibility.h
index 394648abb..c1aa2b87c 100644
--- a/zlibWrapper/gzcompatibility.h
+++ b/zlibWrapper/gzcompatibility.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 0ae5012a1..a4867c20b 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index e791043e1..042ab9f84 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

From a98a6e2091a4d64df396216c2fbf695917a53e8c Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 4 Jan 2021 15:04:29 -0800
Subject: [PATCH 19/80] [test][regression] Add no source size with dictionary
 test

* Add a test that runs without a pledgedSrcSize and with a dictionary.
* Add github.tar data with uses the github dictionary while compressing
  github.tar, instead of each file individually.
---
 tests/regression/config.c    |   9 ++
 tests/regression/data.c      |  17 ++
 tests/regression/results.csv | 292 +++++++++++++++++++++++++++++++++++
 3 files changed, 318 insertions(+)

diff --git a/tests/regression/config.c b/tests/regression/config.c
index 232f0fc0f..ee0a71e4d 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -59,6 +59,14 @@ static config_t no_pledged_src_size = {
     .no_pledged_src_size = 1,
 };
 
+static config_t no_pledged_src_size_with_dict = {
+    .name = "no source size with dict",
+    .cli_args = "",
+    .param_values = PARAM_VALUES(level_0_param_values),
+    .no_pledged_src_size = 1,
+    .use_dictionary = 1,
+};
+
 static param_value_t const ldm_param_values[] = {
     {.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
 };
@@ -192,6 +200,7 @@ static config_t const* g_configs[] = {
 #undef FAST_LEVEL
 
     &no_pledged_src_size,
+    &no_pledged_src_size_with_dict,
     &ldm,
     &mt,
     &mt_ldm,
diff --git a/tests/regression/data.c b/tests/regression/data.c
index c3ccf84f8..b06c6914f 100644
--- a/tests/regression/data.c
+++ b/tests/regression/data.c
@@ -67,10 +67,27 @@ data_t github = {
         },
 };
 
+data_t github_tar = {
+    .name = "github.tar",
+    .type = data_type_file,
+    .data =
+        {
+            .url = REGRESSION_RELEASE("github.tar.zst"),
+            .xxhash64 = 0xa9b1b44b020df292LL,
+        },
+    .dict =
+        {
+            .url = REGRESSION_RELEASE("github.dict.zst"),
+            .xxhash64 = 0x1eddc6f737d3cb53LL,
+
+        },
+};
+
 static data_t* g_data[] = {
     &silesia,
     &silesia_tar,
     &github,
+    &github_tar,
     NULL,
 };
 
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 979b1d250..587b6afbe 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -16,6 +16,23 @@ silesia.tar,                        level 19,                           compress
 silesia.tar,                        uncompressed literals,              compress simple,                    4861425
 silesia.tar,                        uncompressed literals optimal,      compress simple,                    4281605
 silesia.tar,                        huffman literals,                   compress simple,                    6186042
+github.tar,                         level -5,                           compress simple,                    46856
+github.tar,                         level -3,                           compress simple,                    43754
+github.tar,                         level -1,                           compress simple,                    42490
+github.tar,                         level 0,                            compress simple,                    38441
+github.tar,                         level 1,                            compress simple,                    39265
+github.tar,                         level 3,                            compress simple,                    38441
+github.tar,                         level 4,                            compress simple,                    38467
+github.tar,                         level 5,                            compress simple,                    39788
+github.tar,                         level 6,                            compress simple,                    39603
+github.tar,                         level 7,                            compress simple,                    39206
+github.tar,                         level 9,                            compress simple,                    36717
+github.tar,                         level 13,                           compress simple,                    35621
+github.tar,                         level 16,                           compress simple,                    40255
+github.tar,                         level 19,                           compress simple,                    32837
+github.tar,                         uncompressed literals,              compress simple,                    38441
+github.tar,                         uncompressed literals optimal,      compress simple,                    32837
+github.tar,                         huffman literals,                   compress simple,                    42490
 silesia,                            level -5,                           compress cctx,                      6737607
 silesia,                            level -3,                           compress cctx,                      6444677
 silesia,                            level -1,                           compress cctx,                      6178460
@@ -170,6 +187,47 @@ github,                             uncompressed literals,              zstdcli,
 github,                             uncompressed literals optimal,      zstdcli,                            159227
 github,                             huffman literals,                   zstdcli,                            144465
 github,                             multithreaded with advanced params, zstdcli,                            167915
+github.tar,                         level -5,                           zstdcli,                            46751
+github.tar,                         level -5 with dict,                 zstdcli,                            43975
+github.tar,                         level -3,                           zstdcli,                            43541
+github.tar,                         level -3 with dict,                 zstdcli,                            40809
+github.tar,                         level -1,                           zstdcli,                            42469
+github.tar,                         level -1 with dict,                 zstdcli,                            41126
+github.tar,                         level 0,                            zstdcli,                            38445
+github.tar,                         level 0 with dict,                  zstdcli,                            37999
+github.tar,                         level 1,                            zstdcli,                            39346
+github.tar,                         level 1 with dict,                  zstdcli,                            38313
+github.tar,                         level 3,                            zstdcli,                            38445
+github.tar,                         level 3 with dict,                  zstdcli,                            37999
+github.tar,                         level 4,                            zstdcli,                            38471
+github.tar,                         level 4 with dict,                  zstdcli,                            37952
+github.tar,                         level 5,                            zstdcli,                            39792
+github.tar,                         level 5 with dict,                  zstdcli,                            39231
+github.tar,                         level 6,                            zstdcli,                            39607
+github.tar,                         level 6 with dict,                  zstdcli,                            38669
+github.tar,                         level 7,                            zstdcli,                            39210
+github.tar,                         level 7 with dict,                  zstdcli,                            37958
+github.tar,                         level 9,                            zstdcli,                            36721
+github.tar,                         level 9 with dict,                  zstdcli,                            36886
+github.tar,                         level 13,                           zstdcli,                            35625
+github.tar,                         level 13 with dict,                 zstdcli,                            38730
+github.tar,                         level 16,                           zstdcli,                            40259
+github.tar,                         level 16 with dict,                 zstdcli,                            33643
+github.tar,                         level 19,                           zstdcli,                            32841
+github.tar,                         level 19 with dict,                 zstdcli,                            32899
+github.tar,                         no source size,                     zstdcli,                            38442
+github.tar,                         no source size with dict,           zstdcli,                            213380
+github.tar,                         long distance mode,                 zstdcli,                            39680
+github.tar,                         multithreaded,                      zstdcli,                            38445
+github.tar,                         multithreaded long distance mode,   zstdcli,                            39680
+github.tar,                         small window log,                   zstdcli,                            199432
+github.tar,                         small hash log,                     zstdcli,                            129874
+github.tar,                         small chain log,                    zstdcli,                            41673
+github.tar,                         explicit params,                    zstdcli,                            41199
+github.tar,                         uncompressed literals,              zstdcli,                            41126
+github.tar,                         uncompressed literals optimal,      zstdcli,                            35392
+github.tar,                         huffman literals,                   zstdcli,                            38804
+github.tar,                         multithreaded with advanced params, zstdcli,                            41126
 silesia,                            level -5,                           advanced one pass,                  6737607
 silesia,                            level -3,                           advanced one pass,                  6444677
 silesia,                            level -1,                           advanced one pass,                  6178460
@@ -251,6 +309,7 @@ github,                             level 16 with dict,                 advanced
 github,                             level 19,                           advanced one pass,                  134064
 github,                             level 19 with dict,                 advanced one pass,                  37576
 github,                             no source size,                     advanced one pass,                  136335
+github,                             no source size with dict,           advanced one pass,                  41148
 github,                             long distance mode,                 advanced one pass,                  136335
 github,                             multithreaded,                      advanced one pass,                  136335
 github,                             multithreaded long distance mode,   advanced one pass,                  136335
@@ -262,6 +321,47 @@ github,                             uncompressed literals,              advanced
 github,                             uncompressed literals optimal,      advanced one pass,                  157227
 github,                             huffman literals,                   advanced one pass,                  142465
 github,                             multithreaded with advanced params, advanced one pass,                  165915
+github.tar,                         level -5,                           advanced one pass,                  46856
+github.tar,                         level -5 with dict,                 advanced one pass,                  43971
+github.tar,                         level -3,                           advanced one pass,                  43754
+github.tar,                         level -3 with dict,                 advanced one pass,                  40805
+github.tar,                         level -1,                           advanced one pass,                  42490
+github.tar,                         level -1 with dict,                 advanced one pass,                  41122
+github.tar,                         level 0,                            advanced one pass,                  38441
+github.tar,                         level 0 with dict,                  advanced one pass,                  37995
+github.tar,                         level 1,                            advanced one pass,                  39265
+github.tar,                         level 1 with dict,                  advanced one pass,                  38309
+github.tar,                         level 3,                            advanced one pass,                  38441
+github.tar,                         level 3 with dict,                  advanced one pass,                  37995
+github.tar,                         level 4,                            advanced one pass,                  38467
+github.tar,                         level 4 with dict,                  advanced one pass,                  37948
+github.tar,                         level 5,                            advanced one pass,                  39788
+github.tar,                         level 5 with dict,                  advanced one pass,                  39715
+github.tar,                         level 6,                            advanced one pass,                  39603
+github.tar,                         level 6 with dict,                  advanced one pass,                  38800
+github.tar,                         level 7,                            advanced one pass,                  39206
+github.tar,                         level 7 with dict,                  advanced one pass,                  38071
+github.tar,                         level 9,                            advanced one pass,                  36717
+github.tar,                         level 9 with dict,                  advanced one pass,                  36898
+github.tar,                         level 13,                           advanced one pass,                  35621
+github.tar,                         level 13 with dict,                 advanced one pass,                  38726
+github.tar,                         level 16,                           advanced one pass,                  40255
+github.tar,                         level 16 with dict,                 advanced one pass,                  33639
+github.tar,                         level 19,                           advanced one pass,                  32837
+github.tar,                         level 19 with dict,                 advanced one pass,                  32895
+github.tar,                         no source size,                     advanced one pass,                  38441
+github.tar,                         no source size with dict,           advanced one pass,                  37995
+github.tar,                         long distance mode,                 advanced one pass,                  39676
+github.tar,                         multithreaded,                      advanced one pass,                  38441
+github.tar,                         multithreaded long distance mode,   advanced one pass,                  39676
+github.tar,                         small window log,                   advanced one pass,                  198540
+github.tar,                         small hash log,                     advanced one pass,                  129870
+github.tar,                         small chain log,                    advanced one pass,                  41669
+github.tar,                         explicit params,                    advanced one pass,                  41199
+github.tar,                         uncompressed literals,              advanced one pass,                  41122
+github.tar,                         uncompressed literals optimal,      advanced one pass,                  35388
+github.tar,                         huffman literals,                   advanced one pass,                  38777
+github.tar,                         multithreaded with advanced params, advanced one pass,                  41122
 silesia,                            level -5,                           advanced one pass small out,        6737607
 silesia,                            level -3,                           advanced one pass small out,        6444677
 silesia,                            level -1,                           advanced one pass small out,        6178460
@@ -343,6 +443,7 @@ github,                             level 16 with dict,                 advanced
 github,                             level 19,                           advanced one pass small out,        134064
 github,                             level 19 with dict,                 advanced one pass small out,        37576
 github,                             no source size,                     advanced one pass small out,        136335
+github,                             no source size with dict,           advanced one pass small out,        41148
 github,                             long distance mode,                 advanced one pass small out,        136335
 github,                             multithreaded,                      advanced one pass small out,        136335
 github,                             multithreaded long distance mode,   advanced one pass small out,        136335
@@ -354,6 +455,47 @@ github,                             uncompressed literals,              advanced
 github,                             uncompressed literals optimal,      advanced one pass small out,        157227
 github,                             huffman literals,                   advanced one pass small out,        142465
 github,                             multithreaded with advanced params, advanced one pass small out,        165915
+github.tar,                         level -5,                           advanced one pass small out,        46856
+github.tar,                         level -5 with dict,                 advanced one pass small out,        43971
+github.tar,                         level -3,                           advanced one pass small out,        43754
+github.tar,                         level -3 with dict,                 advanced one pass small out,        40805
+github.tar,                         level -1,                           advanced one pass small out,        42490
+github.tar,                         level -1 with dict,                 advanced one pass small out,        41122
+github.tar,                         level 0,                            advanced one pass small out,        38441
+github.tar,                         level 0 with dict,                  advanced one pass small out,        37995
+github.tar,                         level 1,                            advanced one pass small out,        39265
+github.tar,                         level 1 with dict,                  advanced one pass small out,        38309
+github.tar,                         level 3,                            advanced one pass small out,        38441
+github.tar,                         level 3 with dict,                  advanced one pass small out,        37995
+github.tar,                         level 4,                            advanced one pass small out,        38467
+github.tar,                         level 4 with dict,                  advanced one pass small out,        37948
+github.tar,                         level 5,                            advanced one pass small out,        39788
+github.tar,                         level 5 with dict,                  advanced one pass small out,        39715
+github.tar,                         level 6,                            advanced one pass small out,        39603
+github.tar,                         level 6 with dict,                  advanced one pass small out,        38800
+github.tar,                         level 7,                            advanced one pass small out,        39206
+github.tar,                         level 7 with dict,                  advanced one pass small out,        38071
+github.tar,                         level 9,                            advanced one pass small out,        36717
+github.tar,                         level 9 with dict,                  advanced one pass small out,        36898
+github.tar,                         level 13,                           advanced one pass small out,        35621
+github.tar,                         level 13 with dict,                 advanced one pass small out,        38726
+github.tar,                         level 16,                           advanced one pass small out,        40255
+github.tar,                         level 16 with dict,                 advanced one pass small out,        33639
+github.tar,                         level 19,                           advanced one pass small out,        32837
+github.tar,                         level 19 with dict,                 advanced one pass small out,        32895
+github.tar,                         no source size,                     advanced one pass small out,        38441
+github.tar,                         no source size with dict,           advanced one pass small out,        37995
+github.tar,                         long distance mode,                 advanced one pass small out,        39676
+github.tar,                         multithreaded,                      advanced one pass small out,        38441
+github.tar,                         multithreaded long distance mode,   advanced one pass small out,        39676
+github.tar,                         small window log,                   advanced one pass small out,        198540
+github.tar,                         small hash log,                     advanced one pass small out,        129870
+github.tar,                         small chain log,                    advanced one pass small out,        41669
+github.tar,                         explicit params,                    advanced one pass small out,        41199
+github.tar,                         uncompressed literals,              advanced one pass small out,        41122
+github.tar,                         uncompressed literals optimal,      advanced one pass small out,        35388
+github.tar,                         huffman literals,                   advanced one pass small out,        38777
+github.tar,                         multithreaded with advanced params, advanced one pass small out,        41122
 silesia,                            level -5,                           advanced streaming,                 6882505
 silesia,                            level -3,                           advanced streaming,                 6568376
 silesia,                            level -1,                           advanced streaming,                 6183403
@@ -435,6 +577,7 @@ github,                             level 16 with dict,                 advanced
 github,                             level 19,                           advanced streaming,                 134064
 github,                             level 19 with dict,                 advanced streaming,                 37576
 github,                             no source size,                     advanced streaming,                 136335
+github,                             no source size with dict,           advanced streaming,                 41148
 github,                             long distance mode,                 advanced streaming,                 136335
 github,                             multithreaded,                      advanced streaming,                 136335
 github,                             multithreaded long distance mode,   advanced streaming,                 136335
@@ -446,6 +589,47 @@ github,                             uncompressed literals,              advanced
 github,                             uncompressed literals optimal,      advanced streaming,                 157227
 github,                             huffman literals,                   advanced streaming,                 142465
 github,                             multithreaded with advanced params, advanced streaming,                 165915
+github.tar,                         level -5,                           advanced streaming,                 46747
+github.tar,                         level -5 with dict,                 advanced streaming,                 43971
+github.tar,                         level -3,                           advanced streaming,                 43537
+github.tar,                         level -3 with dict,                 advanced streaming,                 40805
+github.tar,                         level -1,                           advanced streaming,                 42465
+github.tar,                         level -1 with dict,                 advanced streaming,                 41122
+github.tar,                         level 0,                            advanced streaming,                 38441
+github.tar,                         level 0 with dict,                  advanced streaming,                 37995
+github.tar,                         level 1,                            advanced streaming,                 39342
+github.tar,                         level 1 with dict,                  advanced streaming,                 38309
+github.tar,                         level 3,                            advanced streaming,                 38441
+github.tar,                         level 3 with dict,                  advanced streaming,                 37995
+github.tar,                         level 4,                            advanced streaming,                 38467
+github.tar,                         level 4 with dict,                  advanced streaming,                 37948
+github.tar,                         level 5,                            advanced streaming,                 39788
+github.tar,                         level 5 with dict,                  advanced streaming,                 39715
+github.tar,                         level 6,                            advanced streaming,                 39603
+github.tar,                         level 6 with dict,                  advanced streaming,                 38800
+github.tar,                         level 7,                            advanced streaming,                 39206
+github.tar,                         level 7 with dict,                  advanced streaming,                 38071
+github.tar,                         level 9,                            advanced streaming,                 36717
+github.tar,                         level 9 with dict,                  advanced streaming,                 36898
+github.tar,                         level 13,                           advanced streaming,                 35621
+github.tar,                         level 13 with dict,                 advanced streaming,                 38726
+github.tar,                         level 16,                           advanced streaming,                 40255
+github.tar,                         level 16 with dict,                 advanced streaming,                 33639
+github.tar,                         level 19,                           advanced streaming,                 32837
+github.tar,                         level 19 with dict,                 advanced streaming,                 32895
+github.tar,                         no source size,                     advanced streaming,                 38438
+github.tar,                         no source size with dict,           advanced streaming,                 214381
+github.tar,                         long distance mode,                 advanced streaming,                 39676
+github.tar,                         multithreaded,                      advanced streaming,                 38441
+github.tar,                         multithreaded long distance mode,   advanced streaming,                 39676
+github.tar,                         small window log,                   advanced streaming,                 199558
+github.tar,                         small hash log,                     advanced streaming,                 129870
+github.tar,                         small chain log,                    advanced streaming,                 41669
+github.tar,                         explicit params,                    advanced streaming,                 41199
+github.tar,                         uncompressed literals,              advanced streaming,                 41122
+github.tar,                         uncompressed literals optimal,      advanced streaming,                 35388
+github.tar,                         huffman literals,                   advanced streaming,                 38800
+github.tar,                         multithreaded with advanced params, advanced streaming,                 41122
 silesia,                            level -5,                           old streaming,                      6882505
 silesia,                            level -3,                           old streaming,                      6568376
 silesia,                            level -1,                           old streaming,                      6183403
@@ -511,9 +695,43 @@ github,                             level 16 with dict,                 old stre
 github,                             level 19,                           old streaming,                      134064
 github,                             level 19 with dict,                 old streaming,                      37576
 github,                             no source size,                     old streaming,                      140632
+github,                             no source size with dict,           old streaming,                      40678
 github,                             uncompressed literals,              old streaming,                      136335
 github,                             uncompressed literals optimal,      old streaming,                      134064
 github,                             huffman literals,                   old streaming,                      175568
+github.tar,                         level -5,                           old streaming,                      46747
+github.tar,                         level -5 with dict,                 old streaming,                      43971
+github.tar,                         level -3,                           old streaming,                      43537
+github.tar,                         level -3 with dict,                 old streaming,                      40805
+github.tar,                         level -1,                           old streaming,                      42465
+github.tar,                         level -1 with dict,                 old streaming,                      41122
+github.tar,                         level 0,                            old streaming,                      38441
+github.tar,                         level 0 with dict,                  old streaming,                      37995
+github.tar,                         level 1,                            old streaming,                      39342
+github.tar,                         level 1 with dict,                  old streaming,                      38309
+github.tar,                         level 3,                            old streaming,                      38441
+github.tar,                         level 3 with dict,                  old streaming,                      37995
+github.tar,                         level 4,                            old streaming,                      38467
+github.tar,                         level 4 with dict,                  old streaming,                      37948
+github.tar,                         level 5,                            old streaming,                      39788
+github.tar,                         level 5 with dict,                  old streaming,                      39715
+github.tar,                         level 6,                            old streaming,                      39603
+github.tar,                         level 6 with dict,                  old streaming,                      38800
+github.tar,                         level 7,                            old streaming,                      39206
+github.tar,                         level 7 with dict,                  old streaming,                      38071
+github.tar,                         level 9,                            old streaming,                      36717
+github.tar,                         level 9 with dict,                  old streaming,                      36898
+github.tar,                         level 13,                           old streaming,                      35621
+github.tar,                         level 13 with dict,                 old streaming,                      38726
+github.tar,                         level 16,                           old streaming,                      40255
+github.tar,                         level 16 with dict,                 old streaming,                      33639
+github.tar,                         level 19,                           old streaming,                      32837
+github.tar,                         level 19 with dict,                 old streaming,                      32895
+github.tar,                         no source size,                     old streaming,                      38438
+github.tar,                         no source size with dict,           old streaming,                      214384
+github.tar,                         uncompressed literals,              old streaming,                      38441
+github.tar,                         uncompressed literals optimal,      old streaming,                      32837
+github.tar,                         huffman literals,                   old streaming,                      42465
 silesia,                            level -5,                           old streaming advanced,             6882505
 silesia,                            level -3,                           old streaming advanced,             6568376
 silesia,                            level -1,                           old streaming advanced,             6183403
@@ -595,6 +813,7 @@ github,                             level 16 with dict,                 old stre
 github,                             level 19,                           old streaming advanced,             134064
 github,                             level 19 with dict,                 old streaming advanced,             37576
 github,                             no source size,                     old streaming advanced,             140632
+github,                             no source size with dict,           old streaming advanced,             40643
 github,                             long distance mode,                 old streaming advanced,             141104
 github,                             multithreaded,                      old streaming advanced,             141104
 github,                             multithreaded long distance mode,   old streaming advanced,             141104
@@ -606,6 +825,47 @@ github,                             uncompressed literals,              old stre
 github,                             uncompressed literals optimal,      old streaming advanced,             134064
 github,                             huffman literals,                   old streaming advanced,             181108
 github,                             multithreaded with advanced params, old streaming advanced,             141104
+github.tar,                         level -5,                           old streaming advanced,             46747
+github.tar,                         level -5 with dict,                 old streaming advanced,             44824
+github.tar,                         level -3,                           old streaming advanced,             43537
+github.tar,                         level -3 with dict,                 old streaming advanced,             41800
+github.tar,                         level -1,                           old streaming advanced,             42465
+github.tar,                         level -1 with dict,                 old streaming advanced,             41471
+github.tar,                         level 0,                            old streaming advanced,             38441
+github.tar,                         level 0 with dict,                  old streaming advanced,             38013
+github.tar,                         level 1,                            old streaming advanced,             39342
+github.tar,                         level 1 with dict,                  old streaming advanced,             38940
+github.tar,                         level 3,                            old streaming advanced,             38441
+github.tar,                         level 3 with dict,                  old streaming advanced,             38013
+github.tar,                         level 4,                            old streaming advanced,             38467
+github.tar,                         level 4 with dict,                  old streaming advanced,             38063
+github.tar,                         level 5,                            old streaming advanced,             39788
+github.tar,                         level 5 with dict,                  old streaming advanced,             39310
+github.tar,                         level 6,                            old streaming advanced,             39603
+github.tar,                         level 6 with dict,                  old streaming advanced,             39279
+github.tar,                         level 7,                            old streaming advanced,             39206
+github.tar,                         level 7 with dict,                  old streaming advanced,             38728
+github.tar,                         level 9,                            old streaming advanced,             36717
+github.tar,                         level 9 with dict,                  old streaming advanced,             36504
+github.tar,                         level 13,                           old streaming advanced,             35621
+github.tar,                         level 13 with dict,                 old streaming advanced,             36035
+github.tar,                         level 16,                           old streaming advanced,             40255
+github.tar,                         level 16 with dict,                 old streaming advanced,             38736
+github.tar,                         level 19,                           old streaming advanced,             32837
+github.tar,                         level 19 with dict,                 old streaming advanced,             32876
+github.tar,                         no source size,                     old streaming advanced,             38438
+github.tar,                         no source size with dict,           old streaming advanced,             214384
+github.tar,                         long distance mode,                 old streaming advanced,             38441
+github.tar,                         multithreaded,                      old streaming advanced,             38441
+github.tar,                         multithreaded long distance mode,   old streaming advanced,             38441
+github.tar,                         small window log,                   old streaming advanced,             199561
+github.tar,                         small hash log,                     old streaming advanced,             129870
+github.tar,                         small chain log,                    old streaming advanced,             41669
+github.tar,                         explicit params,                    old streaming advanced,             41199
+github.tar,                         uncompressed literals,              old streaming advanced,             38441
+github.tar,                         uncompressed literals optimal,      old streaming advanced,             32837
+github.tar,                         huffman literals,                   old streaming advanced,             42465
+github.tar,                         multithreaded with advanced params, old streaming advanced,             38441
 github,                             level -5 with dict,                 old streaming cdcit,                46718
 github,                             level -3 with dict,                 old streaming cdcit,                45395
 github,                             level -1 with dict,                 old streaming cdcit,                43170
@@ -620,6 +880,22 @@ github,                             level 9 with dict,                  old stre
 github,                             level 13 with dict,                 old streaming cdcit,                39743
 github,                             level 16 with dict,                 old streaming cdcit,                37577
 github,                             level 19 with dict,                 old streaming cdcit,                37576
+github,                             no source size with dict,           old streaming cdcit,                40678
+github.tar,                         level -5 with dict,                 old streaming cdcit,                45018
+github.tar,                         level -3 with dict,                 old streaming cdcit,                41886
+github.tar,                         level -1 with dict,                 old streaming cdcit,                41636
+github.tar,                         level 0 with dict,                  old streaming cdcit,                37956
+github.tar,                         level 1 with dict,                  old streaming cdcit,                38766
+github.tar,                         level 3 with dict,                  old streaming cdcit,                37956
+github.tar,                         level 4 with dict,                  old streaming cdcit,                37927
+github.tar,                         level 5 with dict,                  old streaming cdcit,                39209
+github.tar,                         level 6 with dict,                  old streaming cdcit,                38983
+github.tar,                         level 7 with dict,                  old streaming cdcit,                38584
+github.tar,                         level 9 with dict,                  old streaming cdcit,                36363
+github.tar,                         level 13 with dict,                 old streaming cdcit,                36372
+github.tar,                         level 16 with dict,                 old streaming cdcit,                39353
+github.tar,                         level 19 with dict,                 old streaming cdcit,                32676
+github.tar,                         no source size with dict,           old streaming cdcit,                214384
 github,                             level -5 with dict,                 old streaming advanced cdict,       49562
 github,                             level -3 with dict,                 old streaming advanced cdict,       44956
 github,                             level -1 with dict,                 old streaming advanced cdict,       42383
@@ -634,3 +910,19 @@ github,                             level 9 with dict,                  old stre
 github,                             level 13 with dict,                 old streaming advanced cdict,       39731
 github,                             level 16 with dict,                 old streaming advanced cdict,       40789
 github,                             level 19 with dict,                 old streaming advanced cdict,       37576
+github,                             no source size with dict,           old streaming advanced cdict,       40643
+github.tar,                         level -5 with dict,                 old streaming advanced cdict,       44307
+github.tar,                         level -3 with dict,                 old streaming advanced cdict,       41359
+github.tar,                         level -1 with dict,                 old streaming advanced cdict,       41322
+github.tar,                         level 0 with dict,                  old streaming advanced cdict,       38013
+github.tar,                         level 1 with dict,                  old streaming advanced cdict,       39002
+github.tar,                         level 3 with dict,                  old streaming advanced cdict,       38013
+github.tar,                         level 4 with dict,                  old streaming advanced cdict,       38063
+github.tar,                         level 5 with dict,                  old streaming advanced cdict,       39310
+github.tar,                         level 6 with dict,                  old streaming advanced cdict,       39279
+github.tar,                         level 7 with dict,                  old streaming advanced cdict,       38728
+github.tar,                         level 9 with dict,                  old streaming advanced cdict,       36504
+github.tar,                         level 13 with dict,                 old streaming advanced cdict,       36035
+github.tar,                         level 16 with dict,                 old streaming advanced cdict,       38736
+github.tar,                         level 19 with dict,                 old streaming advanced cdict,       32876
+github.tar,                         no source size with dict,           old streaming advanced cdict,       214384

From 9d31c704d569cd5bfed24d1f13c767c5e25d183d Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 4 Jan 2021 13:43:47 -0800
Subject: [PATCH 20/80] Don't shrink window log when streaming with a
 dictionary

Fixes #2442.

1. When creating a dictionary keep the same behavior as before.
   Assume the source size is 513 bytes when adjusting parameters.
2. When calling ZSTD_getCParams() or ZSTD_adjustCParams() keep
   the same behavior as before.
3. When attaching a dictionary keep the same behavior of ignoring
   the dictionary size. When streaming this will select the
   largest parameters and not adjust them down. But, the CDict
   will use the correctly sized parameters, which seems like the
   right tradeoff.
4. When not attaching a dictionary (either forced not to, or
   using a prefix dictionary) we select parameters based on the
   dictionary size + source size, and assume the source size is
   small, which is the same behavior as before. But, now we don't
   adjust the window log (and hash and chain log) down when the
   source size is unknown.

When the source size is unknown all cdicts should attach, except
when the user disables attaching, or `forceWindow` is used. This
means that when streaming with a CDict we end up in the good case
where we get small CDict parameters, and large source parameters.

TODO: Add a streaming + dictionary regression test case.
---
 lib/compress/zstd_compress.c | 28 ++++++++++++++++++++++------
 tests/regression/results.csv | 20 ++++++++++----------
 2 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3cebbe173..c0628d832 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1188,15 +1188,30 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
     assert(ZSTD_checkCParams(cPar)==0);
 
-    if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN)
-        srcSize = minSrcSize;
-
     switch (mode) {
     case ZSTD_cpm_noAttachDict:
+        /* If we don't know the source size, don't make any
+         * assumptions about it. We will already have selected
+         * smaller parameters if a dictionary is in use.
+         */
+        break;
     case ZSTD_cpm_unknown:
+        /* Keep the legacy behavior of assuming small source
+         * sizes when the cparam mode is unkown.
+         */
+        /* fall-through */
     case ZSTD_cpm_createCDict:
+        /* Assume a small source size when creating a dictionary
+         * with an unkown source size.
+         */
+        if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN)
+            srcSize = minSrcSize;
         break;
     case ZSTD_cpm_attachDict:
+        /* Dictionary has its own dedicated parameters which have
+         * already been selected. We are selecting parameters
+         * for only the source.
+         */
         dictSize = 0;
         break;
     default:
@@ -1213,7 +1228,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
                             ZSTD_highbit32(tSize-1) + 1;
         if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
     }
-    {   U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize);
+    if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+        U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize);
         U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
         if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1;
         if (cycleLog > dictAndWindowLog)
@@ -2955,9 +2971,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
 }
 
 /* ZSTD_writeSkippableFrame_advanced() :
- * Writes out a skippable frame with the specified magic number variant (16 are supported), 
+ * Writes out a skippable frame with the specified magic number variant (16 are supported),
  * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data.
- * 
+ *
  * Returns the total number of bytes written, or a ZSTD error code.
  */
 size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 587b6afbe..0f818149b 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -216,7 +216,7 @@ github.tar,                         level 16 with dict,                 zstdcli,
 github.tar,                         level 19,                           zstdcli,                            32841
 github.tar,                         level 19 with dict,                 zstdcli,                            32899
 github.tar,                         no source size,                     zstdcli,                            38442
-github.tar,                         no source size with dict,           zstdcli,                            213380
+github.tar,                         no source size with dict,           zstdcli,                            38004
 github.tar,                         long distance mode,                 zstdcli,                            39680
 github.tar,                         multithreaded,                      zstdcli,                            38445
 github.tar,                         multithreaded long distance mode,   zstdcli,                            39680
@@ -618,7 +618,7 @@ github.tar,                         level 16 with dict,                 advanced
 github.tar,                         level 19,                           advanced streaming,                 32837
 github.tar,                         level 19 with dict,                 advanced streaming,                 32895
 github.tar,                         no source size,                     advanced streaming,                 38438
-github.tar,                         no source size with dict,           advanced streaming,                 214381
+github.tar,                         no source size with dict,           advanced streaming,                 38000
 github.tar,                         long distance mode,                 advanced streaming,                 39676
 github.tar,                         multithreaded,                      advanced streaming,                 38441
 github.tar,                         multithreaded long distance mode,   advanced streaming,                 39676
@@ -695,7 +695,7 @@ github,                             level 16 with dict,                 old stre
 github,                             level 19,                           old streaming,                      134064
 github,                             level 19 with dict,                 old streaming,                      37576
 github,                             no source size,                     old streaming,                      140632
-github,                             no source size with dict,           old streaming,                      40678
+github,                             no source size with dict,           old streaming,                      40654
 github,                             uncompressed literals,              old streaming,                      136335
 github,                             uncompressed literals optimal,      old streaming,                      134064
 github,                             huffman literals,                   old streaming,                      175568
@@ -728,7 +728,7 @@ github.tar,                         level 16 with dict,                 old stre
 github.tar,                         level 19,                           old streaming,                      32837
 github.tar,                         level 19 with dict,                 old streaming,                      32895
 github.tar,                         no source size,                     old streaming,                      38438
-github.tar,                         no source size with dict,           old streaming,                      214384
+github.tar,                         no source size with dict,           old streaming,                      38000
 github.tar,                         uncompressed literals,              old streaming,                      38441
 github.tar,                         uncompressed literals optimal,      old streaming,                      32837
 github.tar,                         huffman literals,                   old streaming,                      42465
@@ -813,7 +813,7 @@ github,                             level 16 with dict,                 old stre
 github,                             level 19,                           old streaming advanced,             134064
 github,                             level 19 with dict,                 old streaming advanced,             37576
 github,                             no source size,                     old streaming advanced,             140632
-github,                             no source size with dict,           old streaming advanced,             40643
+github,                             no source size with dict,           old streaming advanced,             40608
 github,                             long distance mode,                 old streaming advanced,             141104
 github,                             multithreaded,                      old streaming advanced,             141104
 github,                             multithreaded long distance mode,   old streaming advanced,             141104
@@ -854,7 +854,7 @@ github.tar,                         level 16 with dict,                 old stre
 github.tar,                         level 19,                           old streaming advanced,             32837
 github.tar,                         level 19 with dict,                 old streaming advanced,             32876
 github.tar,                         no source size,                     old streaming advanced,             38438
-github.tar,                         no source size with dict,           old streaming advanced,             214384
+github.tar,                         no source size with dict,           old streaming advanced,             38015
 github.tar,                         long distance mode,                 old streaming advanced,             38441
 github.tar,                         multithreaded,                      old streaming advanced,             38441
 github.tar,                         multithreaded long distance mode,   old streaming advanced,             38441
@@ -880,7 +880,7 @@ github,                             level 9 with dict,                  old stre
 github,                             level 13 with dict,                 old streaming cdcit,                39743
 github,                             level 16 with dict,                 old streaming cdcit,                37577
 github,                             level 19 with dict,                 old streaming cdcit,                37576
-github,                             no source size with dict,           old streaming cdcit,                40678
+github,                             no source size with dict,           old streaming cdcit,                40654
 github.tar,                         level -5 with dict,                 old streaming cdcit,                45018
 github.tar,                         level -3 with dict,                 old streaming cdcit,                41886
 github.tar,                         level -1 with dict,                 old streaming cdcit,                41636
@@ -895,7 +895,7 @@ github.tar,                         level 9 with dict,                  old stre
 github.tar,                         level 13 with dict,                 old streaming cdcit,                36372
 github.tar,                         level 16 with dict,                 old streaming cdcit,                39353
 github.tar,                         level 19 with dict,                 old streaming cdcit,                32676
-github.tar,                         no source size with dict,           old streaming cdcit,                214384
+github.tar,                         no source size with dict,           old streaming cdcit,                38000
 github,                             level -5 with dict,                 old streaming advanced cdict,       49562
 github,                             level -3 with dict,                 old streaming advanced cdict,       44956
 github,                             level -1 with dict,                 old streaming advanced cdict,       42383
@@ -910,7 +910,7 @@ github,                             level 9 with dict,                  old stre
 github,                             level 13 with dict,                 old streaming advanced cdict,       39731
 github,                             level 16 with dict,                 old streaming advanced cdict,       40789
 github,                             level 19 with dict,                 old streaming advanced cdict,       37576
-github,                             no source size with dict,           old streaming advanced cdict,       40643
+github,                             no source size with dict,           old streaming advanced cdict,       40608
 github.tar,                         level -5 with dict,                 old streaming advanced cdict,       44307
 github.tar,                         level -3 with dict,                 old streaming advanced cdict,       41359
 github.tar,                         level -1 with dict,                 old streaming advanced cdict,       41322
@@ -925,4 +925,4 @@ github.tar,                         level 9 with dict,                  old stre
 github.tar,                         level 13 with dict,                 old streaming advanced cdict,       36035
 github.tar,                         level 16 with dict,                 old streaming advanced cdict,       38736
 github.tar,                         level 19 with dict,                 old streaming advanced cdict,       32876
-github.tar,                         no source size with dict,           old streaming advanced cdict,       214384
+github.tar,                         no source size with dict,           old streaming advanced cdict,       38015

From 58476bcf7fb04e1827821a7723e1033bc4cd51f0 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 4 Jan 2021 15:51:23 -0800
Subject: [PATCH 21/80] Don't shrink window log in ZSTD_getCParams()

Treat ZSTD_getCParams() and ZSTD_adjustCParams() in the same way
we treat streaming compression. Choose parameters based on the
dictionary size + source size, and assume the source size is small
if unkown. But, don't shrink the window log down in
ZSTD_adjustCParams_internal().
---
 lib/compress/zstd_compress.c |  6 +-----
 tests/fuzzer.c               | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c0628d832..9e704a4b2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1189,17 +1189,13 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     assert(ZSTD_checkCParams(cPar)==0);
 
     switch (mode) {
+    case ZSTD_cpm_unknown:
     case ZSTD_cpm_noAttachDict:
         /* If we don't know the source size, don't make any
          * assumptions about it. We will already have selected
          * smaller parameters if a dictionary is in use.
          */
         break;
-    case ZSTD_cpm_unknown:
-        /* Keep the legacy behavior of assuming small source
-         * sizes when the cparam mode is unkown.
-         */
-        /* fall-through */
     case ZSTD_cpm_createCDict:
         /* Assume a small source size when creating a dictionary
          * with an unkown source size.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c22871878..7e3b4628e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -3051,6 +3051,32 @@ static int basicUnitTests(U32 const seed, double compressibility)
         free(dict);
     }
     DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
+    {
+        ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
+        ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
+        ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
+        ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
+        ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
+
+        assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
+        assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
+        assert(!memcmp(&large, &largeDict, sizeof(large)));
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
+    {
+        ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
+        ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
+        ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
+
+        assert(smallSrcAndDict.windowLog == 10);
+        assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 #endif
 
 _end:

From 0d793a675a34a820258ab4c7c5a84e57bbfd7d62 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Wed, 6 Jan 2021 01:35:52 -0800
Subject: [PATCH 22/80] removed internal dependency from CLI

ZSTD_cycleLog() is a very short function,
creating a rather large dependency onto libzstd's internal just for it is overkill.
Prefer duplicating this 2-lines function.

This PR makes the zstd CLI one step closer to being linkable to the dynamic library (see #2450)
More steps are still needed to reach this goal.
---
 programs/fileio.c | 43 ++++++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 055fee09f..51956f681 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -45,7 +45,6 @@
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
 #include "../lib/zstd.h"
 #include "../lib/common/zstd_errors.h"  /* ZSTD_error_frameParameter_windowTooLarge */
-#include "../lib/compress/zstd_compress_internal.h"
 
 #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
 #  include <zlib.h>
@@ -77,6 +76,11 @@
 /*-*************************************
 *  Macros
 ***************************************/
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+#undef MAX
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
 
 struct FIO_display_prefs_s {
     int displayLevel;   /* 0 : no display;  1: errors;  2: + result + interaction + warnings;  3: + progression;  4: + information */
@@ -840,7 +844,7 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
 /* FIO_removeMultiFilesWarning() :
  * Returns 1 if the console should abort, 0 if console should proceed.
  * This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts.
- * 
+ *
  * If -f is specified, or there is just 1 file, zstd will always proceed as usual.
  * If --rm is specified, there will be a prompt asking for user confirmation.
  *         If -f is specified with --rm, zstd will proceed as usual
@@ -897,6 +901,15 @@ typedef struct {
     ZSTD_CStream* cctx;
 } cRess_t;
 
+/** ZSTD_cycleLog() :
+ *  condition for correct operation : hashLog > 1 */
+static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
+{
+    U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
+    assert(hashLog > 1);
+    return hashLog - btScale;
+}
+
 static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
                                     ZSTD_compressionParameters* comprParams,
                                     unsigned long long const dictSize,
@@ -983,7 +996,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
     CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) );
     CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
     CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
-    CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
+    CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, (int)comprParams.strategy) );
     CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) );
     CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableDedicatedDictSearch, 1) );
     /* multi-threading */
@@ -1350,7 +1363,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
             /* display notification; and adapt compression level */
             if (READY_FOR_UPDATE()) {
                 ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
-                double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
+                double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
 
                 /* display progress notifications */
                 if (g_display_prefs.displayLevel >= 3) {
@@ -1545,7 +1558,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
     fCtx->totalBytesOutput += (size_t)compressedfilesize;
     DISPLAYLEVEL(2, "\r%79s\r", "");
     if (g_display_prefs.displayLevel >= 2 &&
-        !fCtx->hasStdoutOutput && 
+        !fCtx->hasStdoutOutput &&
         (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) {
         if (readsize == 0) {
             DISPLAYLEVEL(2,"%-20s :  (%6llu => %6llu bytes, %s) \n",
@@ -1555,7 +1568,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
         } else {
             DISPLAYLEVEL(2,"%-20s :%6.2f%%   (%6llu => %6llu bytes, %s) \n",
                 srcFileName,
-                (double)compressedfilesize / readsize * 100,
+                (double)compressedfilesize / (double)readsize * 100,
                 (unsigned long long)readsize, (unsigned long long) compressedfilesize,
                 dstFileName);
         }
@@ -1795,7 +1808,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
     int status;
     int error = 0;
     cRess_t ress = FIO_createCResources(prefs, dictFileName,
-        FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFilesTotal),
+        FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal),
         compressionLevel, comprParams);
 
     /* init */
@@ -1821,7 +1834,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
         }
     } else {
         if (outMirroredRootDirName)
-            UTIL_mirrorSourceFilesDirectories(inFileNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName);
+            UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);
 
         for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) {
             const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx];
@@ -1845,7 +1858,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
         }
 
         if (outDirName)
-            FIO_checkFilenameCollisions(inFileNamesTable , fCtx->nbFilesTotal);
+            FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
     }
 
     if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
@@ -1892,7 +1905,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
         EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
     CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) );
     CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));
-    
+
     ress.srcBufferSize = ZSTD_DStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
     ress.dstBufferSize = ZSTD_DStreamOutSize();
@@ -2747,7 +2760,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
                         strerror(errno));
     } else {
         if (outMirroredRootDirName)
-            UTIL_mirrorSourceFilesDirectories(srcNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName);
+            UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);
 
         for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) {   /* create dstFileName */
             const char* const srcFileName = srcNamesTable[fCtx->currFileIdx];
@@ -2769,9 +2782,9 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
             error |= status;
         }
         if (outDirName)
-            FIO_checkFilenameCollisions(srcNamesTable , fCtx->nbFilesTotal);
+            FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
     }
-    
+
     if (fCtx->nbFilesProcessed >= 1  && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0)
         DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
 
@@ -2938,7 +2951,7 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel)
     double const windowSizeUnit = (double)info->windowSize / unit;
     double const compressedSizeUnit = (double)info->compressedSize / unit;
     double const decompressedSizeUnit = (double)info->decompressedSize / unit;
-    double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
+    double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/(double)info->compressedSize;
     const char* const checkString = (info->usesCheck ? "XXH64" : "None");
     if (displayLevel <= 2) {
         if (!info->decompUnavailable) {
@@ -3059,7 +3072,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
             const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
             double const compressedSizeUnit = (double)total.compressedSize / unit;
             double const decompressedSizeUnit = (double)total.decompressedSize / unit;
-            double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize;
+            double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/(double)total.compressedSize;
             const char* const checkString = (total.usesCheck ? "XXH64" : "");
             DISPLAYOUT("----------------------------------------------------------------- \n");
             if (total.decompUnavailable) {

From 9866148e223330f25ff2e5b8a8945d9d46434936 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 6 Jan 2021 15:37:46 -0800
Subject: [PATCH 23/80] removed redundant tests

clang v3.8 tests are either flacky or redundant,
prefer using clang-latest.
---
 .github/workflows/generic-dev.yml     | 12 ------------
 .github/workflows/generic-release.yml |  9 +++------
 .travis.yml                           |  5 -----
 lib/zstd.h                            | 17 ++++++++++-------
 4 files changed, 13 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml
index fab0f2da5..72e739f4b 100644
--- a/.github/workflows/generic-dev.yml
+++ b/.github/workflows/generic-dev.yml
@@ -128,18 +128,6 @@ jobs:
         make gcc6install libc6install
         CC=gcc-6 CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
-  clang-38-msan-fuzz:
-    runs-on: ubuntu-16.04 # fails on 18.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: clang-3.8 + MSan + Fuzz Test
-      run: |
-        # make clang38install (doesn't work)
-        sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
-        sudo apt-get update
-        sudo apt-get install clang-3.8
-        CC=clang-3.8 make clean msan-fuzztest
-
   asan-ubsan-msan-regression:
     runs-on: ubuntu-latest
     steps:
diff --git a/.github/workflows/generic-release.yml b/.github/workflows/generic-release.yml
index 632a6b7d2..1ee26f31e 100644
--- a/.github/workflows/generic-release.yml
+++ b/.github/workflows/generic-release.yml
@@ -34,16 +34,13 @@ jobs:
         make -C tests test-zbuff
 
   tsan:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: thread sanitizer
       run: |
-        sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
-        sudo apt-get update
-        sudo apt-get install clang-3.8
-        CC=clang-3.8 make tsan-test-zstream
-        CC=clang-3.8 make tsan-fuzztest
+        CC=clang make tsan-test-zstream
+        CC=clang make tsan-fuzztest
 
   zlib-wrapper:
     runs-on: ubuntu-16.04
diff --git a/.travis.yml b/.travis.yml
index 4757c6b02..7fcae88cd 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,11 +32,6 @@ matrix:
       script:
         - make check
 
-    - name: make test (complete)
-      script:
-        # DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
-        - DEVNULLRIGHTS=test make test
-
     - name: Minimal Decompressor Macros    # ~5mn
       script:
         - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
diff --git a/lib/zstd.h b/lib/zstd.h
index 4ff03e1e9..ff941fbbf 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1374,15 +1374,15 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size
 
 /*! ZSTD_writeSkippableFrame() :
  * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
- * 
+ *
  * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
  * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
  * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
  * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
- * 
+ *
  * Returns an error if destination buffer is not large enough, if the source size is not representable
  * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
- * 
+ *
  * @return : number of bytes written or a ZSTD error.
  */
 ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
@@ -1530,6 +1530,7 @@ ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
 ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);
 ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
 
+
 /*
  * This API is temporary and is expected to change or disappear in the future!
  */
@@ -1540,10 +1541,12 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
     const ZSTD_CCtx_params* cctxParams,
     ZSTD_customMem customMem);
 
-ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
-                                                  ZSTD_dictLoadMethod_e dictLoadMethod,
-                                                  ZSTD_dictContentType_e dictContentType,
-                                                  ZSTD_customMem customMem);
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(
+    const void* dict, size_t dictSize,
+    ZSTD_dictLoadMethod_e dictLoadMethod,
+    ZSTD_dictContentType_e dictContentType,
+    ZSTD_customMem customMem);
+
 
 /***************************************
 *  Advanced compression functions

From 890d85bdb4293118b6b9ba4efe82e790a1dd904b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 6 Jan 2021 16:19:42 -0800
Subject: [PATCH 24/80] removed CLI dependency to legacy unsafe function

makint the CLI ons step closer to being linkable to the dynamic library
---
 programs/.gitignore |  1 +
 programs/dibio.c    | 18 +++---------------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/programs/.gitignore b/programs/.gitignore
index 662f70839..2d4edbe45 100644
--- a/programs/.gitignore
+++ b/programs/.gitignore
@@ -8,6 +8,7 @@ zstd-frugal
 zstd-small
 zstd-nolegacy
 zstd-dictBuilder
+zstd-dll
 
 # Object files
 *.o
diff --git a/programs/dibio.c b/programs/dibio.c
index 6673301ec..c6d267cea 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -255,18 +255,6 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si
 }
 
 
-/*! ZDICT_trainFromBuffer_unsafe_legacy() :
-    Strictly Internal use only !!
-    Same as ZDICT_trainFromBuffer_legacy(), but does not control `samplesBuffer`.
-    `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
-    @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
-              or an error code.
-*/
-size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCapacity,
-                                           const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
-                                           ZDICT_legacy_params_t parameters);
-
-
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
                        ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
@@ -319,9 +307,9 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     {   size_t dictSize;
         if (params) {
             DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH);   /* guard band, for end of buffer condition */
-            dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize,
-                                                           srcBuffer, sampleSizes, fs.nbSamples,
-                                                           *params);
+            dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize,
+                                                    srcBuffer, sampleSizes, fs.nbSamples,
+                                                    *params);
         } else if (coverParams) {
             if (optimize) {
               dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,

From cefdc023f7727a7a0f9b5ad28182c534dd162b79 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 6 Jan 2021 17:54:07 -0800
Subject: [PATCH 25/80] The CLI can be linked to libzstd dynamic library

invoking target zstd-dll
---
 lib/dictBuilder/zdict.c |  9 ++-------
 lib/dictBuilder/zdict.h |  5 +++--
 programs/Makefile       | 29 +++++++++++++----------------
 3 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 2b7659aaa..4df5a94a2 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -971,16 +971,11 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
     return MIN(dictBufferCapacity, hSize+dictContentSize);
 }
 
-/* Hidden declaration for dbio.c */
-size_t ZDICT_trainFromBuffer_unsafe_legacy(
-                            void* dictBuffer, size_t maxDictSize,
-                            const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
-                            ZDICT_legacy_params_t params);
 /*! ZDICT_trainFromBuffer_unsafe_legacy() :
-*   Warning : `samplesBuffer` must be followed by noisy guard band.
+*   Warning : `samplesBuffer` must be followed by noisy guard band !!!
 *   @return : size of dictionary, or an error code which can be tested with ZDICT_isError()
 */
-size_t ZDICT_trainFromBuffer_unsafe_legacy(
+static size_t ZDICT_trainFromBuffer_unsafe_legacy(
                             void* dictBuffer, size_t maxDictSize,
                             const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                             ZDICT_legacy_params_t params)
diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h
index d1ac3d41f..190ffa52a 100644
--- a/lib/dictBuilder/zdict.h
+++ b/lib/dictBuilder/zdict.h
@@ -264,10 +264,11 @@ typedef struct {
  *  Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
  */
 ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
-    void *dictBuffer, size_t dictBufferCapacity,
-    const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
+    void* dictBuffer, size_t dictBufferCapacity,
+    const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
     ZDICT_legacy_params_t parameters);
 
+
 /* Deprecation warnings */
 /* It is generally possible to disable deprecation warnings from compiler,
    for example with -Wno-deprecated-declarations for gcc
diff --git a/programs/Makefile b/programs/Makefile
index 65fcb0d98..48a60ad0f 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -90,13 +90,13 @@ endif
 
 # Sort files in alphabetical order for reproducible builds
 ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC))
-ZSTDLIB_LOCAL_SRC := $(notdir $(ZSTDLIB_FULL_SRC))
+ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC))
 ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_SRC:.c=.o)
 
 ZSTD_CLI_SRC := $(wildcard *.c)
 ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
 
-ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
+ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
 ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o)
 
 UNAME := $(shell uname)
@@ -111,7 +111,7 @@ HASH ?= md5sum
 HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
 
 ifndef BUILD_DIR
-HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ")
+HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(LDLIBS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ")
 ifeq ($(HAVE_HASH),0)
   $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
   BUILD_DIR := obj/generic_noconf
@@ -199,7 +199,8 @@ SET_CACHE_DIRECTORY = \
     CPPFLAGS="$(CPPFLAGS)" \
     CFLAGS="$(CFLAGS)" \
     LDFLAGS="$(LDFLAGS)" \
-    LDLIBS="$(LDLIBS)"
+    LDLIBS="$(LDLIBS)" \
+    ZSTD_ALL_SRC="$(ZSTD_ALL_SRC)"
 
 
 .PHONY: all
@@ -289,17 +290,11 @@ zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
 zstd-noxz : zstd
 
 ## zstd-dll: zstd executable linked to dynamic library libzstd (must already exist)
-# note : the following target doesn't link
-#        because zstd uses non-public symbols from libzstd
-#        such as XXH64 (for benchmark),
-#        ZDICT_trainFromBuffer_unsafe_legacy (for dictionary builder)
-#        and ZSTD_cycleLog (likely for --patch-from).
-#        It's unclear at this stage if this is a scenario that must be supported
 .PHONY: zstd-dll
-zstd-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
-zstd-dll : ZSTDLIB_FULL_SRC =
-zstd-dll : $(ZSTD_CLI_OBJ)
-	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
+zstd-dll : LDFLAGS+= -L$(ZSTDDIR)
+zstd-dll : LDLIBS += -lzstd
+zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c
+zstd-dll : zstd
 
 
 ## zstd-pgo: zstd executable optimized with PGO.
@@ -350,9 +345,11 @@ endif
 .PHONY: clean
 clean:
 	$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
-        zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
+        zstd$(EXT) zstd32$(EXT) zstd-dll$(EXT) \
+        zstd-compress$(EXT) zstd-decompress$(EXT) \
         zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
-        zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT)
+        zstd-dictBuilder$(EXT) \
+        *.gcda default*.profraw default.profdata have_zlib$(EXT)
 	$(RM) -r obj/*
 	@echo Cleaning completed
 

From f933668d3f9dc527528ce13b58bfa78509dbcc2e Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Dec 2020 15:46:22 -0500
Subject: [PATCH 26/80] Implement hashset for dictIDs

---
 lib/decompress/zstd_decompress.c | 94 ++++++++++++++++++++++++++++++++
 programs/have_lzma.c             |  2 +
 2 files changed, 96 insertions(+)
 create mode 100644 programs/have_lzma.c

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 172d46c04..c3d4552f8 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1307,6 +1307,100 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
 size_t ZSTD_DStreamInSize(void)  { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
 size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
 
+
+/***************************
+ * Multiple DDicts HashSet *
+ ***************************/
+
+#define DDICT_HASHSET_MAX_LOAD_FACTOR 0.75
+#define DDICT_HASHSET_TABLE_BASE_SIZE 64
+#define DDICT_HASHSET_RESIZE_FACTOR 2
+
+/* Hashset using linear probing */
+typedef struct {
+    ZSTD_DDict** ddictPtrTable;
+    size_t ddictPtrTableSize;
+    size_t ddictPtrCount;
+} ZSTD_DDictHashSet;
+
+/* Hash function to determine starting position of dict insertion */
+static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, size_t seed) {
+    return seed % hashSet->ddictPtrTableSize;
+}
+
+/* Adds ddict to a hashset without any chance of resizing it 
+ * Returns 0 on success or a zstd error code
+ */
+static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, ZSTD_DDict* ddict, int dictID) {
+    RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
+    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    while (hashSet->ddictPtrTable[idx] != NULL) {
+        /* Replace existing ddict if inserting ddict with same dictID */
+        if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) break;
+        idx++;
+    }
+    hashSet->ddictPtrTable[idx] = ddict;
+    hashSet->ddictPtrCount++;
+    return 0;
+}
+
+/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and rehashes all values */
+static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+    size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
+    ZSTD_DDict** newTable = (ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
+    ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
+    size_t oldTableSize = hashSet->ddictPtrTableSize;
+    size_t i = 0;
+    
+    RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
+    hashSet->ddictPtrTable = newTable;
+    hashSet->ddictPtrTableSize = newTableSize;
+    hashSet->ddictPtrCount = 0;
+    for (i = 0; i < oldTableSize; ++i) {
+        if (oldTable[i] != NULL) {
+            FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i], ZSTD_getDictID_fromDDict(oldTable[i])), "");
+        }
+    }
+    ZSTD_customFree(oldTable, customMem);
+    return 0;
+}
+
+/* Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
+ * Returns 0 on success, or a ZSTD error.
+ */
+static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, ZSTD_DDict* ddict, int dictID, ZSTD_customMem customMem) {
+    if ((float)hashSet->ddictPtrCount / (float)hashSet->ddictPtrTableSize > DDICT_HASHSET_MAX_LOAD_FACTOR) {
+        FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
+    }
+    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict, dictID), "");
+    return 0;
+}
+
+/* Fetches a DDict with the given dictID */
+static ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, int dictID) {
+    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    for (;;) {
+        size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
+        if (currDictID == dictID || currDictID == 0) {
+            /* currDictID == 0 implies a NULL ddict entry */
+            break;
+        } else {
+            idx++;
+        }
+    }
+    return hashSet->ddictPtrTable[idx];
+}
+
+/* Allocates space for and returns a ddict hash set */
+static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
+    ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
+    ret->ddictPtrTable = (ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
+    ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
+    ret->ddictPtrCount = 0;
+    return ret;
+}
+
+
 size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
                                    const void* dict, size_t dictSize,
                                          ZSTD_dictLoadMethod_e dictLoadMethod,
diff --git a/programs/have_lzma.c b/programs/have_lzma.c
new file mode 100644
index 000000000..bad7f6e8a
--- /dev/null
+++ b/programs/have_lzma.c
@@ -0,0 +1,2 @@
+#include <lzma.h>
+int main(void) { return 0; }
\ No newline at end of file

From fd5b608f1ca9dcf883d58bb253ee951117b4ce24 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Dec 2020 16:09:31 -0500
Subject: [PATCH 27/80] Add parameter to control multiple DDicts

---
 lib/decompress/zstd_decompress.c          |  9 +++++++++
 lib/decompress/zstd_decompress_internal.h |  1 +
 lib/zstd.h                                | 23 ++++++++++++++++++++++-
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index c3d4552f8..3cb94eae9 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1530,6 +1530,9 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
             bounds.lowerBound = (int)ZSTD_d_validateChecksum;
             bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
             return bounds;
+        case ZSTD_d_refMultipleDDicts:
+            bounds.lowerBound = (int)ZSTD_d_refSingleDict;
+            bounds.upperBound = (int)ZSTD_d_refMultipleDicts;
         default:;
     }
     bounds.error = ERROR(parameter_unsupported);
@@ -1567,6 +1570,9 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value
         case ZSTD_d_forceIgnoreChecksum:
             *value = (int)dctx->forceIgnoreChecksum;
             return 0;
+        case ZSTD_d_refMultipleDDicts:
+            *value = (int)dctx->refMultipleDDicts;
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
@@ -1593,6 +1599,9 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
             dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
             return 0;
+        case ZSTD_d_refMultipleDDicts:
+            CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
+            dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 453487de5..3724521e5 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -136,6 +136,7 @@ struct ZSTD_DCtx_s
     U32 dictID;
     int ddictIsCold;             /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
     ZSTD_dictUses_e dictUses;
+    ZSTD_refMultipleDDicts_e refMultipleDDicts;     /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
 
     /* streaming */
     ZSTD_dStreamStage streamStage;
diff --git a/lib/zstd.h b/lib/zstd.h
index 4ff03e1e9..b240f2dd4 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -546,12 +546,14 @@ typedef enum {
      * ZSTD_d_format
      * ZSTD_d_stableOutBuffer
      * ZSTD_d_forceIgnoreChecksum
+     * ZSTD_d_refMultipleDDicts
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
      ZSTD_d_experimentalParam2=1001,
-     ZSTD_d_experimentalParam3=1002
+     ZSTD_d_experimentalParam3=1002,
+     ZSTD_d_experimentalParam4=1003
 
 } ZSTD_dParameter;
 
@@ -1205,6 +1207,12 @@ typedef enum {
     ZSTD_d_ignoreChecksum = 1
 } ZSTD_forceIgnoreChecksum_e;
 
+typedef enum {
+    /* Note: this enum controls ZSTD_d_refMultipleDDicts */
+    ZSTD_d_refSingleDict = 0,
+    ZSTD_d_refMultipleDicts = 1,
+} ZSTD_refMultipleDDicts_e;
+
 typedef enum {
     /* Note: this enum and the behavior it controls are effectively internal
      * implementation details of the compressor. They are expected to continue
@@ -2000,6 +2008,19 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
  */
 #define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
 
+/* ZSTD_d_refMultipleDDicts
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * If enabled and dctx is allocated on the heap, then additional memory will be allocated
+ * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
+ * using a given ZSTD_DCtx, rather than overwriting the previous DCtx referenced, will
+ * store all references, and at decompression time, the appropriate dictID is selected
+ * from the set of DDicts based on the dictID in the frame. 
+ */
+#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
+
+
 /*! ZSTD_DCtx_setFormat() :
  *  Instruct the decoder context about what kind of data to decode next.
  *  This instruction is mandatory to decode data without a fully-formed header,

From 5a6d3eef2b54111f6946c91c3454d6ef9318139e Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Dec 2020 16:33:57 -0500
Subject: [PATCH 28/80] Allocate memory for DDict hash set when parameter is
 set

---
 lib/decompress/zstd_decompress.c          | 22 ++++++++++++++++++++++
 lib/decompress/zstd_decompress_internal.h |  1 +
 2 files changed, 23 insertions(+)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 3cb94eae9..21fae1ae6 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -122,6 +122,8 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
     dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
     ZSTD_DCtx_resetParameters(dctx);
     dctx->validateChecksum = 1;
+    dctx->refMultipleDDicts = 0;
+    dctx->ddictSet = NULL;
 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     dctx->dictContentEndForFuzzing = NULL;
 #endif
@@ -178,6 +180,10 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
         if (dctx->legacyContext)
             ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
 #endif
+        if (dctx->ddictSet) {
+            ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
+            dctx->ddictSet = NULL;
+        }
         ZSTD_customFree(dctx, cMem);
         return 0;
     }
@@ -1400,6 +1406,10 @@ static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
     return ret;
 }
 
+static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+    if (hashSet->ddictPtrTable) ZSTD_customFree(hashSet->ddictPtrTable, customMem);
+    ZSTD_customFree(hashSet, customMem);
+}
 
 size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
                                    const void* dict, size_t dictSize,
@@ -1602,6 +1612,18 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
         case ZSTD_d_refMultipleDDicts:
             CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
+            if (dctx->refMultipleDDicts == ZSTD_d_refMultipleDicts) {
+                if (dctx->ddictSet == NULL) {
+                    if (dctx->staticSize != 0) {
+                        RETURN_ERROR(memory_allocation, "Multiple DDicts are not allowed with static dctx!");
+                    }
+                    dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
+                }
+            } else {
+                ZSTD_freeDDictHashSet(dctx->ddictSet, dctx->customMem);
+                dctx->ddictSet = NULL;
+            }
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 3724521e5..053ecc162 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -136,6 +136,7 @@ struct ZSTD_DCtx_s
     U32 dictID;
     int ddictIsCold;             /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
     ZSTD_dictUses_e dictUses;
+    ZSTD_DDictHashSet* ddictSet;                    /* Hash set for multiple ddicts */
     ZSTD_refMultipleDDicts_e refMultipleDDicts;     /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
 
     /* streaming */

From d1a6a9d285d3a84ca30574b40a4e613322276f3c Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 28 Dec 2020 10:40:29 -0500
Subject: [PATCH 29/80] Reference requested dict ID at decompression time

---
 lib/decompress/zstd_decompress.c | 34 +++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 21fae1ae6..abcb48320 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -648,6 +648,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
     }
 
+    /* Reference DDict requested by frame if dctx references multiple ddicts */
+    if (dctx->refMultipleDDicts && dctx->ddictSet) {
+        ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
+        if (frameDDict) {
+            ZSTD_clearDict(dctx);
+            dctx->ddict = frameDDict;
+            dctx->dictUses = ZSTD_use_indefinitely;
+        }
+    }
+
     /* Loop on each block */
     while (1) {
         size_t decodedSize;
@@ -1495,6 +1505,10 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
     if (ddict) {
         dctx->ddict = ddict;
         dctx->dictUses = ZSTD_use_indefinitely;
+        if (dctx->refMultipleDDicts && dctx->ddictSet) {
+            assert(!dctx->staticSize);  /* Impossible: ddictSet cannot have been allocated if static dctx */
+            ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, ZSTD_getDictID_fromDDict(ddict), dctx->customMem);
+        }
     }
     return 0;
 }
@@ -1611,14 +1625,12 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             return 0;
         case ZSTD_d_refMultipleDDicts:
             CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
+            if (dctx->staticSize != 0) {
+                RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
+            }
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
-            if (dctx->refMultipleDDicts == ZSTD_d_refMultipleDicts) {
-                if (dctx->ddictSet == NULL) {
-                    if (dctx->staticSize != 0) {
-                        RETURN_ERROR(memory_allocation, "Multiple DDicts are not allowed with static dctx!");
-                    }
-                    dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
-                }
+            if (dctx->refMultipleDDicts == ZSTD_d_refMultipleDicts && dctx->ddictSet == NULL) {
+                dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
             } else {
                 ZSTD_freeDDictHashSet(dctx->ddictSet, dctx->customMem);
                 dctx->ddictSet = NULL;
@@ -1805,6 +1817,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 #endif
             {   size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
+                if (zds->refMultipleDDicts && zds->ddictSet) {
+                    ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(zds->ddictSet, zds->fParams.dictID);
+                    if (frameDDict) {
+                        ZSTD_clearDict(zds);
+                        zds->ddict = frameDDict;
+                        zds->dictUses = ZSTD_use_indefinitely;
+                    }
+                }
                 DEBUGLOG(5, "header size : %u", (U32)hSize);
                 if (ZSTD_isError(hSize)) {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)

From 7c1a79f232f0661a0e119aea6e57d9a8e40c96f4 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 28 Dec 2020 11:59:58 -0500
Subject: [PATCH 30/80] Add debuglog statements

---
 lib/decompress/zstd_decompress.c          | 233 +++++++++++-----------
 lib/decompress/zstd_decompress_internal.h |   7 +
 lib/zstd.h                                |   3 +
 programs/have_lzma.c                      |   2 -
 4 files changed, 132 insertions(+), 113 deletions(-)
 delete mode 100644 programs/have_lzma.c

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index abcb48320..ce6c9fe4f 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -72,6 +72,108 @@
 #endif
 
 
+
+/***************************
+ * Multiple DDicts Hashset *
+ ***************************/
+
+#define DDICT_HASHSET_MAX_LOAD_FACTOR 0.75
+#define DDICT_HASHSET_TABLE_BASE_SIZE 64
+#define DDICT_HASHSET_RESIZE_FACTOR 2
+
+/* Hash function to determine starting position of dict insertion */
+static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, size_t seed) {
+    return seed % hashSet->ddictPtrTableSize;
+}
+
+/* Adds ddict to a hashset without any chance of resizing it 
+ * Returns 0 on success or a zstd error code
+ */
+static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, int dictID) {
+    RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
+    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    DEBUGLOG(2, "Hashed index: for dictID: %d is %zu", dictID, idx);
+    while (hashSet->ddictPtrTable[idx] != NULL) {
+        /* Replace existing ddict if inserting ddict with same dictID */
+        if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
+            DEBUGLOG(2, "DictID already resides here!");
+            break;
+        }
+        idx++;
+    }
+    DEBUGLOG(2, "Final idx after probing for dictID %d is: %zu", dictID, idx);
+    hashSet->ddictPtrTable[idx] = ddict;
+    hashSet->ddictPtrCount++;
+    return 0;
+}
+
+/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and rehashes all values */
+static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+    size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
+    const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
+    const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
+    size_t oldTableSize = hashSet->ddictPtrTableSize;
+    size_t i = 0;
+
+    DEBUGLOG(2, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
+    RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
+    hashSet->ddictPtrTable = newTable;
+    hashSet->ddictPtrTableSize = newTableSize;
+    hashSet->ddictPtrCount = 0;
+    for (i = 0; i < oldTableSize; ++i) {
+        if (oldTable[i] != NULL) {
+            FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i], ZSTD_getDictID_fromDDict(oldTable[i])), "");
+        }
+    }
+    ZSTD_customFree(oldTable, customMem);
+    return 0;
+}
+
+/* Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
+ * Returns 0 on success, or a ZSTD error.
+ */
+static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, int dictID, ZSTD_customMem customMem) {
+    DEBUGLOG(2, "Adding dict ID: %d to set. Count: %zu Tablesize: %zu", dictID, hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
+    if ((float)hashSet->ddictPtrCount / (float)hashSet->ddictPtrTableSize > DDICT_HASHSET_MAX_LOAD_FACTOR) {
+        FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
+    }
+    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict, dictID), "");
+    return 0;
+}
+
+/* Fetches a DDict with the given dictID */
+static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, int dictID) {
+    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    DEBUGLOG(2, "Hashed index: for dictID: %d is %zu", dictID, idx);
+    for (;;) {
+        size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
+        if (currDictID == dictID || currDictID == 0) {
+            /* currDictID == 0 implies a NULL ddict entry */
+            break;
+        } else {
+            idx++;
+        }
+    }
+    DEBUGLOG(2, "Final idx after probing for dictID %d is: %zu", dictID, idx);
+    return hashSet->ddictPtrTable[idx];
+}
+
+/* Allocates space for and returns a ddict hash set */
+static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
+    ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
+    ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
+    ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
+    ret->ddictPtrCount = 0;
+    return ret;
+}
+
+static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+    if (hashSet->ddictPtrTable) ZSTD_customFree(hashSet->ddictPtrTable, customMem);
+    ZSTD_customFree(hashSet, customMem);
+}
+
+
+
 /*-*************************************************************
 *   Context management
 ***************************************************************/
@@ -447,12 +549,25 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
 
 /** ZSTD_decodeFrameHeader() :
  * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
+ * If multiple DDict references are enabled, also will choose the correct DDict to use.
  * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
 static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
 {
     size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
     if (ZSTD_isError(result)) return result;    /* invalid header */
     RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
+    /* Reference DDict requested by frame if dctx references multiple ddicts */
+    if (dctx->refMultipleDDicts && dctx->ddictSet) {
+        DEBUGLOG(2, "Choosing the correct DDict at decompression time!");
+        const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
+        if (frameDDict) {
+            DEBUGLOG(2, "DDict found!");
+            ZSTD_clearDict(dctx);
+            dctx->dictID = dctx->fParams.dictID;
+            dctx->ddict = frameDDict;
+            dctx->dictUses = ZSTD_use_indefinitely;
+        }
+    }
 #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     /* Skip the dictID check in fuzzing mode, because it makes the search
      * harder.
@@ -574,7 +689,6 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
     return bound;
 }
 
-
 /*-*************************************************************
  *   Frame decoding
  ***************************************************************/
@@ -648,16 +762,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
     }
 
-    /* Reference DDict requested by frame if dctx references multiple ddicts */
-    if (dctx->refMultipleDDicts && dctx->ddictSet) {
-        ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
-        if (frameDDict) {
-            ZSTD_clearDict(dctx);
-            dctx->ddict = frameDDict;
-            dctx->dictUses = ZSTD_use_indefinitely;
-        }
-    }
-
     /* Loop on each block */
     while (1) {
         size_t decodedSize;
@@ -1323,104 +1427,6 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
 size_t ZSTD_DStreamInSize(void)  { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
 size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
 
-
-/***************************
- * Multiple DDicts HashSet *
- ***************************/
-
-#define DDICT_HASHSET_MAX_LOAD_FACTOR 0.75
-#define DDICT_HASHSET_TABLE_BASE_SIZE 64
-#define DDICT_HASHSET_RESIZE_FACTOR 2
-
-/* Hashset using linear probing */
-typedef struct {
-    ZSTD_DDict** ddictPtrTable;
-    size_t ddictPtrTableSize;
-    size_t ddictPtrCount;
-} ZSTD_DDictHashSet;
-
-/* Hash function to determine starting position of dict insertion */
-static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, size_t seed) {
-    return seed % hashSet->ddictPtrTableSize;
-}
-
-/* Adds ddict to a hashset without any chance of resizing it 
- * Returns 0 on success or a zstd error code
- */
-static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, ZSTD_DDict* ddict, int dictID) {
-    RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
-    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
-    while (hashSet->ddictPtrTable[idx] != NULL) {
-        /* Replace existing ddict if inserting ddict with same dictID */
-        if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) break;
-        idx++;
-    }
-    hashSet->ddictPtrTable[idx] = ddict;
-    hashSet->ddictPtrCount++;
-    return 0;
-}
-
-/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and rehashes all values */
-static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
-    size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
-    ZSTD_DDict** newTable = (ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
-    ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
-    size_t oldTableSize = hashSet->ddictPtrTableSize;
-    size_t i = 0;
-    
-    RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
-    hashSet->ddictPtrTable = newTable;
-    hashSet->ddictPtrTableSize = newTableSize;
-    hashSet->ddictPtrCount = 0;
-    for (i = 0; i < oldTableSize; ++i) {
-        if (oldTable[i] != NULL) {
-            FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i], ZSTD_getDictID_fromDDict(oldTable[i])), "");
-        }
-    }
-    ZSTD_customFree(oldTable, customMem);
-    return 0;
-}
-
-/* Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
- * Returns 0 on success, or a ZSTD error.
- */
-static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, ZSTD_DDict* ddict, int dictID, ZSTD_customMem customMem) {
-    if ((float)hashSet->ddictPtrCount / (float)hashSet->ddictPtrTableSize > DDICT_HASHSET_MAX_LOAD_FACTOR) {
-        FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
-    }
-    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict, dictID), "");
-    return 0;
-}
-
-/* Fetches a DDict with the given dictID */
-static ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, int dictID) {
-    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
-    for (;;) {
-        size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
-        if (currDictID == dictID || currDictID == 0) {
-            /* currDictID == 0 implies a NULL ddict entry */
-            break;
-        } else {
-            idx++;
-        }
-    }
-    return hashSet->ddictPtrTable[idx];
-}
-
-/* Allocates space for and returns a ddict hash set */
-static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
-    ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
-    ret->ddictPtrTable = (ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
-    ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
-    ret->ddictPtrCount = 0;
-    return ret;
-}
-
-static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
-    if (hashSet->ddictPtrTable) ZSTD_customFree(hashSet->ddictPtrTable, customMem);
-    ZSTD_customFree(hashSet, customMem);
-}
-
 size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
                                    const void* dict, size_t dictSize,
                                          ZSTD_dictLoadMethod_e dictLoadMethod,
@@ -1507,7 +1513,8 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
         dctx->dictUses = ZSTD_use_indefinitely;
         if (dctx->refMultipleDDicts && dctx->ddictSet) {
             assert(!dctx->staticSize);  /* Impossible: ddictSet cannot have been allocated if static dctx */
-            ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, ZSTD_getDictID_fromDDict(ddict), dctx->customMem);
+            FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict,
+                                                        ZSTD_getDictID_fromDDict(ddict), dctx->customMem), "");
         }
     }
     return 0;
@@ -1557,6 +1564,7 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
         case ZSTD_d_refMultipleDDicts:
             bounds.lowerBound = (int)ZSTD_d_refSingleDict;
             bounds.upperBound = (int)ZSTD_d_refMultipleDicts;
+            return bounds;
         default:;
     }
     bounds.error = ERROR(parameter_unsupported);
@@ -1625,11 +1633,13 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             return 0;
         case ZSTD_d_refMultipleDDicts:
             CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
+            DEBUGLOG(2, "Referencing multiple ddicts param enabled");
             if (dctx->staticSize != 0) {
                 RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
             }
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
             if (dctx->refMultipleDDicts == ZSTD_d_refMultipleDicts && dctx->ddictSet == NULL) {
+                DEBUGLOG(2, "Allocating new hash set");
                 dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
             } else {
                 ZSTD_freeDDictHashSet(dctx->ddictSet, dctx->customMem);
@@ -1818,9 +1828,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 #endif
             {   size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
                 if (zds->refMultipleDDicts && zds->ddictSet) {
-                    ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(zds->ddictSet, zds->fParams.dictID);
+                    const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(zds->ddictSet, zds->fParams.dictID);
                     if (frameDDict) {
                         ZSTD_clearDict(zds);
+                        zds->dictID = zds->fParams.dictID;
                         zds->ddict = frameDDict;
                         zds->dictUses = ZSTD_use_indefinitely;
                     }
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 053ecc162..8877e4d2c 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -99,6 +99,13 @@ typedef enum {
     ZSTD_use_once = 1            /* Use the dictionary once and set to ZSTD_dont_use */
 } ZSTD_dictUses_e;
 
+/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
+typedef struct {
+    const ZSTD_DDict** ddictPtrTable;
+    size_t ddictPtrTableSize;
+    size_t ddictPtrCount;
+} ZSTD_DDictHashSet;
+
 struct ZSTD_DCtx_s
 {
     const ZSTD_seqSymbol* LLTptr;
diff --git a/lib/zstd.h b/lib/zstd.h
index b240f2dd4..c0b590c8d 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2017,6 +2017,9 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
  * using a given ZSTD_DCtx, rather than overwriting the previous DCtx referenced, will
  * store all references, and at decompression time, the appropriate dictID is selected
  * from the set of DDicts based on the dictID in the frame. 
+ * 
+ * WARNING: Enabling this parameter will trigger memory allocation for the hash table, and disabling
+ * this parameter will trigger memory freeing for the hashtable. 
  */
 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
 
diff --git a/programs/have_lzma.c b/programs/have_lzma.c
deleted file mode 100644
index bad7f6e8a..000000000
--- a/programs/have_lzma.c
+++ /dev/null
@@ -1,2 +0,0 @@
-#include <lzma.h>
-int main(void) { return 0; }
\ No newline at end of file

From ea52fc36069f05c581020191e5c878e77db10d7c Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 28 Dec 2020 13:14:58 -0500
Subject: [PATCH 31/80] Use XXHash for hash function, create a sensible public
 interface

---
 lib/decompress/zstd_decompress.c | 135 ++++++++++++++++++++-----------
 lib/zstd.h                       |   3 +-
 2 files changed, 88 insertions(+), 50 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index ce6c9fe4f..f4f8269df 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -81,33 +81,45 @@
 #define DDICT_HASHSET_TABLE_BASE_SIZE 64
 #define DDICT_HASHSET_RESIZE_FACTOR 2
 
-/* Hash function to determine starting position of dict insertion */
-static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, size_t seed) {
-    return seed % hashSet->ddictPtrTableSize;
+/* Hash function to determine starting position of dict insertion within the table
+ * Returns an index between [0, hashSet->ddictPtrTableSize]
+ */
+static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
+    U32 hash = XXH32(&dictID, sizeof(U32), 0);
+    /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
+    return hash & (hashSet->ddictPtrTableSize - 1);
 }
 
-/* Adds ddict to a hashset without any chance of resizing it 
- * Returns 0 on success or a zstd error code
+/* Adds ddict to a hashset resizing it 
+ * Returns 0 if successful (ddict may or may not have actually been added), or a zstd error code if something went wrong
  */
-static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, int dictID) {
+static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, U32 dictID) {
     RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
     size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
-    DEBUGLOG(2, "Hashed index: for dictID: %d is %zu", dictID, idx);
+    DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
     while (hashSet->ddictPtrTable[idx] != NULL) {
         /* Replace existing ddict if inserting ddict with same dictID */
         if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
-            DEBUGLOG(2, "DictID already resides here!");
-            break;
+            DEBUGLOG(2, "DictID already exists, replacing rather than adding");
+            hashSet->ddictPtrTable[idx] = ddict;
+            return 0;
+        }
+        if (idx == hashSet->ddictPtrTableSize - 1) {
+            idx = 0;
+            continue;
         }
         idx++;
     }
-    DEBUGLOG(2, "Final idx after probing for dictID %d is: %zu", dictID, idx);
+    DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
     hashSet->ddictPtrTable[idx] = ddict;
     hashSet->ddictPtrCount++;
     return 0;
 }
 
-/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and rehashes all values */
+/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
+ * rehashes all values, allocates new table, frees old table.
+ * Returns 0 on success, otherwise a zstd error code.
+ */
 static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
     size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
     const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
@@ -115,7 +127,7 @@ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMe
     size_t oldTableSize = hashSet->ddictPtrTableSize;
     size_t i = 0;
 
-    DEBUGLOG(2, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
+    DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
     RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
     hashSet->ddictPtrTable = newTable;
     hashSet->ddictPtrTableSize = newTableSize;
@@ -126,53 +138,67 @@ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMe
         }
     }
     ZSTD_customFree(oldTable, customMem);
+    DEBUGLOG(4, "Finished re-hash");
     return 0;
 }
 
-/* Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
- * Returns 0 on success, or a ZSTD error.
+/* Fetches a DDict with the given dictID
+ * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
  */
-static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, int dictID, ZSTD_customMem customMem) {
-    DEBUGLOG(2, "Adding dict ID: %d to set. Count: %zu Tablesize: %zu", dictID, hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
-    if ((float)hashSet->ddictPtrCount / (float)hashSet->ddictPtrTableSize > DDICT_HASHSET_MAX_LOAD_FACTOR) {
-        FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
-    }
-    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict, dictID), "");
-    return 0;
-}
-
-/* Fetches a DDict with the given dictID */
-static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, int dictID) {
+static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
     size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
-    DEBUGLOG(2, "Hashed index: for dictID: %d is %zu", dictID, idx);
+    DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
     for (;;) {
         size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
         if (currDictID == dictID || currDictID == 0) {
             /* currDictID == 0 implies a NULL ddict entry */
+            DEBUGLOG(4, "Dict ID not found");
             break;
         } else {
+            if (idx == hashSet->ddictPtrTableSize - 1) {
+                idx = 0;
+                continue;
+            }
             idx++;
         }
     }
-    DEBUGLOG(2, "Final idx after probing for dictID %d is: %zu", dictID, idx);
+    DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
     return hashSet->ddictPtrTable[idx];
 }
 
-/* Allocates space for and returns a ddict hash set */
+/* Allocates space for and returns a ddict hash set
+ * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
+ * Returns NULL if allocation failed.
+ */
 static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
     ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
     ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
     ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
     ret->ddictPtrCount = 0;
+    if (!ret || !ret->ddictPtrTable) {
+        return NULL;
+    }
     return ret;
 }
 
+/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
+ */
 static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
     if (hashSet->ddictPtrTable) ZSTD_customFree(hashSet->ddictPtrTable, customMem);
     ZSTD_customFree(hashSet, customMem);
 }
 
-
+/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
+ * Returns 0 on success, or a ZSTD error.
+ */
+static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, U32 dictID, ZSTD_customMem customMem) {
+    DEBUGLOG(4, "Adding dict ID: %d to hashset with - Count: %zu Tablesize: %zu", dictID, hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
+    if ((float)hashSet->ddictPtrCount / (float)hashSet->ddictPtrTableSize >= DDICT_HASHSET_MAX_LOAD_FACTOR) {
+        FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
+    }
+    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict, dictID), "");
+    return 0;
+}
 
 /*-*************************************************************
 *   Context management
@@ -298,6 +324,24 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
     ZSTD_memcpy(dstDCtx, srcDCtx, toCopy);  /* no need to copy workspace */
 }
 
+/* Public function: Given a dctx with a digested frame params, re-selects the correct DDict based on
+ * the requested dict ID from the frame. ZSTD_d_refMultipleDDicts must be enabled for this function
+ * to be called.
+ */
+static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
+    assert(dctx->refMultipleDDicts && dctx->ddictSet);
+    DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
+    {   const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
+        if (frameDDict) {
+            DEBUGLOG(4, "DDict found!");
+            ZSTD_clearDict(dctx);
+            dctx->dictID = dctx->fParams.dictID;
+            dctx->ddict = frameDDict;
+            dctx->dictUses = ZSTD_use_indefinitely;
+        }
+    }
+}
+
 
 /*-*************************************************************
  *   Frame header decoding
@@ -556,18 +600,12 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
     size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
     if (ZSTD_isError(result)) return result;    /* invalid header */
     RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
+
     /* Reference DDict requested by frame if dctx references multiple ddicts */
     if (dctx->refMultipleDDicts && dctx->ddictSet) {
-        DEBUGLOG(2, "Choosing the correct DDict at decompression time!");
-        const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
-        if (frameDDict) {
-            DEBUGLOG(2, "DDict found!");
-            ZSTD_clearDict(dctx);
-            dctx->dictID = dctx->fParams.dictID;
-            dctx->ddict = frameDDict;
-            dctx->dictUses = ZSTD_use_indefinitely;
-        }
+        ZSTD_DCtx_selectFrameDDict(dctx);
     }
+
 #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     /* Skip the dictID check in fuzzing mode, because it makes the search
      * harder.
@@ -1633,17 +1671,22 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             return 0;
         case ZSTD_d_refMultipleDDicts:
             CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
-            DEBUGLOG(2, "Referencing multiple ddicts param enabled");
+            DEBUGLOG(4, "Referencing multiple ddicts param enabled");
             if (dctx->staticSize != 0) {
                 RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
             }
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
             if (dctx->refMultipleDDicts == ZSTD_d_refMultipleDicts && dctx->ddictSet == NULL) {
-                DEBUGLOG(2, "Allocating new hash set");
+                DEBUGLOG(4, "Allocating new hash set");
                 dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
+                if (!dctx->ddictSet) {
+                    RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
+                }
             } else {
-                ZSTD_freeDDictHashSet(dctx->ddictSet, dctx->customMem);
-                dctx->ddictSet = NULL;
+                if (dctx->ddictSet) {
+                    ZSTD_freeDDictHashSet(dctx->ddictSet, dctx->customMem);
+                    dctx->ddictSet = NULL;
+                }
             }
             return 0;
         default:;
@@ -1828,13 +1871,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 #endif
             {   size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
                 if (zds->refMultipleDDicts && zds->ddictSet) {
-                    const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(zds->ddictSet, zds->fParams.dictID);
-                    if (frameDDict) {
-                        ZSTD_clearDict(zds);
-                        zds->dictID = zds->fParams.dictID;
-                        zds->ddict = frameDDict;
-                        zds->dictUses = ZSTD_use_indefinitely;
-                    }
+                    ZSTD_DCtx_selectFrameDDict(zds);
                 }
                 DEBUGLOG(5, "header size : %u", (U32)hSize);
                 if (ZSTD_isError(hSize)) {
diff --git a/lib/zstd.h b/lib/zstd.h
index c0b590c8d..ede9c5785 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -2019,7 +2019,8 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
  * from the set of DDicts based on the dictID in the frame. 
  * 
  * WARNING: Enabling this parameter will trigger memory allocation for the hash table, and disabling
- * this parameter will trigger memory freeing for the hashtable. 
+ * this parameter will free the memory allocated for the hashtable. Memory is allocated as per
+ * ZSTD_DCtx::customMem.
  */
 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
 

From 22b7bff2bcdfab8ece992420bd703fb14fd0f49b Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 28 Dec 2020 16:43:04 -0500
Subject: [PATCH 32/80] Add unit test, improve documentation

---
 lib/decompress/zstd_decompress.c | 70 +++++++++++++------------
 lib/zstd.h                       | 30 ++++++++---
 tests/fuzzer.c                   | 88 +++++++++++++++++++++++++++-----
 3 files changed, 134 insertions(+), 54 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index f4f8269df..72abba3b9 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -73,9 +73,9 @@
 
 
 
-/***************************
- * Multiple DDicts Hashset *
- ***************************/
+/*************************************
+ * Multiple DDicts Hashset internals *
+ *************************************/
 
 #define DDICT_HASHSET_MAX_LOAD_FACTOR 0.75
 #define DDICT_HASHSET_TABLE_BASE_SIZE 64
@@ -85,7 +85,7 @@
  * Returns an index between [0, hashSet->ddictPtrTableSize]
  */
 static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
-    U32 hash = XXH32(&dictID, sizeof(U32), 0);
+    U32 hash = XXH64(&dictID, sizeof(U32), 0);
     /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
     return hash & (hashSet->ddictPtrTableSize - 1);
 }
@@ -94,8 +94,8 @@ static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 d
  * Returns 0 if successful (ddict may or may not have actually been added), or a zstd error code if something went wrong
  */
 static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, U32 dictID) {
-    RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
     size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
     DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
     while (hashSet->ddictPtrTable[idx] != NULL) {
         /* Replace existing ddict if inserting ddict with same dictID */
@@ -125,7 +125,7 @@ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMe
     const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
     const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
     size_t oldTableSize = hashSet->ddictPtrTableSize;
-    size_t i = 0;
+    size_t i;
 
     DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
     RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
@@ -152,7 +152,6 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet,
         size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
         if (currDictID == dictID || currDictID == 0) {
             /* currDictID == 0 implies a NULL ddict entry */
-            DEBUGLOG(4, "Dict ID not found");
             break;
         } else {
             if (idx == hashSet->ddictPtrTableSize - 1) {
@@ -172,6 +171,7 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet,
  */
 static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
     ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
+    DEBUGLOG(4, "Allocating new hash set");
     ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
     ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
     ret->ddictPtrCount = 0;
@@ -182,10 +182,16 @@ static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
 }
 
 /* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
+ * Note: The ZSTD_DDict* within the table are NOT freed.
  */
 static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
-    if (hashSet->ddictPtrTable) ZSTD_customFree(hashSet->ddictPtrTable, customMem);
-    ZSTD_customFree(hashSet, customMem);
+    DEBUGLOG(4, "Freeing ddict hash set");
+    if (hashSet && hashSet->ddictPtrTable) {
+        ZSTD_customFree(hashSet->ddictPtrTable, customMem);
+    }
+    if (hashSet) {
+        ZSTD_customFree(hashSet, customMem);
+    }
 }
 
 /* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
@@ -229,6 +235,7 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
     dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     dctx->outBufferMode = ZSTD_bm_buffered;
     dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
+    dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
 }
 
 static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
@@ -248,10 +255,8 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
     dctx->noForwardProgress = 0;
     dctx->oversizedDuration = 0;
     dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
-    ZSTD_DCtx_resetParameters(dctx);
-    dctx->validateChecksum = 1;
-    dctx->refMultipleDDicts = 0;
     dctx->ddictSet = NULL;
+    ZSTD_DCtx_resetParameters(dctx);
 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     dctx->dictContentEndForFuzzing = NULL;
 #endif
@@ -324,14 +329,19 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
     ZSTD_memcpy(dstDCtx, srcDCtx, toCopy);  /* no need to copy workspace */
 }
 
-/* Public function: Given a dctx with a digested frame params, re-selects the correct DDict based on
- * the requested dict ID from the frame. ZSTD_d_refMultipleDDicts must be enabled for this function
- * to be called.
+/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
+ * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
+ * accordingly sets the ddict to be used to decompress the frame.
+ * 
+ * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
+ * 
+ * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
  */
 static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
     assert(dctx->refMultipleDDicts && dctx->ddictSet);
     DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
-    {   const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
+    if (dctx->ddict) {
+        const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
         if (frameDDict) {
             DEBUGLOG(4, "DDict found!");
             ZSTD_clearDict(dctx);
@@ -602,7 +612,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
     RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
 
     /* Reference DDict requested by frame if dctx references multiple ddicts */
-    if (dctx->refMultipleDDicts && dctx->ddictSet) {
+    if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
         ZSTD_DCtx_selectFrameDDict(dctx);
     }
 
@@ -727,6 +737,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
     return bound;
 }
 
+
 /*-*************************************************************
  *   Frame decoding
  ***************************************************************/
@@ -1549,7 +1560,13 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
     if (ddict) {
         dctx->ddict = ddict;
         dctx->dictUses = ZSTD_use_indefinitely;
-        if (dctx->refMultipleDDicts && dctx->ddictSet) {
+        if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
+            if (dctx->ddictSet == NULL) {
+                dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
+                if (!dctx->ddictSet) {
+                    RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
+                }
+            }
             assert(!dctx->staticSize);  /* Impossible: ddictSet cannot have been allocated if static dctx */
             FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict,
                                                         ZSTD_getDictID_fromDDict(ddict), dctx->customMem), "");
@@ -1600,8 +1617,8 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
             bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
             return bounds;
         case ZSTD_d_refMultipleDDicts:
-            bounds.lowerBound = (int)ZSTD_d_refSingleDict;
-            bounds.upperBound = (int)ZSTD_d_refMultipleDicts;
+            bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
+            bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
             return bounds;
         default:;
     }
@@ -1671,23 +1688,10 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             return 0;
         case ZSTD_d_refMultipleDDicts:
             CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
-            DEBUGLOG(4, "Referencing multiple ddicts param enabled");
             if (dctx->staticSize != 0) {
                 RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
             }
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
-            if (dctx->refMultipleDDicts == ZSTD_d_refMultipleDicts && dctx->ddictSet == NULL) {
-                DEBUGLOG(4, "Allocating new hash set");
-                dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
-                if (!dctx->ddictSet) {
-                    RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
-                }
-            } else {
-                if (dctx->ddictSet) {
-                    ZSTD_freeDDictHashSet(dctx->ddictSet, dctx->customMem);
-                    dctx->ddictSet = NULL;
-                }
-            }
             return 0;
         default:;
     }
diff --git a/lib/zstd.h b/lib/zstd.h
index ede9c5785..884a2f54a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1001,6 +1001,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
 /*! ZSTD_DCtx_refDDict() :
  *  Reference a prepared dictionary, to be used to decompress next frames.
  *  The dictionary remains active for decompression of future frames using same DCtx.
+ * 
+ *  If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
+ *  will store the DDict references in a table, and the DDict used for decompression
+ *  will be determined at decompression time, as per the dict ID in the frame.
+ *  The memory for the table is allocated on the first call to refDDict, and can be
+ *  freed with ZSTD_freeDCtx().
+ * 
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Note 1 : Currently, only one dictionary can be managed.
  *           Referencing a new dictionary effectively "discards" any previous one.
@@ -1209,8 +1216,8 @@ typedef enum {
 
 typedef enum {
     /* Note: this enum controls ZSTD_d_refMultipleDDicts */
-    ZSTD_d_refSingleDict = 0,
-    ZSTD_d_refMultipleDicts = 1,
+    ZSTD_rmd_refSingleDDict = 0,
+    ZSTD_rmd_refMultipleDDicts = 1
 } ZSTD_refMultipleDDicts_e;
 
 typedef enum {
@@ -2014,13 +2021,20 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
  *
  * If enabled and dctx is allocated on the heap, then additional memory will be allocated
  * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
- * using a given ZSTD_DCtx, rather than overwriting the previous DCtx referenced, will
- * store all references, and at decompression time, the appropriate dictID is selected
- * from the set of DDicts based on the dictID in the frame. 
+ * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
+ * store all references. At decompression time, the appropriate dictID is selected
+ * from the set of DDicts based on the dictID in the frame.
  * 
- * WARNING: Enabling this parameter will trigger memory allocation for the hash table, and disabling
- * this parameter will free the memory allocated for the hashtable. Memory is allocated as per
- * ZSTD_DCtx::customMem.
+ * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
+ * 
+ * Param has values of byte ZSTD_refMultipleDDicts_e
+ * 
+ * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
+ * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
+ * Memory is allocated as per ZSTD_DCtx::customMem.
+ * 
+ * Although this function allocates memory for the table, the user is still responsible for
+ * memory management of the underlying ZSTD_DDict* themselves.
  */
 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7e3b4628e..a67113edd 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1770,6 +1770,19 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t dictSize;
         U32 dictID;
         size_t dictHeaderSize;
+        size_t dictBufferFixedSize = 144;
+        unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
+                                                    0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                                                    0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
+                                                    0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
+                                                    0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+                                                    0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+                                                    0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
+                                                    0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
+                                                    0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
+                                                    0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
+                                                    0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
+                                                    0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
 
         if (dictBuffer==NULL || samplesSizes==NULL) {
             free(dictBuffer);
@@ -1865,19 +1878,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
 
         DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
-        {   unsigned char const dictBufferFixed[144] = { 0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
-                                                         0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-                                                         0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
-                                                         0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
-                                                         0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
-                                                         0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
-                                                         0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
-                                                         0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
-                                                         0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
-                                                         0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
-                                                         0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
-                                                         0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69 };
-            dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, 144);
+        {   dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
             if (dictHeaderSize != 115) goto _output_error;
         }
         DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
@@ -2331,6 +2332,67 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
+        DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
+        {
+            const size_t numDicts = 128;
+            const size_t numFrames = 4;
+            size_t i;
+            ZSTD_DCtx* dctx = ZSTD_createDCtx();
+            ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
+            ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
+            U32 dictIDSeed = seed;
+            /* Create new compressed buffer that will hold frames with differing dictIDs */
+            char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize);  /* Modifiable copy of fixed full dict buffer */
+
+            ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
+            cSize = 0;
+            /* Create a bunch of DDicts with random dict IDs */
+            for (i = 0; i < numDicts; ++i) {
+                U32 currDictID = FUZ_rand(&dictIDSeed);
+                MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
+                ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
+                cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
+                if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
+                    goto _output_error;
+                }
+            }
+            /* Compress a few frames using random CDicts */
+            {
+                size_t off = 0;
+                /* only use the first half so we don't push against size limit of compressedBuffer */
+                size_t const segSize = (CNBuffSize / 2) / numFrames;
+                for (i = 0; i < numFrames; i++) {
+                    size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
+                    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+                    {   CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
+                                    (BYTE*)compressedBuffer + off, CNBuffSize - off,
+                                    (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
+                                    cdictTable[dictIdx]));
+                        off += r;
+                    }
+                }
+                cSize = off;
+            }
+
+            /* We should succeed to decompression even though different dicts were used on different frames */
+            ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
+            /* Reference every single ddict we made */
+            for (i = 0; i < numDicts; ++i) {
+                CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
+            }
+            CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+            ZSTD_freeDCtx(dctx);
+            for (i = 0; i < numDicts; ++i) {
+                ZSTD_freeCDict(cdictTable[i]);
+                ZSTD_freeDDict(ddictTable[i]);
+            }
+            free(dictBufferMulti);
+            free(ddictTable);
+            free(cdictTable);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         ZSTD_freeCCtx(cctx);
         free(dictBuffer);
         free(samplesSizes);

From c2c9b8a7eca5b3080deb9e4a8dc180f5721ad7ca Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Thu, 7 Jan 2021 11:53:35 -0500
Subject: [PATCH 33/80] Address comments, clean up interface/internals

---
 lib/decompress/zstd_decompress.c | 35 ++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 72abba3b9..14f94f31b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -62,6 +62,7 @@
 #include "../common/fse.h"
 #define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
+#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
 #include "../common/zstd_internal.h"  /* blockProperties_t */
 #include "zstd_decompress_internal.h"   /* ZSTD_DCtx */
 #include "zstd_ddict.h"  /* ZSTD_DDictDictContent */
@@ -77,7 +78,13 @@
  * Multiple DDicts Hashset internals *
  *************************************/
 
-#define DDICT_HASHSET_MAX_LOAD_FACTOR 0.75
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3   /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
+                                                     * Currently, that means a 0.75 load factor. 
+                                                     * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
+                                                     * the load factor of the ddict hash set.
+                                                     */
+
 #define DDICT_HASHSET_TABLE_BASE_SIZE 64
 #define DDICT_HASHSET_RESIZE_FACTOR 2
 
@@ -90,17 +97,19 @@ static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 d
     return hash & (hashSet->ddictPtrTableSize - 1);
 }
 
-/* Adds ddict to a hashset resizing it 
- * Returns 0 if successful (ddict may or may not have actually been added), or a zstd error code if something went wrong
+/* Adds DDict to a hashset without resizing it.
+ * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
+ * Returns 0 if successful, or a zstd error code if something went wrong.
  */
-static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, U32 dictID) {
+static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
+    U32 dictID = ZSTD_getDictID_fromDDict(ddict);
     size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
     RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
     DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
     while (hashSet->ddictPtrTable[idx] != NULL) {
         /* Replace existing ddict if inserting ddict with same dictID */
         if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
-            DEBUGLOG(2, "DictID already exists, replacing rather than adding");
+            DEBUGLOG(4, "DictID already exists, replacing rather than adding");
             hashSet->ddictPtrTable[idx] = ddict;
             return 0;
         }
@@ -134,7 +143,7 @@ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMe
     hashSet->ddictPtrCount = 0;
     for (i = 0; i < oldTableSize; ++i) {
         if (oldTable[i] != NULL) {
-            FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i], ZSTD_getDictID_fromDDict(oldTable[i])), "");
+            FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
         }
     }
     ZSTD_customFree(oldTable, customMem);
@@ -154,10 +163,7 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet,
             /* currDictID == 0 implies a NULL ddict entry */
             break;
         } else {
-            if (idx == hashSet->ddictPtrTableSize - 1) {
-                idx = 0;
-                continue;
-            }
+            idx &= (hashSet->ddictPtrTableSize - 1);    /* Goes to start of table when we reach the end */
             idx++;
         }
     }
@@ -197,12 +203,12 @@ static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem cus
 /* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
  * Returns 0 on success, or a ZSTD error.
  */
-static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, U32 dictID, ZSTD_customMem customMem) {
+static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
     DEBUGLOG(4, "Adding dict ID: %d to hashset with - Count: %zu Tablesize: %zu", dictID, hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
-    if ((float)hashSet->ddictPtrCount / (float)hashSet->ddictPtrTableSize >= DDICT_HASHSET_MAX_LOAD_FACTOR) {
+    if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
         FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
     }
-    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict, dictID), "");
+    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
     return 0;
 }
 
@@ -1568,8 +1574,7 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
                 }
             }
             assert(!dctx->staticSize);  /* Impossible: ddictSet cannot have been allocated if static dctx */
-            FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict,
-                                                        ZSTD_getDictID_fromDDict(ddict), dctx->customMem), "");
+            FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
         }
     }
     return 0;

From 17222654bff8172f95cbb0b364fdc5a873477db6 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Thu, 7 Jan 2021 12:07:35 -0500
Subject: [PATCH 34/80] Add streaming decompression to unit test

---
 tests/fuzzer.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a67113edd..b808ad2d2 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -2382,6 +2382,14 @@ static int basicUnitTests(U32 const seed, double compressibility)
                 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
             }
             CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+            /* Streaming decompression should also work */
+            {
+                ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
+                ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
+                while (in.pos < in.size) {
+                    CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
+                }
+            }
             ZSTD_freeDCtx(dctx);
             for (i = 0; i < numDicts; ++i) {
                 ZSTD_freeCDict(cdictTable[i]);

From 4f7584e7a337cf2da7b2768d006fc83f4d5e413e Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Thu, 7 Jan 2021 12:09:46 -0500
Subject: [PATCH 35/80] Allow freestanding lib script regex to detect XXH64(

---
 contrib/freestanding_lib/freestanding.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py
index f8f206944..32d4fcbea 100755
--- a/contrib/freestanding_lib/freestanding.py
+++ b/contrib/freestanding_lib/freestanding.py
@@ -576,7 +576,7 @@ class Freestanding(object):
             )
         if self._xxh64_prefix is not None:
             replacements.append(
-                (re.compile(r"([^\w]|^)(?P<orig>XXH64)_"), self._xxh64_prefix)
+                (re.compile(r"([^\w]|^)(?P<orig>XXH64)[\(_]"), self._xxh64_prefix)
             )
         for filepath in self._dst_lib_file_paths():
             file = FileLines(filepath)

From f9802d80a0d53797f19152334893ec63ade0089c Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 7 Jan 2021 18:47:23 +0100
Subject: [PATCH 36/80] fix typos (work done by Andrea Gelmini)

---
 lib/compress/zstd_compress_superblock.c | 2 +-
 lib/compress/zstd_cwksp.h               | 2 +-
 lib/compress/zstd_fast.c                | 2 +-
 lib/compress/zstd_ldm.c                 | 2 +-
 lib/dictBuilder/divsufsort.c            | 2 +-
 lib/zstd.h                              | 4 ++--
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index 593ac416f..6a7b02342 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -304,7 +304,7 @@ ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
  *  before we know the table size + compressed size, so we have a bound on the
  *  table size. If we guessed incorrectly, we fall back to uncompressed literals.
  *
- *  We write the header when writeEntropy=1 and set entropyWrriten=1 when we succeeded
+ *  We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
  *  in writing the header, otherwise it is set to 0.
  *
  *  hufMetadata->hType has literals block type info.
diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 9050cfa8d..daec8bd4d 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -503,7 +503,7 @@ MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
 
 /**
  * Moves the management of a workspace from one cwksp to another. The src cwksp
- * is left in an invalid state (src must be re-init()'ed before its used again).
+ * is left in an invalid state (src must be re-init()'ed before it's used again).
  */
 MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
     *dst = *src;
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 9ac8ab5c9..8f8dfcd7a 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -242,7 +242,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
     assert(endIndex - prefixStartIndex <= maxDistance);
     (void)maxDistance; (void)endIndex;   /* these variables are not used when assert() is disabled */
 
-    /* ensure there will be no no underflow
+    /* ensure there will be no underflow
      * when translating a dict index into a local index */
     assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
 
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index e7fead31b..42e0a5e10 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -620,7 +620,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
 
     assert(rawSeqStore->pos <= rawSeqStore->size);
     assert(rawSeqStore->size <= rawSeqStore->capacity);
-    /* Loop through each sequence and apply the block compressor to the lits */
+    /* Loop through each sequence and apply the block compressor to the lists */
     while (rawSeqStore->pos < rawSeqStore->size && ip < iend) {
         /* maybeSplitSequence updates rawSeqStore->pos */
         rawSeq const sequence = maybeSplitSequence(rawSeqStore,
diff --git a/lib/dictBuilder/divsufsort.c b/lib/dictBuilder/divsufsort.c
index ead922044..a2870fb3b 100644
--- a/lib/dictBuilder/divsufsort.c
+++ b/lib/dictBuilder/divsufsort.c
@@ -1576,7 +1576,7 @@ note:
     /* Construct the inverse suffix array of type B* suffixes using trsort. */
     trsort(ISAb, SA, m, 1);
 
-    /* Set the sorted order of tyoe B* suffixes. */
+    /* Set the sorted order of type B* suffixes. */
     for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) {
       for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { }
       if(0 <= i) {
diff --git a/lib/zstd.h b/lib/zstd.h
index 4ff03e1e9..e5b55a25e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -948,7 +948,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
  *  Reference a prepared dictionary, to be used for all next compressed frames.
  *  Note that compression parameters are enforced from within CDict,
  *  and supersede any compression parameter previously set within CCtx.
- *  The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+ *  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
  *  The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
  *  The dictionary will remain valid for future compressed frames using same CCtx.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
@@ -1286,7 +1286,7 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
  *  `srcSize` must be the _exact_ size of this series
  *       (i.e. there should be a frame boundary at `src + srcSize`)
  *  @return : - upper-bound for the decompressed size of all data in all successive frames
- *            - if an error occured: ZSTD_CONTENTSIZE_ERROR
+ *            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
  *
  *  note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame.
  *  note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.

From 3324e87cff43bdb0ebfe63375264cad60da56046 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 7 Jan 2021 10:37:27 -0800
Subject: [PATCH 37/80] Added library version check

---
 programs/Makefile  |  2 +-
 programs/zstdcli.c | 25 ++++++++++++++++++++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 48a60ad0f..3eb658f42 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -289,7 +289,7 @@ zstd-noxz : LZMALD  :=
 zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
 zstd-noxz : zstd
 
-## zstd-dll: zstd executable linked to dynamic library libzstd (must already exist)
+## zstd-dll: zstd executable linked to dynamic library libzstd (must have same version)
 .PHONY: zstd-dll
 zstd-dll : LDFLAGS+= -L$(ZSTDDIR)
 zstd-dll : LDLIBS += -lzstd
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 1b0b6761d..a46878375 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -103,6 +103,24 @@ typedef enum { cover, fastCover, legacy } dictType;
 static int g_displayLevel = DISPLAY_LEVEL_DEFAULT;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
 
 
+/*-************************************
+*  Check Version (when CLI linked to dynamic library)
+**************************************/
+
+/* Due to usage of experimental symbols and capabilities by the CLI,
+ * the CLI must be linked against a dynamic library of same version */
+static void checkLibVersion(void)
+{
+    if (strcmp(ZSTD_VERSION_STRING, ZSTD_versionString())) {
+        DISPLAYLEVEL(1, "Error : incorrect library version (expecting : %s ; actual : %s ) \n",
+                    ZSTD_VERSION_STRING, ZSTD_versionString());
+        DISPLAYLEVEL(1, "Please update library to version %s, or use stand-alone zstd binary \n",
+                    ZSTD_VERSION_STRING);
+        exit(1);
+    }
+}
+
+
 /*-************************************
 *  Command Line
 **************************************/
@@ -753,6 +771,7 @@ int main(int const argCount, const char* argv[])
 
 
     /* init */
+    checkLibVersion();
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
     (void)memLimit;
     assert(argCount >= 1);
@@ -1281,15 +1300,15 @@ int main(int const argCount, const char* argv[])
         DISPLAY("error : can't use --patch-from=# on multiple files \n");
         CLEAN_RETURN(1);
     }
-    
-    /* No status message in pipe mode (stdin - stdout) */	
+
+    /* No status message in pipe mode (stdin - stdout) */
     hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
 
     if (hasStdout && (g_displayLevel==2)) g_displayLevel=1;
 
     /* IO Stream/File */
     FIO_setHasStdoutOutput(fCtx, hasStdout);
-    FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize); 
+    FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize);
     FIO_determineHasStdinInput(fCtx, filenames);
     FIO_setNotificationLevel(g_displayLevel);
     FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);

From 92a2b5ccc9b02c17e3004b88eaeb264463ddae47 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 7 Jan 2021 23:30:42 +0100
Subject: [PATCH 38/80] fixup: lits means literals

---
 lib/compress/zstd_ldm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 42e0a5e10..b7c901ba1 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -620,7 +620,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
 
     assert(rawSeqStore->pos <= rawSeqStore->size);
     assert(rawSeqStore->size <= rawSeqStore->capacity);
-    /* Loop through each sequence and apply the block compressor to the lists */
+    /* Loop through each sequence and apply the block compressor to the literals */
     while (rawSeqStore->pos < rawSeqStore->size && ip < iend) {
         /* maybeSplitSequence updates rawSeqStore->pos */
         rawSeq const sequence = maybeSplitSequence(rawSeqStore,

From 9ae0dd93368e3526b6e57e143425d395cdff171c Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Thu, 7 Jan 2021 12:13:27 -0500
Subject: [PATCH 39/80] Fix Visual and staticanalyze warnings

---
 lib/decompress/zstd_decompress.c | 19 +++++++++----------
 tests/fuzzer.c                   |  1 -
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 14f94f31b..3701170cd 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -92,7 +92,7 @@
  * Returns an index between [0, hashSet->ddictPtrTableSize]
  */
 static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
-    U32 hash = XXH64(&dictID, sizeof(U32), 0);
+    const U64 hash = XXH64(&dictID, sizeof(U32), 0);
     /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
     return hash & (hashSet->ddictPtrTableSize - 1);
 }
@@ -102,8 +102,9 @@ static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 d
  * Returns 0 if successful, or a zstd error code if something went wrong.
  */
 static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
-    U32 dictID = ZSTD_getDictID_fromDDict(ddict);
+    const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
     size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
     RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
     DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
     while (hashSet->ddictPtrTable[idx] != NULL) {
@@ -113,10 +114,7 @@ static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const Z
             hashSet->ddictPtrTable[idx] = ddict;
             return 0;
         }
-        if (idx == hashSet->ddictPtrTableSize - 1) {
-            idx = 0;
-            continue;
-        }
+        idx &= idxRangeMask;
         idx++;
     }
     DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
@@ -146,7 +144,7 @@ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMe
             FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
         }
     }
-    ZSTD_customFree(oldTable, customMem);
+    ZSTD_customFree((void*)oldTable, customMem);
     DEBUGLOG(4, "Finished re-hash");
     return 0;
 }
@@ -156,6 +154,7 @@ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMe
  */
 static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
     size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
     DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
     for (;;) {
         size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
@@ -163,7 +162,7 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet,
             /* currDictID == 0 implies a NULL ddict entry */
             break;
         } else {
-            idx &= (hashSet->ddictPtrTableSize - 1);    /* Goes to start of table when we reach the end */
+            idx &= idxRangeMask;    /* Goes to start of table when we reach the end */
             idx++;
         }
     }
@@ -193,7 +192,7 @@ static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
 static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
     DEBUGLOG(4, "Freeing ddict hash set");
     if (hashSet && hashSet->ddictPtrTable) {
-        ZSTD_customFree(hashSet->ddictPtrTable, customMem);
+        ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
     }
     if (hashSet) {
         ZSTD_customFree(hashSet, customMem);
@@ -204,7 +203,7 @@ static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem cus
  * Returns 0 on success, or a ZSTD error.
  */
 static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
-    DEBUGLOG(4, "Adding dict ID: %d to hashset with - Count: %zu Tablesize: %zu", dictID, hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
+    DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
     if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
         FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
     }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index b808ad2d2..157b798ac 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -2345,7 +2345,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
             char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize);  /* Modifiable copy of fixed full dict buffer */
 
             ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
-            cSize = 0;
             /* Create a bunch of DDicts with random dict IDs */
             for (i = 0; i < numDicts; ++i) {
                 U32 currDictID = FUZ_rand(&dictIDSeed);

From 821d9acd17f251a8ee548eadbc3c5bccdfca7691 Mon Sep 17 00:00:00 2001
From: yumeyao <yumeyao@gmail.com>
Date: Sat, 9 Jan 2021 06:27:31 +0800
Subject: [PATCH 40/80] Fix visibility of symbols in .so (#2441)

Fix visibility of symbols in .so and add an alias for renamed API
---
 lib/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index a0c0ca10b..a3b2ee740 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -255,8 +255,8 @@ else  # not Windows
 
 LIBZSTD = libzstd.$(SHARED_EXT_VER)
 .PHONY: $(LIBZSTD)  # must be run every time
-$(LIBZSTD): CFLAGS += -fPIC
-$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden
+$(LIBZSTD): CFLAGS += -fPIC -fvisibility=hidden
+$(LIBZSTD): LDFLAGS += -shared
 
 ifndef BUILD_DIR
 # determine BUILD_DIR from compilation flags

From 0766540b598a5a1e4d8935bfa2513772f7f1e91b Mon Sep 17 00:00:00 2001
From: Christoph Reiter <reiter.christoph@gmail.com>
Date: Sat, 9 Jan 2021 09:24:45 +0100
Subject: [PATCH 41/80] cmake: use configure_file() for creating the .pc file

Escaping in add_custom_target() seems to depend on the shell used in the cmake
generator and using Ninja on Windows, which uses cmd.exe, results in stray backslashes
in the .pc file.

Instead of going through escaping hell just use configure_file() with the existing
libzstd.pc.in file already used by the simple Makefile based build system.

This fixes the .pc file syntax when building zstd with CMake+Ninja+gcc on Windows.
---
 build/cmake/lib/CMakeLists.txt  | 19 ++++---------------
 build/cmake/lib/pkgconfig.cmake |  1 -
 2 files changed, 4 insertions(+), 16 deletions(-)
 delete mode 100644 build/cmake/lib/pkgconfig.cmake

diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 088c8760a..f34c950a3 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -137,7 +137,7 @@ endif ()
 if (UNIX OR MINGW)
     # pkg-config
     set(PREFIX "${CMAKE_INSTALL_PREFIX}")
-    set(EXEC_PREFIX "\\$$\{prefix}")
+    set(EXEC_PREFIX "\${prefix}")
     set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
     set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
     set(VERSION "${zstd_VERSION}")
@@ -149,24 +149,13 @@ if (UNIX OR MINGW)
     string(SUBSTRING "${INCLUDEDIR}" ${PREFIX_LENGTH} -1 INCLUDEDIR_SUFFIX)
 
     if ("${INCLUDEDIR_PREFIX}" STREQUAL "${PREFIX}")
-        set(INCLUDEDIR_PREFIX "\\$$\{prefix}")
+        set(INCLUDEDIR "\${prefix}${INCLUDEDIR_SUFFIX}")
     endif()
     if ("${LIBDIR_PREFIX}" STREQUAL "${PREFIX}")
-        set(LIBDIR_PREFIX "\\$$\{exec_prefix}")
+        set(LIBDIR "\${exec_prefix}${LIBDIR_SUFFIX}")
     endif()
 
-    add_custom_target(libzstd.pc ALL
-            ${CMAKE_COMMAND}
-            -DIN=${LIBRARY_DIR}/libzstd.pc.in
-            -DOUT="libzstd.pc"
-            -DPREFIX="${PREFIX}"
-            -DEXEC_PREFIX="${EXEC_PREFIX}"
-            -DINCLUDEDIR="${INCLUDEDIR_PREFIX}${INCLUDEDIR_SUFFIX}"
-            -DLIBDIR="${LIBDIR_PREFIX}${LIBDIR_SUFFIX}"
-            -DVERSION="${VERSION}"
-            -P ${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake
-            COMMENT "Creating pkg-config file")
-
+    configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
     install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 endif ()
 
diff --git a/build/cmake/lib/pkgconfig.cmake b/build/cmake/lib/pkgconfig.cmake
deleted file mode 100644
index 8f805a197..000000000
--- a/build/cmake/lib/pkgconfig.cmake
+++ /dev/null
@@ -1 +0,0 @@
-configure_file("${IN}" "${OUT}" @ONLY)

From 1d6d64afa30b4f5aeee73a95c9a7f661efd51de7 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 11 Jan 2021 08:53:29 -0500
Subject: [PATCH 42/80] Change year to 2021 for compression format file

---
 doc/zstd_compression_format.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 0af6bf91a..5c7deb9c4 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -3,7 +3,7 @@ Zstandard Compression Format
 
 ### Notices
 
-Copyright (c) 2016-2020 Yann Collet, Facebook, Inc.
+Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,

From 8b6a4b5b7c46c44507cb84af716bd0699741be19 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 11 Jan 2021 17:53:20 -0500
Subject: [PATCH 43/80] Allow Input From Console When `--force` is Passed

Also update option flag documentation.
---
 programs/zstdcli.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index a46878375..b279287d4 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -144,7 +144,8 @@ static void usage(FILE* f, const char* programName)
 #endif
     DISPLAY_F(f, " -D DICT: use DICT as Dictionary for compression or decompression \n");
     DISPLAY_F(f, " -o file: result stored into `file` (only 1 output file) \n");
-    DISPLAY_F(f, " -f     : overwrite output without prompting, also (de)compress links \n");
+    DISPLAY_F(f, " -f     : disable input and output checks. Allows overwriting existing files,\n");
+    DISPLAY_F(f, "          input from console, output to stdout, operating on links, etc.\n");
     DISPLAY_F(f, "--rm    : remove source file(s) after successful de/compression \n");
     DISPLAY_F(f, " -k     : preserve source file(s) (default) \n");
     DISPLAY_F(f, " -h/-H  : display help/long help and exit \n");
@@ -714,6 +715,7 @@ int main(int const argCount, const char* argv[])
 {
     int argNb,
         followLinks = 0,
+        forceStdin = 0,
         forceStdout = 0,
         hasStdout = 0,
         ldmFlag = 0,
@@ -826,7 +828,7 @@ int main(int const argCount, const char* argv[])
                 if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
                 if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
                 if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
-                if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; continue; }
+                if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; continue; }
                 if (!strcmp(argument, "--version")) { printVersion(); CLEAN_RETURN(0); }
                 if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); }
                 if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
@@ -1007,7 +1009,7 @@ int main(int const argCount, const char* argv[])
                 case 'D': argument++; NEXT_FIELD(dictFileName); break;
 
                     /* Overwrite */
-                case 'f': FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; argument++; break;
+                case 'f': FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; argument++; break;
 
                     /* Verbose mode */
                 case 'v': g_displayLevel++; argument++; break;
@@ -1262,7 +1264,9 @@ int main(int const argCount, const char* argv[])
         outFileName = stdoutmark;  /* when input is stdin, default output is stdout */
 
     /* Check if input/output defined as console; trigger an error in this case */
-    if (!strcmp(filenames->fileNames[0], stdinmark) && IS_CONSOLE(stdin) ) {
+    if (!forceStdin
+     && !strcmp(filenames->fileNames[0], stdinmark)
+     && IS_CONSOLE(stdin) ) {
         DISPLAYLEVEL(1, "stdin is a console, aborting\n");
         CLEAN_RETURN(1);
     }

From 927859f5e8e728a63dd1e4225491b77c9dadb43e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 11 Jan 2021 17:55:58 -0500
Subject: [PATCH 44/80] Also Update Man Page Documentation

---
 programs/zstd.1.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 73670daf6..1e4a7f4fe 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -201,7 +201,8 @@ the last one takes effect.
 * `-o FILE`:
     save result into `FILE`
 * `-f`, `--force`:
-    overwrite output without prompting, and (de)compress symbolic links
+    disable input and output checks. Allows overwriting existing files, input
+    from console, output to stdout, operating on links, etc.
 * `-c`, `--stdout`:
     force write to standard output, even if it is the console
 * `--[no-]sparse`:

From b45d22c85145ec6420987bba067a2d162c7c9a84 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 15 Jan 2021 08:41:42 -0800
Subject: [PATCH 45/80] [contrib][recovery] Add recovery_directory program

This program takes a file with concatenated zstd frames and splits the
file up by frame. E.g. if `dir.zst` has 4 frames:

```
> ./recover_directory dir.zst recovery/file
Recovering 4 files...
Recovered recovery/file0
Recovered recovery/file1
Recovered recovery/file2
Recovered recovery/file3
Complete
```
---
 contrib/recovery/Makefile            |  35 ++++++
 contrib/recovery/recover_directory.c | 152 +++++++++++++++++++++++++++
 2 files changed, 187 insertions(+)
 create mode 100644 contrib/recovery/Makefile
 create mode 100644 contrib/recovery/recover_directory.c

diff --git a/contrib/recovery/Makefile b/contrib/recovery/Makefile
new file mode 100644
index 000000000..9a9f4f2e8
--- /dev/null
+++ b/contrib/recovery/Makefile
@@ -0,0 +1,35 @@
+# ################################################################
+# Copyright (c) 2019-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+.PHONY: all
+all: recover_directory
+
+ZSTDLIBDIR ?= ../../lib
+PROGRAMDIR ?= ../../programs
+
+CFLAGS     ?= -O3
+CFLAGS     += -I$(ZSTDLIBDIR) -I$(PROGRAMDIR)
+CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+              -Wstrict-aliasing=1 -Wswitch-enum                               \
+              -Wstrict-prototypes -Wundef                                     \
+              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+              -Wredundant-decls -Wmissing-prototypes
+CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+
+.PHONY: $(ZSTDLIBDIR)/libzstd.a
+$(ZSTDLIBDIR)/libzstd.a:
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
+
+recover_directory: recover_directory.c $(ZSTDLIBDIR)/libzstd.a $(PROGRAMDIR)/util.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
+.PHONY: clean
+clean:
+	rm -f recover_directory
diff --git a/contrib/recovery/recover_directory.c b/contrib/recovery/recover_directory.c
new file mode 100644
index 000000000..13f83fd10
--- /dev/null
+++ b/contrib/recovery/recover_directory.c
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "util.h"
+#include "zstd.h"
+
+#define CHECK(cond, ...)                                                       \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      fprintf(stderr, "%s:%d CHECK(%s) failed: ", __FILE__, __LINE__, #cond);  \
+      fprintf(stderr, "" __VA_ARGS__);                                         \
+      fprintf(stderr, "\n");                                                   \
+      exit(1);                                                                 \
+    }                                                                          \
+  } while (0)
+
+static void usage(char const *program) {
+  fprintf(stderr, "USAGE: %s FILE.zst PREFIX\n", program);
+  fprintf(stderr, "FILE.zst: A zstd compressed file with multiple frames\n");
+  fprintf(stderr, "PREFIX:   The output prefix. Uncompressed files will be "
+                  "created named ${PREFIX}0 ${PREFIX}1...\n\n");
+  fprintf(stderr, "This program takes concatenated zstd frames and "
+                  "decompresses them into individual files.\n");
+  fprintf(stderr, "E.g. files created with a command like: zstd -r directory "
+                  "-o file.zst\n");
+}
+
+typedef struct {
+  char *data;
+  size_t size;
+  size_t frames;
+  size_t maxFrameSize;
+} ZstdFrames;
+
+static ZstdFrames readFile(char const *fileName) {
+  U64 const fileSize = UTIL_getFileSize(fileName);
+  CHECK(fileSize != UTIL_FILESIZE_UNKNOWN, "Unknown file size!");
+
+  char *const data = (char *)malloc(fileSize);
+  CHECK(data != NULL, "Allocation failed");
+
+  FILE *file = fopen(fileName, "rb");
+  CHECK(file != NULL, "fopen failed");
+
+  size_t const readSize = fread(data, 1, fileSize, file);
+  CHECK(readSize == fileSize, "fread failed");
+
+  fclose(file);
+  ZstdFrames frames;
+  frames.data = (char *)data;
+  frames.size = fileSize;
+  frames.frames = 0;
+
+  size_t index;
+  size_t maxFrameSize = 0;
+  for (index = 0; index < fileSize;) {
+    size_t const frameSize =
+        ZSTD_findFrameCompressedSize(data + index, fileSize - index);
+    CHECK(!ZSTD_isError(frameSize), "Bad zstd frame: %s",
+          ZSTD_getErrorName(frameSize));
+    if (frameSize > maxFrameSize)
+      maxFrameSize = frameSize;
+    frames.frames += 1;
+    index += frameSize;
+  }
+  CHECK(index == fileSize, "Zstd file corrupt!");
+  frames.maxFrameSize = maxFrameSize;
+
+  return frames;
+}
+
+static int computePadding(size_t numFrames) {
+  return snprintf(NULL, 0, "%u", (unsigned)numFrames);
+}
+
+int main(int argc, char **argv) {
+  if (argc != 3) {
+    usage(argv[0]);
+    exit(1);
+  }
+  char const *const zstdFile = argv[1];
+  char const *const prefix = argv[2];
+
+  ZstdFrames frames = readFile(zstdFile);
+
+  if (frames.frames <= 1) {
+    fprintf(
+        stderr,
+        "%s only has %u zstd frame. Simply use `zstd -d` to decompress it.\n",
+        zstdFile, (unsigned)frames.frames);
+    exit(1);
+  }
+
+  int const padding = computePadding(frames.frames - 1);
+
+  size_t const outFileNameSize = strlen(prefix) + padding + 1;
+  char* outFileName = malloc(outFileNameSize);
+  CHECK(outFileName != NULL, "Allocation failure");
+
+  size_t const bufferSize = 128 * 1024;
+  void *buffer = malloc(bufferSize);
+  CHECK(buffer != NULL, "Allocation failure");
+
+  ZSTD_DCtx* dctx = ZSTD_createDCtx();
+  CHECK(dctx != NULL, "Allocation failure");
+
+  fprintf(stderr, "Recovering %u files...\n", (unsigned)frames.frames);
+
+  size_t index;
+  size_t frame = 0;
+  for (index = 0; index < frames.size; ++frame) {
+    size_t const frameSize =
+        ZSTD_findFrameCompressedSize(frames.data + index, frames.size - index);
+
+    int const ret = snprintf(outFileName, outFileNameSize, "%s%0*u", prefix, padding, (unsigned)frame);
+    CHECK(ret >= 0 && (size_t)ret <= outFileNameSize, "snprintf failed!");
+
+    FILE* outFile = fopen(outFileName, "wb");
+    CHECK(outFile != NULL, "fopen failed");
+
+    ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
+    ZSTD_inBuffer in = {frames.data + index, frameSize, 0};
+    while (in.pos < in.size) {
+        ZSTD_outBuffer out = {buffer, bufferSize, 0};
+        CHECK(!ZSTD_isError(ZSTD_decompressStream(dctx, &out, &in)), "decompression failed");
+        size_t const writeSize = fwrite(out.dst, 1, out.pos, outFile);
+        CHECK(writeSize == out.pos, "fwrite failed");
+    }
+    fclose(outFile);
+    fprintf(stderr, "Recovered %s\n", outFileName);
+    index += frameSize;
+  }
+  fprintf(stderr, "Complete\n");
+
+  free(outFileName);
+  ZSTD_freeDCtx(dctx);
+  free(buffer);
+  free(frames.data);
+  return 0;
+}

From 0bad3e5c0f96b77d320b2174acfc93c224e1b10c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 18 Jan 2021 11:33:03 -0800
Subject: [PATCH 46/80] parallel make build on linux

fix #2474
---
 lib/Makefile      | 2 +-
 programs/Makefile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index a3b2ee740..9a3cc0096 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -208,7 +208,7 @@ else
 endif
 
 SET_CACHE_DIRECTORY = \
-	$(MAKE) --no-print-directory $@ \
+   +$(MAKE) --no-print-directory $@ \
     BUILD_DIR=obj/$(HASH_DIR) \
     CPPFLAGS="$(CPPFLAGS)" \
     CFLAGS="$(CFLAGS)" \
diff --git a/programs/Makefile b/programs/Makefile
index 3eb658f42..b254549bc 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -194,7 +194,7 @@ endif
 endif
 
 SET_CACHE_DIRECTORY = \
-	$(MAKE) --no-print-directory $@ \
+   +$(MAKE) --no-print-directory $@ \
     BUILD_DIR=obj/$(HASH_DIR) \
     CPPFLAGS="$(CPPFLAGS)" \
     CFLAGS="$(CFLAGS)" \

From 1e65711ca5b2be520a6cb0bbee82366bb7d55a02 Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Fri, 8 Jan 2021 00:54:06 -0800
Subject: [PATCH 47/80] a couple performance improvement changes for ldm

---
 lib/compress/zstd_compress.c   |  6 +--
 lib/compress/zstd_ldm.c        | 73 +++++++++++++++++-----------------
 lib/compress/zstd_ldm.h        |  2 +-
 lib/compress/zstdmt_compress.c |  6 +--
 4 files changed, 43 insertions(+), 44 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9e704a4b2..c90fb320e 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1723,11 +1723,11 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         /* ldm bucketOffsets table */
         if (params.ldmParams.enableLdm) {
             /* TODO: avoid memset? */
-            size_t const ldmBucketSize =
+            size_t const numBuckets =
                   ((size_t)1) << (params.ldmParams.hashLog -
                                   params.ldmParams.bucketSizeLog);
-            zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize);
-            ZSTD_memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize);
+            zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets);
+            ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets);
         }
 
         /* sequences storage */
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index b7c901ba1..bbd3b4ea1 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -17,7 +17,6 @@
 #define LDM_BUCKET_SIZE_LOG 3
 #define LDM_MIN_MATCH_LENGTH 64
 #define LDM_HASH_RLOG 7
-#define LDM_HASH_CHAR_OFFSET 10
 
 void ZSTD_ldm_adjustParameters(ldmParams_t* params,
                                ZSTD_compressionParameters const* cParams)
@@ -57,11 +56,11 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
 /** ZSTD_ldm_getSmallHash() :
  *  numBits should be <= 32
  *  If numBits==0, returns 0.
- *  @return : the most significant numBits of value. */
-static U32 ZSTD_ldm_getSmallHash(U64 value, U32 numBits)
+ *  @return : the most significant numBits of hash. */
+static U32 ZSTD_ldm_getSmallHash(U64 hash, U32 numBits)
 {
     assert(numBits <= 32);
-    return numBits == 0 ? 0 : (U32)(value >> (64 - numBits));
+    return numBits == 0 ? 0 : (U32)(hash >> (64 - numBits));
 }
 
 /** ZSTD_ldm_getChecksum() :
@@ -70,22 +69,19 @@ static U32 ZSTD_ldm_getSmallHash(U64 value, U32 numBits)
 static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard)
 {
     assert(numBitsToDiscard <= 32);
-    return (hash >> (64 - 32 - numBitsToDiscard)) & 0xFFFFFFFF;
+    return (hash >> (64 - (32 + numBitsToDiscard))) & 0xFFFFFFFF;
 }
 
-/** ZSTD_ldm_getTag() ;
- *  Given the hash, returns the most significant numTagBits bits
- *  after (32 + hbits) bits.
- *
- *  If there are not enough bits remaining, return the last
- *  numTagBits bits. */
-static U32 ZSTD_ldm_getTag(U64 hash, U32 hbits, U32 numTagBits)
+/** ZSTD_ldm_getTagMask() :
+ *  Returns the mask against which the rolling hash must be
+ *  checked. */
+static U64 ZSTD_ldm_getTagMask(U32 hbits, U32 hashRateLog)
 {
     assert(numTagBits < 32 && hbits <= 32);
-    if (32 - hbits < numTagBits) {
-        return hash & (((U32)1 << numTagBits) - 1);
+    if (32 - hbits < hashRateLog) {
+        return (((U64)1 << hashRateLog) - 1);
     } else {
-        return (hash >> (32 - hbits - numTagBits)) & (((U32)1 << numTagBits) - 1);
+        return (((U64)1 << hashRateLog) - 1) << (32 - hbits - hashRateLog);
     }
 }
 
@@ -103,10 +99,12 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
                                  size_t const hash, const ldmEntry_t entry,
                                  ldmParams_t const ldmParams)
 {
-    BYTE* const bucketOffsets = ldmState->bucketOffsets;
-    *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry;
-    bucketOffsets[hash]++;
-    bucketOffsets[hash] &= ((U32)1 << ldmParams.bucketSizeLog) - 1;
+    BYTE* const pOffset = ldmState->bucketOffsets + hash;
+    unsigned const offset = *pOffset;
+
+    *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
+    *pOffset = (offset + 1) & (((U32)1 << ldmParams.bucketSizeLog) - 1);
+
 }
 
 /** ZSTD_ldm_makeEntryAndInsertByTag() :
@@ -125,9 +123,8 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
                                              U32 const offset,
                                              ldmParams_t const ldmParams)
 {
-    U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
-    U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
-    if (tag == tagMask) {
+    U64 const tagMask = ZSTD_ldm_getTagMask(hBits, ldmParams.hashRateLog);
+    if ((rollingHash & tagMask) == tagMask) {
         U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
         U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
         ldmEntry_t entry;
@@ -276,9 +273,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
     U32 const minMatchLength = params->minMatchLength;
     U64 const hashPower = ldmState->hashPower;
     U32 const hBits = params->hashLog - params->bucketSizeLog;
-    U32 const ldmBucketSize = 1U << params->bucketSizeLog;
-    U32 const hashRateLog = params->hashRateLog;
-    U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
+    U32 const entsPerBucket = 1U << params->bucketSizeLog;
+    U64 const tagMask = ZSTD_ldm_getTagMask(hBits, params->hashRateLog);
     /* Prefix and extDict parameters */
     U32 const dictLimit = ldmState->window.dictLimit;
     U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
@@ -299,10 +295,12 @@ static size_t ZSTD_ldm_generateSequences_internal(
     U64 rollingHash = 0;
 
     while (ip <= ilimit) {
+        U32 hash, checksum;
         size_t mLength;
         U32 const curr = (U32)(ip - base);
         size_t forwardMatchLength = 0, backwardMatchLength = 0;
-        ldmEntry_t* bestEntry = NULL;
+        ldmEntry_t const* bestEntry = NULL;
+
         if (ip != istart) {
             rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
                                                   lastHashed[minMatchLength],
@@ -313,22 +311,21 @@ static size_t ZSTD_ldm_generateSequences_internal(
         lastHashed = ip;
 
         /* Do not insert and do not look for a match */
-        if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
+        if ((rollingHash & tagMask) != tagMask) {
            ip++;
            continue;
         }
 
+        hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
+        checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
+
         /* Get the best entry and compute the match lengths */
         {
-            ldmEntry_t* const bucket =
-                ZSTD_ldm_getBucket(ldmState,
-                                   ZSTD_ldm_getSmallHash(rollingHash, hBits),
-                                   *params);
-            ldmEntry_t* cur;
+            ldmEntry_t* const bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
+            ldmEntry_t const* cur;
             size_t bestMatchLength = 0;
-            U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
 
-            for (cur = bucket; cur < bucket + ldmBucketSize; ++cur) {
+            for (cur = bucket; cur < bucket + entsPerBucket; ++cur) {
                 size_t curForwardMatchLength, curBackwardMatchLength,
                        curTotalMatchLength;
                 if (cur->checksum != checksum || cur->offset <= lowestIndex) {
@@ -379,9 +376,11 @@ static size_t ZSTD_ldm_generateSequences_internal(
 
         /* No match found -- continue searching */
         if (bestEntry == NULL) {
-            ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash,
-                                             hBits, curr,
-                                             *params);
+            ldmEntry_t entry;
+
+            entry.offset = curr;
+            entry.checksum = checksum;
+            ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
             ip++;
             continue;
         }
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 7102e4227..e9137f063 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -73,7 +73,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
  *
  * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`.
  * Avoids emitting matches less than `minMatch` bytes.
- * Must be called for data with is not passed to ZSTD_ldm_blockCompress().
+ * Must be called for data that is not passed to ZSTD_ldm_blockCompress().
  */
 void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
     U32 const minMatch);
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 1a642a456..c3ead0878 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -486,10 +486,10 @@ ZSTDMT_serialState_reset(serialState_t* serialState,
         size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t);
         unsigned const bucketLog =
             params.ldmParams.hashLog - params.ldmParams.bucketSizeLog;
-        size_t const bucketSize = (size_t)1 << bucketLog;
         unsigned const prevBucketLog =
             serialState->params.ldmParams.hashLog -
             serialState->params.ldmParams.bucketSizeLog;
+        size_t const numBuckets = (size_t)1 << bucketLog;
         /* Size the seq pool tables */
         ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
         /* Reset the window */
@@ -501,13 +501,13 @@ ZSTDMT_serialState_reset(serialState_t* serialState,
         }
         if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) {
             ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem);
-            serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(bucketSize, cMem);
+            serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem);
         }
         if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets)
             return 1;
         /* Zero the tables */
         ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize);
-        ZSTD_memset(serialState->ldmState.bucketOffsets, 0, bucketSize);
+        ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets);
 
         /* Update window state and fill hash table with dict */
         serialState->ldmState.loadedDictEnd = 0;

From e0d5eca8fae75c44fe54f343a4b7c3ceada2e2f0 Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Mon, 11 Jan 2021 02:41:17 -0800
Subject: [PATCH 48/80] fix forgotten numTagBits in getTagMask

---
 lib/compress/zstd_ldm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index bbd3b4ea1..8a42f0a91 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -77,7 +77,7 @@ static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard)
  *  checked. */
 static U64 ZSTD_ldm_getTagMask(U32 hbits, U32 hashRateLog)
 {
-    assert(numTagBits < 32 && hbits <= 32);
+    assert(hashRateLog < 32 && hbits <= 32);
     if (32 - hbits < hashRateLog) {
         return (((U64)1 << hashRateLog) - 1);
     } else {

From d6e3de77dce0e061a782fda0ec55e48267792d69 Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Wed, 20 Jan 2021 01:05:29 -0800
Subject: [PATCH 49/80] fix warning and remove one more occurrence of
 makeEntryAndInsertByTag

---
 lib/compress/zstd_ldm.c | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 8a42f0a91..36d63d052 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -103,7 +103,7 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
     unsigned const offset = *pOffset;
 
     *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
-    *pOffset = (offset + 1) & (((U32)1 << ldmParams.bucketSizeLog) - 1);
+    *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
 
 }
 
@@ -295,11 +295,12 @@ static size_t ZSTD_ldm_generateSequences_internal(
     U64 rollingHash = 0;
 
     while (ip <= ilimit) {
+        U32 const currentOffset = (U32)(ip - base);
         U32 hash, checksum;
         size_t mLength;
-        U32 const curr = (U32)(ip - base);
         size_t forwardMatchLength = 0, backwardMatchLength = 0;
         ldmEntry_t const* bestEntry = NULL;
+        ldmEntry_t newEntry;
 
         if (ip != istart) {
             rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
@@ -319,6 +320,9 @@ static size_t ZSTD_ldm_generateSequences_internal(
         hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
         checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
 
+        newEntry.offset = currentOffset;
+        newEntry.checksum = checksum;
+
         /* Get the best entry and compute the match lengths */
         {
             ldmEntry_t* const bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
@@ -376,11 +380,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
 
         /* No match found -- continue searching */
         if (bestEntry == NULL) {
-            ldmEntry_t entry;
-
-            entry.offset = curr;
-            entry.checksum = checksum;
-            ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
+            ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
             ip++;
             continue;
         }
@@ -391,11 +391,11 @@ static size_t ZSTD_ldm_generateSequences_internal(
 
         {
             /* Store the sequence:
-             * ip = curr - backwardMatchLength
+             * ip = currentOffset - backwardMatchLength
              * The match is at (bestEntry->offset - backwardMatchLength)
              */
             U32 const matchIndex = bestEntry->offset;
-            U32 const offset = curr - matchIndex;
+            U32 const offset = currentOffset - matchIndex;
             rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
 
             /* Out of sequence storage */
@@ -407,10 +407,9 @@ static size_t ZSTD_ldm_generateSequences_internal(
             rawSeqStore->size++;
         }
 
-        /* Insert the current entry into the hash table */
-        ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits,
-                                         (U32)(lastHashed - base),
-                                         *params);
+        /* Insert the current entry into the hash table --- it must be
+         * done after the previous block to avoid clobbering bestEntry */
+        ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
 
         assert(ip + backwardMatchLength == lastHashed);
 

From aee3dc877fa48b9df28331d46b64922886204818 Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Fri, 22 Jan 2021 02:24:19 -0800
Subject: [PATCH 50/80] fix a variable name to reflect its nature

---
 lib/compress/zstd_ldm.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 36d63d052..c38e7753b 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -295,7 +295,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
     U64 rollingHash = 0;
 
     while (ip <= ilimit) {
-        U32 const currentOffset = (U32)(ip - base);
+        U32 const currentIndex = (U32)(ip - base);
         U32 hash, checksum;
         size_t mLength;
         size_t forwardMatchLength = 0, backwardMatchLength = 0;
@@ -320,7 +320,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
         hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
         checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
 
-        newEntry.offset = currentOffset;
+        newEntry.offset = currentIndex;
         newEntry.checksum = checksum;
 
         /* Get the best entry and compute the match lengths */
@@ -391,11 +391,11 @@ static size_t ZSTD_ldm_generateSequences_internal(
 
         {
             /* Store the sequence:
-             * ip = currentOffset - backwardMatchLength
+             * ip = currentIndex - backwardMatchLength
              * The match is at (bestEntry->offset - backwardMatchLength)
              */
             U32 const matchIndex = bestEntry->offset;
-            U32 const offset = currentOffset - matchIndex;
+            U32 const offset = currentIndex - matchIndex;
             rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
 
             /* Out of sequence storage */

From 9f327c02fd17a5aad2b24ae06b85d8226add1f93 Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Mon, 11 Jan 2021 08:16:51 -0800
Subject: [PATCH 51/80] new core ldm algorithm

---
 lib/compress/zstd_ldm.c         | 403 +++++++++++++++++---------------
 lib/compress/zstd_ldm_geartab.h | 103 ++++++++
 2 files changed, 320 insertions(+), 186 deletions(-)
 create mode 100644 lib/compress/zstd_ldm_geartab.h

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index c38e7753b..cd6daa0a0 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -11,12 +11,100 @@
 #include "zstd_ldm.h"
 
 #include "../common/debug.h"
+#include "../common/xxhash.h"
 #include "zstd_fast.h"          /* ZSTD_fillHashTable() */
 #include "zstd_double_fast.h"   /* ZSTD_fillDoubleHashTable() */
+#include "zstd_ldm_geartab.h"
 
 #define LDM_BUCKET_SIZE_LOG 3
 #define LDM_MIN_MATCH_LENGTH 64
 #define LDM_HASH_RLOG 7
+#define LDM_LOOKAHEAD_SPLITS 64
+
+typedef struct {
+    U64 rolling;
+    U64 stopMask;
+} ldmRollingHashState_t;
+
+/** ZSTD_ldm_gear_init():
+ *
+ * Initializes the rolling hash state such that it will honor the
+ * settings in params. */
+static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params)
+{
+    unsigned maxBitsInMask = MIN(params->minMatchLength, 64);
+    unsigned hashRateLog = params->hashRateLog;
+
+    state->rolling = ~(U32)0;
+
+    /* The choice of the splitting criterion is subject to two conditions:
+     *   1. it has to trigger on average every 2^(hashRateLog) bytes;
+     *   2. ideally, it has to depend on a window of minMatchLength bytes.
+     *
+     * In the gear hash algorithm, bit n depends on the last n bytes;
+     * so in order to obtain a good quality splitting criterion it is
+     * preferable to use bits with high weight.
+     *
+     * To match condition 1 we use a mask with hashRateLog bits set
+     * and, because of the previous remark, we make sure these bits
+     * have the highest possible weight while still respecting
+     * condition 2.
+     */
+    if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) {
+        state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog);
+    } else {
+        /* In this degenerate case we simply honor the hash rate. */
+        state->stopMask = ((U64)1 << hashRateLog) - 1;
+    }
+}
+
+/** ZSTD_ldm_gear_feed():
+ *
+ * Registers in the splits array all the split points found in the first
+ * size bytes following the data pointer. This function terminates when
+ * either all the data has been processed or LDM_LOOKAHEAD_SPLITS splits
+ * are present in the splits array.
+ *
+ * Precondition: The splits array must not be full.
+ * Returns: The number of bytes processed. */
+static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state,
+                                 BYTE const* data, size_t size,
+                                 size_t* splits, unsigned* numSplits)
+{
+    size_t n;
+    U64 hash, mask;
+
+    hash = state->rolling;
+    mask = state->stopMask;
+    n = 0;
+
+#define GEAR_ITER_ONCE() do { \
+        hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
+        n += 1; \
+        if (UNLIKELY((hash & mask) == 0)) { \
+            splits[*numSplits] = n; \
+            *numSplits += 1; \
+            if (*numSplits == LDM_LOOKAHEAD_SPLITS) \
+                goto done; \
+        } \
+    } while (0)
+
+    while (n + 3 < size) {
+        GEAR_ITER_ONCE();
+        GEAR_ITER_ONCE();
+        GEAR_ITER_ONCE();
+        GEAR_ITER_ONCE();
+    }
+    while (n < size) {
+        GEAR_ITER_ONCE();
+    }
+
+#undef GEAR_ITER_ONCE
+
+done:
+    state->rolling = hash;
+    return n;
+}
 
 void ZSTD_ldm_adjustParameters(ldmParams_t* params,
                                ZSTD_compressionParameters const* cParams)
@@ -53,38 +141,6 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
     return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0;
 }
 
-/** ZSTD_ldm_getSmallHash() :
- *  numBits should be <= 32
- *  If numBits==0, returns 0.
- *  @return : the most significant numBits of hash. */
-static U32 ZSTD_ldm_getSmallHash(U64 hash, U32 numBits)
-{
-    assert(numBits <= 32);
-    return numBits == 0 ? 0 : (U32)(hash >> (64 - numBits));
-}
-
-/** ZSTD_ldm_getChecksum() :
- *  numBitsToDiscard should be <= 32
- *  @return : the next most significant 32 bits after numBitsToDiscard */
-static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard)
-{
-    assert(numBitsToDiscard <= 32);
-    return (hash >> (64 - (32 + numBitsToDiscard))) & 0xFFFFFFFF;
-}
-
-/** ZSTD_ldm_getTagMask() :
- *  Returns the mask against which the rolling hash must be
- *  checked. */
-static U64 ZSTD_ldm_getTagMask(U32 hbits, U32 hashRateLog)
-{
-    assert(hashRateLog < 32 && hbits <= 32);
-    if (32 - hbits < hashRateLog) {
-        return (((U64)1 << hashRateLog) - 1);
-    } else {
-        return (((U64)1 << hashRateLog) - 1) << (32 - hbits - hashRateLog);
-    }
-}
-
 /** ZSTD_ldm_getBucket() :
  *  Returns a pointer to the start of the bucket associated with hash. */
 static ldmEntry_t* ZSTD_ldm_getBucket(
@@ -107,33 +163,6 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
 
 }
 
-/** ZSTD_ldm_makeEntryAndInsertByTag() :
- *
- *  Gets the small hash, checksum, and tag from the rollingHash.
- *
- *  If the tag matches (1 << ldmParams.hashRateLog)-1, then
- *  creates an ldmEntry from the offset, and inserts it into the hash table.
- *
- *  hBits is the length of the small hash, which is the most significant hBits
- *  of rollingHash. The checksum is the next 32 most significant bits, followed
- *  by ldmParams.hashRateLog bits that make up the tag. */
-static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
-                                             U64 const rollingHash,
-                                             U32 const hBits,
-                                             U32 const offset,
-                                             ldmParams_t const ldmParams)
-{
-    U64 const tagMask = ZSTD_ldm_getTagMask(hBits, ldmParams.hashRateLog);
-    if ((rollingHash & tagMask) == tagMask) {
-        U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
-        U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
-        ldmEntry_t entry;
-        entry.offset = offset;
-        entry.checksum = checksum;
-        ZSTD_ldm_insertEntry(ldmState, hash, entry, ldmParams);
-    }
-}
-
 /** ZSTD_ldm_countBackwardsMatch() :
  *  Returns the number of bytes that match backwards before pIn and pMatch.
  *
@@ -209,43 +238,42 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
     return 0;
 }
 
-/** ZSTD_ldm_fillLdmHashTable() :
- *
- *  Fills hashTable from (lastHashed + 1) to iend (non-inclusive).
- *  lastHash is the rolling hash that corresponds to lastHashed.
- *
- *  Returns the rolling hash corresponding to position iend-1. */
-static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
-                                     U64 lastHash, const BYTE* lastHashed,
-                                     const BYTE* iend, const BYTE* base,
-                                     U32 hBits, ldmParams_t const ldmParams)
-{
-    U64 rollingHash = lastHash;
-    const BYTE* cur = lastHashed + 1;
-
-    while (cur < iend) {
-        rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1],
-                                              cur[ldmParams.minMatchLength-1],
-                                              state->hashPower);
-        ZSTD_ldm_makeEntryAndInsertByTag(state,
-                                         rollingHash, hBits,
-                                         (U32)(cur - base), ldmParams);
-        ++cur;
-    }
-    return rollingHash;
-}
-
 void ZSTD_ldm_fillHashTable(
-            ldmState_t* state, const BYTE* ip,
+            ldmState_t* ldmState, const BYTE* ip,
             const BYTE* iend, ldmParams_t const* params)
 {
+    U32 const minMatchLength = params->minMatchLength;
+    U32 const hBits = params->hashLog - params->bucketSizeLog;
+    BYTE const* const base = ldmState->window.base;
+    BYTE const* const istart = ip;
+    ldmRollingHashState_t hashState;
+    size_t splits[LDM_LOOKAHEAD_SPLITS];
+    unsigned numSplits;
+
     DEBUGLOG(5, "ZSTD_ldm_fillHashTable");
-    if ((size_t)(iend - ip) >= params->minMatchLength) {
-        U64 startingHash = ZSTD_rollingHash_compute(ip, params->minMatchLength);
-        ZSTD_ldm_fillLdmHashTable(
-            state, startingHash, ip, iend - params->minMatchLength, state->window.base,
-            params->hashLog - params->bucketSizeLog,
-            *params);
+
+    ZSTD_ldm_gear_init(&hashState, params);
+    while (ip < iend) {
+        size_t hashed;
+        unsigned n;
+        
+        numSplits = 0;
+        hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
+
+        for (n = 0; n < numSplits; n++) {
+            if (ip + splits[n] >= istart + minMatchLength) {
+                BYTE const* const split = ip + splits[n] - minMatchLength;
+                U64 const xxhash = XXH64(split, minMatchLength, 0);
+                U64 const hash = xxhash & (((U32)1 << hBits) - 1);
+                ldmEntry_t entry;
+
+                entry.offset = (U32)(split - base);
+                entry.checksum = (U32)(xxhash >> 32);
+                ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
+            }
+        }
+
+        ip += hashed;
     }
 }
 
@@ -271,10 +299,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
     /* LDM parameters */
     int const extDict = ZSTD_window_hasExtDict(ldmState->window);
     U32 const minMatchLength = params->minMatchLength;
-    U64 const hashPower = ldmState->hashPower;
-    U32 const hBits = params->hashLog - params->bucketSizeLog;
     U32 const entsPerBucket = 1U << params->bucketSizeLog;
-    U64 const tagMask = ZSTD_ldm_getTagMask(hBits, params->hashRateLog);
+    U32 const hBits = params->hashLog - params->bucketSizeLog;
     /* Prefix and extDict parameters */
     U32 const dictLimit = ldmState->window.dictLimit;
     U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
@@ -286,50 +312,78 @@ static size_t ZSTD_ldm_generateSequences_internal(
     /* Input bounds */
     BYTE const* const istart = (BYTE const*)src;
     BYTE const* const iend = istart + srcSize;
-    BYTE const* const ilimit = iend - MAX(minMatchLength, HASH_READ_SIZE);
+    BYTE const* const ilimit = iend - HASH_READ_SIZE;
     /* Input positions */
     BYTE const* anchor = istart;
     BYTE const* ip = istart;
-    /* Rolling hash */
-    BYTE const* lastHashed = NULL;
-    U64 rollingHash = 0;
+    /* Rolling hash state */
+    ldmRollingHashState_t hashState;
+    /* Arrays for staged-processing */
+    size_t splits[LDM_LOOKAHEAD_SPLITS];
+    struct {
+        BYTE const* split;
+        U32 hash;
+        U32 checksum;
+        ldmEntry_t* bucket;
+    } candidates[LDM_LOOKAHEAD_SPLITS];
+    unsigned numSplits;
 
-    while (ip <= ilimit) {
-        U32 const currentIndex = (U32)(ip - base);
-        U32 hash, checksum;
-        size_t mLength;
-        size_t forwardMatchLength = 0, backwardMatchLength = 0;
-        ldmEntry_t const* bestEntry = NULL;
-        ldmEntry_t newEntry;
+    /* Initialize the rolling hash state with the first minMatchLength bytes */
+    ZSTD_ldm_gear_init(&hashState, params);
+    {
+        size_t n = 0;
 
-        if (ip != istart) {
-            rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
-                                                  lastHashed[minMatchLength],
-                                                  hashPower);
-        } else {
-            rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength);
+        while (n < minMatchLength) {
+            numSplits = 0;
+            n += ZSTD_ldm_gear_feed(&hashState, ip + n, minMatchLength - n,
+                                    splits, &numSplits);
         }
-        lastHashed = ip;
+        ip += minMatchLength;
+    }
 
-        /* Do not insert and do not look for a match */
-        if ((rollingHash & tagMask) != tagMask) {
-           ip++;
-           continue;
+    while (ip < ilimit) {
+        size_t hashed;
+        unsigned n;
+
+        numSplits = 0;
+        hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip,
+                                    splits, &numSplits);
+
+        for (n = 0; n < numSplits; n++) {
+            BYTE const* const split = ip + splits[n] - minMatchLength;
+            U64 const xxhash = XXH64(split, minMatchLength, 0);
+            U64 const hash = xxhash & (((U32)1 << hBits) - 1);
+
+            candidates[n].split = split;
+            candidates[n].hash = hash;
+            candidates[n].checksum = (U32)(xxhash >> 32);
+            candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
+            PREFETCH_L1(candidates[n].bucket);
         }
 
-        hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
-        checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
-
-        newEntry.offset = currentIndex;
-        newEntry.checksum = checksum;
-
-        /* Get the best entry and compute the match lengths */
-        {
-            ldmEntry_t* const bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
+        for (n = 0; n < numSplits; n++) {
+            size_t forwardMatchLength = 0, backwardMatchLength = 0,
+                   bestMatchLength = 0, mLength;
+            BYTE const* const split = candidates[n].split;
+            U64 const checksum = candidates[n].checksum;
+            U64 const hash = candidates[n].hash;
+            ldmEntry_t* const bucket = candidates[n].bucket;
             ldmEntry_t const* cur;
-            size_t bestMatchLength = 0;
+            ldmEntry_t const* bestEntry = NULL;
+            ldmEntry_t newEntry;
 
-            for (cur = bucket; cur < bucket + entsPerBucket; ++cur) {
+            newEntry.offset = (U32)(split - base);
+            newEntry.checksum = checksum;
+
+            /* If a split point would generate a sequence overlapping with
+             * the previous one, we merely register it in the hash table and
+             * move on */
+            if (split < anchor) {
+                ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+                continue;
+            }
+
+            for (cur = bucket; cur < bucket + entsPerBucket; cur++) {
                 size_t curForwardMatchLength, curBackwardMatchLength,
                        curTotalMatchLength;
                 if (cur->checksum != checksum || cur->offset <= lowestIndex) {
@@ -343,31 +397,23 @@ static size_t ZSTD_ldm_generateSequences_internal(
                         cur->offset < dictLimit ? dictEnd : iend;
                     BYTE const* const lowMatchPtr =
                         cur->offset < dictLimit ? dictStart : lowPrefixPtr;
-
-                    curForwardMatchLength = ZSTD_count_2segments(
-                                                ip, pMatch, iend,
-                                                matchEnd, lowPrefixPtr);
+                    curForwardMatchLength =
+                        ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr);
                     if (curForwardMatchLength < minMatchLength) {
                         continue;
                     }
-                    curBackwardMatchLength =
-                        ZSTD_ldm_countBackwardsMatch_2segments(ip, anchor,
-                                                               pMatch, lowMatchPtr,
-                                                               dictStart, dictEnd);
-                    curTotalMatchLength = curForwardMatchLength +
-                                          curBackwardMatchLength;
+                    curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments(
+                            split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd);
                 } else { /* !extDict */
                     BYTE const* const pMatch = base + cur->offset;
-                    curForwardMatchLength = ZSTD_count(ip, pMatch, iend);
+                    curForwardMatchLength = ZSTD_count(split, pMatch, iend);
                     if (curForwardMatchLength < minMatchLength) {
                         continue;
                     }
                     curBackwardMatchLength =
-                        ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch,
-                                                     lowPrefixPtr);
-                    curTotalMatchLength = curForwardMatchLength +
-                                          curBackwardMatchLength;
+                        ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr);
                 }
+                curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength;
 
                 if (curTotalMatchLength > bestMatchLength) {
                     bestMatchLength = curTotalMatchLength;
@@ -376,54 +422,39 @@ static size_t ZSTD_ldm_generateSequences_internal(
                     bestEntry = cur;
                 }
             }
-        }
 
-        /* No match found -- continue searching */
-        if (bestEntry == NULL) {
+            /* No match found -- insert an entry into the hash table
+             * and process the next candidate match */
+            if (bestEntry == NULL) {
+                ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+                continue;
+            }
+
+            /* Match found */
+            mLength = forwardMatchLength + backwardMatchLength;
+            {
+                U32 const offset = (U32)(split - base) - bestEntry->offset;
+                rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
+
+                /* Out of sequence storage */
+                if (rawSeqStore->size == rawSeqStore->capacity)
+                    return ERROR(dstSize_tooSmall);
+                seq->litLength = (U32)(split - backwardMatchLength - anchor);
+                seq->matchLength = (U32)mLength;
+                seq->offset = offset;
+                rawSeqStore->size++;
+            }
+
+            /* Insert the current entry into the hash table --- it must be
+             * done after the previous block to avoid clobbering bestEntry */
             ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
-            ip++;
-            continue;
+
+            anchor = split + forwardMatchLength;
         }
 
-        /* Match found */
-        mLength = forwardMatchLength + backwardMatchLength;
-        ip -= backwardMatchLength;
-
-        {
-            /* Store the sequence:
-             * ip = currentIndex - backwardMatchLength
-             * The match is at (bestEntry->offset - backwardMatchLength)
-             */
-            U32 const matchIndex = bestEntry->offset;
-            U32 const offset = currentIndex - matchIndex;
-            rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
-
-            /* Out of sequence storage */
-            if (rawSeqStore->size == rawSeqStore->capacity)
-                return ERROR(dstSize_tooSmall);
-            seq->litLength = (U32)(ip - anchor);
-            seq->matchLength = (U32)mLength;
-            seq->offset = offset;
-            rawSeqStore->size++;
-        }
-
-        /* Insert the current entry into the hash table --- it must be
-         * done after the previous block to avoid clobbering bestEntry */
-        ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
-
-        assert(ip + backwardMatchLength == lastHashed);
-
-        /* Fill the hash table from lastHashed+1 to ip+mLength*/
-        /* Heuristic: don't need to fill the entire table at end of block */
-        if (ip + mLength <= ilimit) {
-            rollingHash = ZSTD_ldm_fillLdmHashTable(
-                              ldmState, rollingHash, lastHashed,
-                              ip + mLength, base, hBits, *params);
-            lastHashed = ip + mLength - 1;
-        }
-        ip += mLength;
-        anchor = ip;
+        ip += hashed;
     }
+
     return iend - anchor;
 }
 
diff --git a/lib/compress/zstd_ldm_geartab.h b/lib/compress/zstd_ldm_geartab.h
new file mode 100644
index 000000000..d24c1f6d3
--- /dev/null
+++ b/lib/compress/zstd_ldm_geartab.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_LDM_GEARTAB_H
+#define ZSTD_LDM_GEARTAB_H
+
+static U64 ZSTD_ldm_gearTab[256] = {
+    0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc,
+    0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05,
+    0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e,
+    0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889,
+    0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e,
+    0x37b628620b628,    0x49a8d455d88caf5,  0x8556d711e6958140,
+    0x4f7ae74fc605c1f,  0x829f0c3468bd3a20, 0x4ffdc885c625179e,
+    0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f,
+    0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391,
+    0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210,
+    0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be,
+    0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a,
+    0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b,
+    0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4,
+    0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb,
+    0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312,
+    0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01,
+    0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc,
+    0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967,
+    0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553,
+    0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f,
+    0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2,
+    0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d,
+    0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a,
+    0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74,
+    0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3,
+    0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1,
+    0xff452823dbb010a,  0x9d42ed614f3dd267, 0x5b9313c06257c57b,
+    0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568,
+    0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a,
+    0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1,
+    0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9,
+    0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463,
+    0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba,
+    0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9,
+    0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61,
+    0x24a5483879c453e3, 0x88026889192b4b9,  0x28da96671782dbec,
+    0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6,
+    0xbc135a0a704b70ba, 0x69cd868f7622ada,  0xbc37ba89e0b9c0ab,
+    0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5,
+    0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59,
+    0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7,
+    0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc,
+    0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb,
+    0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be,
+    0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312,
+    0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1,
+    0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc,
+    0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d,
+    0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445,
+    0x820d471e20b348e,  0x1874383cb83d46dc, 0x97edeec7a1efe11c,
+    0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5,
+    0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5,
+    0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28,
+    0xaf846af6ab7d0bf4, 0xe5af208eb666e49,  0x5e6622f73534cd6a,
+    0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9,
+    0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15,
+    0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef,
+    0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2,
+    0x9f90e4c5fd508d8,  0xa34e5956fbaf3385, 0x2e2f8e151d3ef375,
+    0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3,
+    0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595,
+    0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389,
+    0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4,
+    0x4228e364c5b5ed7,  0x9d7a3edf0da43911, 0x8edcfeda24686756,
+    0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc,
+    0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45,
+    0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea,
+    0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f,
+    0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc,
+    0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c,
+    0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a,
+    0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17,
+    0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3,
+    0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4,
+    0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91,
+    0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40,
+    0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741,
+    0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f,
+    0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4,
+    0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad,
+    0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047,
+    0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2,
+    0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e,
+    0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b,
+    0x2b4da14f2613d8f4
+};
+
+#endif /* ZSTD_LDM_GEARTAB_H */

From 874a590e5ceb9a08beac462ae1e3d3747ccfe17d Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Thu, 4 Feb 2021 11:13:41 +0100
Subject: [PATCH 52/80] deal safely with short inputs in
 ZSTD_ldm_generateSequences

The fuzzer CI found this bug.
---
 lib/compress/zstd_ldm.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index cd6daa0a0..9292aa4d0 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -328,6 +328,9 @@ static size_t ZSTD_ldm_generateSequences_internal(
     } candidates[LDM_LOOKAHEAD_SPLITS];
     unsigned numSplits;
 
+    if (srcSize < minMatchLength)
+        return iend - anchor;
+
     /* Initialize the rolling hash state with the first minMatchLength bytes */
     ZSTD_ldm_gear_init(&hashState, params);
     {

From 134be2731aeba7d3f92d8b25fcfe6c00877ed998 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 5 Feb 2021 09:19:27 -0800
Subject: [PATCH 53/80] removed flackey gcc6 tests

from github actions.
replaced by equivalent "current" gcc tests when it makes sense
---
 .github/workflows/generic-dev.yml | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml
index 72e739f4b..1632101c7 100644
--- a/.github/workflows/generic-dev.yml
+++ b/.github/workflows/generic-dev.yml
@@ -45,15 +45,13 @@ jobs:
         APT_PACKAGES="gcc-multilib" make apt-install
         CFLAGS="-m32 -O1 -fstack-protector" make check V=1
 
-  gcc-6-7-libzstd:
+  gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: gcc-6 + gcc-7 + libzstdmt compilation
+    - name: gcc-7 + libzstdmt compilation
       run: |
-        make gcc6install gcc7install
-        CC=gcc-6 CFLAGS=-Werror make -j all
-        make clean
+        make gcc7install
         CC=gcc-7 CFLAGS=-Werror make -j all
         make clean
         LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
@@ -74,16 +72,16 @@ jobs:
         make clean
         CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
 
-  gcc-6-asan-ubsan-testzstd-32bit:
+  gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode
+    - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
-        make gcc6install libc6install
-        CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
+        make libc6install
+        CFLAGS="-Werror -m32" make -j all32
         make clean
-        CC=gcc-6 make -j uasan-test-zstd32
+        make -j uasan-test-zstd32
 
   clang-msan-testzstd:
     runs-on: ubuntu-latest
@@ -119,14 +117,14 @@ jobs:
         make gcc8install
         CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
-  gcc-6-asan-ubsan-fuzz32:
+  gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: gcc-6 + ASan + UBSan + Fuzz Test 32bit
+    - name: ASan + UBSan + Fuzz Test 32bit
       run: |
-        make gcc6install libc6install
-        CC=gcc-6 CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+        make libc6install
+        CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
   asan-ubsan-msan-regression:
     runs-on: ubuntu-latest

From b9748757b066e48bc8cd4c102079f8245bdda9b9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 5 Feb 2021 09:28:08 -0800
Subject: [PATCH 54/80] fixed minor cast warning

---
 lib/compress/zstd_compress_literals.c  |  4 ++--
 lib/decompress/zstd_decompress.c       |  8 ++++----
 lib/decompress/zstd_decompress_block.c |  6 +++---
 lib/dictBuilder/cover.c                | 20 ++++++++++----------
 lib/dictBuilder/fastcover.c            | 10 +++++-----
 lib/legacy/zstd_v05.c                  |  8 ++++----
 lib/legacy/zstd_v06.c                  |  8 ++++----
 lib/legacy/zstd_v07.c                  |  8 ++++----
 8 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index 8945fd596..1d9188d33 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -15,7 +15,7 @@
 
 size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     U32   const flSize = 1 + (srcSize>31) + (srcSize>4095);
 
     RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
@@ -42,7 +42,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
 
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     U32   const flSize = 1 + (srcSize>31) + (srcSize>4095);
 
     (void)dstCapacity;  /* dstCapacity already guaranteed to be >=4, hence large enough */
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 3701170cd..0619add3a 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -80,7 +80,7 @@
 
 #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
 #define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3   /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
-                                                     * Currently, that means a 0.75 load factor. 
+                                                     * Currently, that means a 0.75 load factor.
                                                      * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
                                                      * the load factor of the ddict hash set.
                                                      */
@@ -337,9 +337,9 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 /* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
  * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
  * accordingly sets the ddict to be used to decompress the frame.
- * 
+ *
  * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
- * 
+ *
  * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
  */
 static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
@@ -794,7 +794,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
                              const void** srcPtr, size_t *srcSizePtr)
 {
     const BYTE* ip = (const BYTE*)(*srcPtr);
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
     BYTE* op = ostart;
     size_t remainingSrcSize = *srcSizePtr;
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 774f1da71..c2ab5e740 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -577,7 +577,7 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb
 size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
                              const void* src, size_t srcSize)
 {
-    const BYTE* const istart = (const BYTE* const)src;
+    const BYTE* const istart = (const BYTE*)src;
     const BYTE* const iend = istart + srcSize;
     const BYTE* ip = istart;
     int nbSeq;
@@ -1108,7 +1108,7 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
 {
     const BYTE* ip = (const BYTE*)seqStart;
     const BYTE* const iend = ip + seqSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + maxDstSize;
     BYTE* op = ostart;
     const BYTE* litPtr = dctx->litPtr;
@@ -1242,7 +1242,7 @@ ZSTD_decompressSequencesLong_body(
 {
     const BYTE* ip = (const BYTE*)seqStart;
     const BYTE* const iend = ip + seqSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + maxDstSize;
     BYTE* op = ostart;
     const BYTE* litPtr = dctx->litPtr;
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 62eb67166..f069b6378 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -1062,18 +1062,19 @@ typedef struct COVER_tryParameters_data_s {
  * This function is thread safe if zstd is compiled with multithreaded support.
  * It takes its parameters as an *OWNING* opaque pointer to support threading.
  */
-static void COVER_tryParameters(void *opaque) {
+static void COVER_tryParameters(void *opaque)
+{
   /* Save parameters as local variables */
-  COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque;
+  COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t*)opaque;
   const COVER_ctx_t *const ctx = data->ctx;
   const ZDICT_cover_params_t parameters = data->parameters;
   size_t dictBufferCapacity = data->dictBufferCapacity;
   size_t totalCompressedSize = ERROR(GENERIC);
   /* Allocate space for hash table, dict, and freqs */
   COVER_map_t activeDmers;
-  BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+  BYTE* const dict = (BYTE*)malloc(dictBufferCapacity);
   COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
-  U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+  U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32));
   if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
     DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
     goto _cleanup;
@@ -1103,15 +1104,14 @@ _cleanup:
   free(data);
   COVER_map_destroy(&activeDmers);
   COVER_dictSelectionFree(selection);
-  if (freqs) {
-    free(freqs);
-  }
+  free(freqs);
 }
 
 ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
-    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
-    const size_t *samplesSizes, unsigned nbSamples,
-    ZDICT_cover_params_t *parameters) {
+    void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer,
+    const size_t* samplesSizes, unsigned nbSamples,
+    ZDICT_cover_params_t* parameters)
+{
   /* constants */
   const unsigned nbThreads = parameters->nbThreads;
   const double splitPoint =
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 303704ec6..5f880e4e4 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -462,20 +462,20 @@ typedef struct FASTCOVER_tryParameters_data_s {
  * This function is thread safe if zstd is compiled with multithreaded support.
  * It takes its parameters as an *OWNING* opaque pointer to support threading.
  */
-static void FASTCOVER_tryParameters(void *opaque)
+static void FASTCOVER_tryParameters(void* opaque)
 {
   /* Save parameters as local variables */
-  FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
+  FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t*)opaque;
   const FASTCOVER_ctx_t *const ctx = data->ctx;
   const ZDICT_cover_params_t parameters = data->parameters;
   size_t dictBufferCapacity = data->dictBufferCapacity;
   size_t totalCompressedSize = ERROR(GENERIC);
   /* Initialize array to keep track of frequency of dmer within activeSegment */
-  U16* segmentFreqs = (U16 *)calloc(((U64)1 << ctx->f), sizeof(U16));
+  U16* segmentFreqs = (U16*)calloc(((U64)1 << ctx->f), sizeof(U16));
   /* Allocate space for hash table, dict, and freqs */
-  BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+  BYTE *const dict = (BYTE*)malloc(dictBufferCapacity);
   COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
-  U32 *freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
+  U32* freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
   if (!segmentFreqs || !dict || !freqs) {
     DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
     goto _cleanup;
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 991991e9b..6027a7b6b 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -2833,7 +2833,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
 
 static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
 {
-    const BYTE* const in = (const BYTE* const)src;
+    const BYTE* const in = (const BYTE*)src;
     BYTE headerFlags;
     U32 cSize;
 
@@ -3002,7 +3002,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
                          FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
                          const void* src, size_t srcSize, U32 flagStaticTable)
 {
-    const BYTE* const istart = (const BYTE* const)src;
+    const BYTE* const istart = (const BYTE*)src;
     const BYTE* ip = istart;
     const BYTE* const iend = istart + srcSize;
     U32 LLtype, Offtype, MLtype;
@@ -3310,7 +3310,7 @@ static size_t ZSTDv05_decompressSequences(
 {
     const BYTE* ip = (const BYTE*)seqStart;
     const BYTE* const iend = ip + seqSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
     BYTE* const oend = ostart + maxDstSize;
     size_t errorCode, dumpsLength=0;
@@ -3423,7 +3423,7 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
 {
     const BYTE* ip = (const BYTE*)src;
     const BYTE* iend = ip + srcSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
     BYTE* const oend = ostart + maxDstSize;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index ccc2336b5..76fc838d9 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3029,7 +3029,7 @@ typedef struct
 *   Provides the size of compressed block from block header `src` */
 static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
 {
-    const BYTE* const in = (const BYTE* const)src;
+    const BYTE* const in = (const BYTE*)src;
     U32 cSize;
 
     if (srcSize < ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
@@ -3223,7 +3223,7 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
                              FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
                              const void* src, size_t srcSize)
 {
-    const BYTE* const istart = (const BYTE* const)src;
+    const BYTE* const istart = (const BYTE*)src;
     const BYTE* const iend = istart + srcSize;
     const BYTE* ip = istart;
 
@@ -3445,7 +3445,7 @@ static size_t ZSTDv06_decompressSequences(
 {
     const BYTE* ip = (const BYTE*)seqStart;
     const BYTE* const iend = ip + seqSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + maxDstSize;
     BYTE* op = ostart;
     const BYTE* litPtr = dctx->litPtr;
@@ -3561,7 +3561,7 @@ static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
 {
     const BYTE* ip = (const BYTE*)src;
     const BYTE* const iend = ip + srcSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
     BYTE* const oend = ostart + dstCapacity;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index fe970a651..1239d814a 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -3258,7 +3258,7 @@ typedef struct
 *   Provides the size of compressed block from block header `src` */
 static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
 {
-    const BYTE* const in = (const BYTE* const)src;
+    const BYTE* const in = (const BYTE*)src;
     U32 cSize;
 
     if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
@@ -3453,7 +3453,7 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
                              FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
                              const void* src, size_t srcSize)
 {
-    const BYTE* const istart = (const BYTE* const)src;
+    const BYTE* const istart = (const BYTE*)src;
     const BYTE* const iend = istart + srcSize;
     const BYTE* ip = istart;
 
@@ -3672,7 +3672,7 @@ static size_t ZSTDv07_decompressSequences(
 {
     const BYTE* ip = (const BYTE*)seqStart;
     const BYTE* const iend = ip + seqSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + maxDstSize;
     BYTE* op = ostart;
     const BYTE* litPtr = dctx->litPtr;
@@ -3799,7 +3799,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
 {
     const BYTE* ip = (const BYTE*)src;
     const BYTE* const iend = ip + srcSize;
-    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstCapacity;
     BYTE* op = ostart;
     size_t remainingSize = srcSize;

From f9b1e711ba7d66130ebf29150512414155f0d823 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 5 Feb 2021 12:16:10 -0800
Subject: [PATCH 55/80] [zstd] Fix NULL pointer addition in
 ZSTD_checkContinuity()

Don't start a new section when `dstSize == 0` to avoid NULL
pointer addition.
---
 lib/decompress/zstd_decompress.c          | 6 +++---
 lib/decompress/zstd_decompress_block.c    | 6 +++---
 lib/decompress/zstd_decompress_internal.h | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 0619add3a..2536cf44b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -752,7 +752,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
 {
     DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
-    ZSTD_checkContinuity(dctx, blockStart);
+    ZSTD_checkContinuity(dctx, blockStart, blockSize);
     dctx->previousDstEnd = (const char*)blockStart + blockSize;
     return blockSize;
 }
@@ -938,7 +938,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
              * use this in all cases but ddict */
             FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
         }
-        ZSTD_checkContinuity(dctx, dst);
+        ZSTD_checkContinuity(dctx, dst, dstCapacity);
 
         {   const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
                                                     &src, &srcSize);
@@ -1073,7 +1073,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
     DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
     /* Sanity check */
     RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
-    if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
+    ZSTD_checkContinuity(dctx, dst, dstCapacity);
 
     switch (dctx->stage)
     {
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index c2ab5e740..b71bc20d6 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1517,9 +1517,9 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 }
 
 
-void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
 {
-    if (dst != dctx->previousDstEnd) {   /* not contiguous */
+    if (dst != dctx->previousDstEnd && dstSize > 0) {   /* not contiguous */
         dctx->dictEnd = dctx->previousDstEnd;
         dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
         dctx->prefixStart = dst;
@@ -1533,7 +1533,7 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
                       const void* src, size_t srcSize)
 {
     size_t dSize;
-    ZSTD_checkContinuity(dctx, dst);
+    ZSTD_checkContinuity(dctx, dst, dstCapacity);
     dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0);
     dctx->previousDstEnd = (char*)dst + dSize;
     return dSize;
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 8877e4d2c..72751c9ff 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -193,7 +193,7 @@ size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
  *  If yes, do nothing (continue on current segment).
  *  If not, classify previous segment as "external dictionary", and start a new segment.
  *  This function cannot fail. */
-void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst);
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
 
 
 #endif /* ZSTD_DECOMPRESS_INTERNAL_H */

From 54a4998a808fbe371c1ace6f2026849f1c93d8e6 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 3 Feb 2021 19:53:00 -0800
Subject: [PATCH 56/80] Add basic tracing functionality

---
 build/VS2008/zstd/zstd.vcproj             |   4 +
 build/VS2010/zstd/zstd.vcxproj            |   1 +
 build/cmake/programs/CMakeLists.txt       |   4 +-
 build/meson/programs/meson.build          |   5 +-
 contrib/linux-kernel/Makefile             |   4 +-
 contrib/single_file_libs/zstd-in.c        |   1 +
 contrib/single_file_libs/zstddeclib-in.c  |   1 +
 lib/common/compiler.h                     |   1 +
 lib/common/zstd_trace.c                   |  42 +++++
 lib/common/zstd_trace.h                   | 118 +++++++++++++
 lib/compress/zstd_compress.c              |  45 ++++-
 lib/compress/zstd_compress_internal.h     |   4 +
 lib/decompress/zstd_decompress.c          |  40 ++++-
 lib/decompress/zstd_decompress_internal.h |   4 +
 lib/zstd.h                                |  14 +-
 programs/Makefile                         |   8 +-
 programs/zstdcli.c                        |  13 ++
 programs/zstdcli_trace.c                  | 192 ++++++++++++++++++++++
 programs/zstdcli_trace.h                  |  24 +++
 tests/playTests.sh                        |  15 ++
 20 files changed, 521 insertions(+), 19 deletions(-)
 create mode 100644 lib/common/zstd_trace.c
 create mode 100644 lib/common/zstd_trace.h
 create mode 100644 programs/zstdcli_trace.c
 create mode 100644 programs/zstdcli_trace.h

diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj
index ab02e6155..6f2652902 100644
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@@ -480,6 +480,10 @@
 				RelativePath="..\..\..\programs\zstdcli.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\programs\zstdcli_trace.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
 				>
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index e320d88a1..e389f9693 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -63,6 +63,7 @@
     <ClCompile Include="..\..\..\programs\dibio.c" />
     <ClCompile Include="..\..\..\programs\fileio.c" />
     <ClCompile Include="..\..\..\programs\zstdcli.c" />
+    <ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\common\pool.h" />
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index b26e97d0d..c61fe56ea 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -32,7 +32,7 @@ if (MSVC)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
 endif ()
 
-add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
 target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
 if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
     target_link_libraries(zstd rt)
@@ -75,7 +75,7 @@ if (UNIX)
 
     add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
     target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
-    set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
+    set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
 endif ()
 
 # Add multi-threading support definitions
diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build
index 363818f9d..d255627cd 100644
--- a/build/meson/programs/meson.build
+++ b/build/meson/programs/meson.build
@@ -17,7 +17,8 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c'),
   join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/dibio.c')]
+  join_paths(zstd_rootdir, 'programs/dibio.c'),
+  join_paths(zstd_rootdir, 'programs/zstdcli_trace.c')]
 
 zstd_c_args = libzstd_debug_cflags
 if use_multi_thread
@@ -73,7 +74,7 @@ zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
 executable('zstd-frugal',
   zstd_frugal_sources,
   dependencies: libzstd_dep,
-  c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ],
+  c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ],
   install: true)
 
 install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile
index a0cc67ac4..1725e7d46 100644
--- a/contrib/linux-kernel/Makefile
+++ b/contrib/linux-kernel/Makefile
@@ -45,7 +45,9 @@ libzstd:
 		-U_MSC_VER \
 		-U_WIN32 \
 		-RZSTDLIB_VISIBILITY= \
-		-RZSTDERRORLIB_VISIBILITY=
+		-RZSTDERRORLIB_VISIBILITY= \
+		-DZSTD_HAVE_WEAK_SYMBOLS=0 \
+		-DZSTD_TRACE=0
 	cp linux_zstd.h linux/include/linux/zstd.h
 	cp zstd_compress_module.c linux/lib/zstd
 	cp zstd_decompress_module.c linux/lib/zstd
diff --git a/contrib/single_file_libs/zstd-in.c b/contrib/single_file_libs/zstd-in.c
index 0addbab3a..1b27953a6 100644
--- a/contrib/single_file_libs/zstd-in.c
+++ b/contrib/single_file_libs/zstd-in.c
@@ -42,6 +42,7 @@
 #ifndef __EMSCRIPTEN__
 #define ZSTD_MULTITHREAD
 #endif
+#define ZSTD_TRACE 0
 
 /* Include zstd_deps.h first with all the options we need enabled. */
 #define ZSTD_DEPS_NEED_MALLOC
diff --git a/contrib/single_file_libs/zstddeclib-in.c b/contrib/single_file_libs/zstddeclib-in.c
index f94467e7e..019d9c260 100644
--- a/contrib/single_file_libs/zstddeclib-in.c
+++ b/contrib/single_file_libs/zstddeclib-in.c
@@ -38,6 +38,7 @@
 #define XXH_INLINE_ALL
 #define ZSTD_LEGACY_SUPPORT 0
 #define ZSTD_STRIP_ERROR_STRINGS
+#define ZSTD_TRACE 0
 
 /* Include zstd_deps.h first with all the options we need enabled. */
 #define ZSTD_DEPS_NEED_MALLOC
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 2697f6051..1142002b0 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -90,6 +90,7 @@
 #  endif
 #endif
 
+
 /* target attribute */
 #ifndef __has_attribute
   #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
diff --git a/lib/common/zstd_trace.c b/lib/common/zstd_trace.c
new file mode 100644
index 000000000..18c3cfb74
--- /dev/null
+++ b/lib/common/zstd_trace.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_trace.h"
+#include "../zstd.h"
+
+#include "compiler.h"
+
+#if ZSTD_TRACE && ZSTD_HAVE_WEAK_SYMBOLS
+
+ZSTD_WEAK_ATTR int ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
+{
+    (void)cctx;
+    return 0;
+}
+
+ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(ZSTD_CCtx const* cctx, ZSTD_trace const* trace)
+{
+    (void)cctx;
+    (void)trace;
+}
+
+ZSTD_WEAK_ATTR int ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
+{
+    (void)dctx;
+    return 0;
+}
+
+ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(ZSTD_DCtx const* dctx, ZSTD_trace const* trace)
+{
+    (void)dctx;
+    (void)trace;
+}
+
+#endif
diff --git a/lib/common/zstd_trace.h b/lib/common/zstd_trace.h
new file mode 100644
index 000000000..1e28b1f83
--- /dev/null
+++ b/lib/common/zstd_trace.h
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_TRACE_H
+#define ZSTD_TRACE_H
+
+#include <stddef.h>
+
+/* weak symbol support */
+#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \
+    !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
+    !defined(__CYGWIN__)
+#  define ZSTD_HAVE_WEAK_SYMBOLS 1
+#else
+#  define ZSTD_HAVE_WEAK_SYMBOLS 0
+#endif
+#if ZSTD_HAVE_WEAK_SYMBOLS
+#  define ZSTD_WEAK_ATTR __attribute__((__weak__))
+#else
+#  define ZSTD_WEAK_ATTR
+#endif
+
+/* Only enable tracing when weak symbols are available. */
+#ifndef ZSTD_TRACE
+#  define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
+#endif
+
+#if ZSTD_TRACE
+
+struct ZSTD_CCtx_s;
+struct ZSTD_DCtx_s;
+struct ZSTD_CCtx_params_s;
+
+typedef struct {
+    /**
+     * ZSTD_VERSION_NUMBER
+     *
+     * This is guaranteed to be the first member of ZSTD_trace.
+     * Otherwise, this struct is not stable between versions. If
+     * the version number does not match your expectation, you
+     * should not interpret the rest of the struct.
+     */
+    unsigned version;
+    /**
+     * Non-zero if streaming (de)compression is used.
+     */
+    unsigned streaming;
+    /**
+     * The dictionary ID.
+     */
+    unsigned dictionaryID;
+    /**
+     * Is the dictionary cold?
+     * Only set on decompression.
+     */
+    unsigned dictionaryIsCold;
+    /**
+     * The dictionary size or zero if no dictionary.
+     */
+    size_t dictionarySize;
+    /**
+     * The uncompressed size of the data.
+     */
+    size_t uncompressedSize;
+    /**
+     * The compressed size of the data.
+     */
+    size_t compressedSize;
+    /**
+     * The fully resolved CCtx parameters (NULL on decompression).
+     */
+    struct ZSTD_CCtx_params_s const* params;
+} ZSTD_trace;
+
+/**
+ * Trace the beginning of a compression call.
+ * @param cctx The dctx pointer for the compression.
+ *             It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled.
+ */
+int ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx);
+
+/**
+ * Trace the end of a compression call.
+ * @param cctx The dctx pointer for the decompression.
+ * @param trace The zstd tracing info.
+ */
+void ZSTD_trace_compress_end(
+    struct ZSTD_CCtx_s const* cctx,
+    ZSTD_trace const* trace);
+
+/**
+ * Trace the beginning of a decompression call.
+ * @param dctx The dctx pointer for the decompression.
+ *             It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled.
+ */
+int ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx);
+
+/**
+ * Trace the end of a decompression call.
+ * @param dctx The dctx pointer for the decompression.
+ * @param trace The zstd tracing info.
+ */
+void ZSTD_trace_decompress_end(
+    struct ZSTD_DCtx_s const* dctx,
+    ZSTD_trace const* trace);
+
+#endif /* ZSTD_TRACE */
+
+#endif /* ZSTD_TRACE_H */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c90fb320e..eea724d48 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -14,6 +14,7 @@
 #include "../common/zstd_deps.h"  /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
 #include "../common/cpu.h"
 #include "../common/mem.h"
+#include "../common/zstd_trace.h"
 #include "hist.h"           /* HIST_countFast_wksp */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "../common/fse.h"
@@ -802,7 +803,7 @@ size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value
 }
 
 size_t ZSTD_CCtxParams_getParameter(
-        ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int* value)
+        ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value)
 {
     switch(param)
     {
@@ -1704,6 +1705,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         XXH64_reset(&zc->xxhState, 0);
         zc->stage = ZSTDcs_init;
         zc->dictID = 0;
+        zc->dictContentSize = 0;
 
         ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);
 
@@ -1864,6 +1866,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
     }   }
 
     cctx->dictID = cdict->dictID;
+    cctx->dictContentSize = cdict->dictContentSize;
 
     /* copy block state */
     ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
@@ -1927,6 +1930,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
     }
 
     cctx->dictID = cdict->dictID;
+    cctx->dictContentSize = cdict->dictContentSize;
 
     /* copy block state */
     ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
@@ -2017,6 +2021,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
         dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
     }
     dstCCtx->dictID = srcCCtx->dictID;
+    dstCCtx->dictContentSize = srcCCtx->dictContentSize;
 
     /* copy block state */
     ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock));
@@ -3380,6 +3385,9 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                                     const ZSTD_CCtx_params* params, U64 pledgedSrcSize,
                                     ZSTD_buffered_policy_e zbuff)
 {
+#if ZSTD_TRACE
+    cctx->tracingEnabled = ZSTD_trace_compress_begin(cctx);
+#endif
     DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog);
     /* params are supposed to be fully validated at this point */
     assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
@@ -3409,6 +3417,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
         FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
         assert(dictID <= UINT_MAX);
         cctx->dictID = (U32)dictID;
+        cctx->dictContentSize = cdict ? cdict->dictContentSize : dictSize;
     }
     return 0;
 }
@@ -3503,6 +3512,28 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
     return op-ostart;
 }
 
+static void ZSTD_CCtx_trace(ZSTD_CCtx const* cctx, size_t extraCSize)
+{
+#if ZSTD_TRACE
+    if (cctx->tracingEnabled) {
+        int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0;
+        ZSTD_trace trace;
+        ZSTD_memset(&trace, 0, sizeof(trace));
+        trace.version = ZSTD_VERSION_NUMBER;
+        trace.streaming = streaming;
+        trace.dictionaryID = cctx->dictID;
+        trace.dictionarySize = cctx->dictContentSize;
+        trace.uncompressedSize = cctx->consumedSrcSize;
+        trace.compressedSize = cctx->producedCSize + extraCSize;
+        trace.params = &cctx->appliedParams;
+        ZSTD_trace_compress_end(cctx, &trace);
+    }
+#else
+    (void)cctx;
+    (void)extraCSize;
+#endif
+}
+
 size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
                          void* dst, size_t dstCapacity,
                    const void* src, size_t srcSize)
@@ -3525,6 +3556,7 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
             (unsigned)cctx->pledgedSrcSizePlusOne-1,
             (unsigned)cctx->consumedSrcSize);
     }
+    ZSTD_CCtx_trace(cctx, endResult);
     return cSize + endResult;
 }
 
@@ -4408,6 +4440,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
         params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
     }
     if (params.nbWorkers > 0) {
+#if ZSTD_TRACE
+        cctx->tracingEnabled = ZSTD_trace_compress_begin(cctx);
+#endif
         /* mt context creation */
         if (cctx->mtctx == NULL) {
             DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
@@ -4421,6 +4456,10 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
                     cctx->mtctx,
                     prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
                     cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
+        cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0;
+        cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize;
+        cctx->consumedSrcSize = 0;
+        cctx->producedCSize = 0;
         cctx->streamStage = zcss_load;
         cctx->appliedParams = params;
     } else
@@ -4482,8 +4521,12 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             size_t const ipos = input->pos;
             size_t const opos = output->pos;
             flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
+            cctx->consumedSrcSize += (U64)(input->pos - ipos);
+            cctx->producedCSize += (U64)(output->pos - opos);
             if ( ZSTD_isError(flushMin)
               || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
+                if (flushMin == 0)
+                    ZSTD_CCtx_trace(cctx, 0);
                 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
             }
             FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 10846ae50..3ee00269d 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -270,6 +270,7 @@ struct ZSTD_CCtx_s {
     ZSTD_CCtx_params requestedParams;
     ZSTD_CCtx_params appliedParams;
     U32   dictID;
+    size_t dictContentSize;
 
     ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
     size_t blockSize;
@@ -321,6 +322,9 @@ struct ZSTD_CCtx_s {
 #ifdef ZSTD_MULTITHREAD
     ZSTDMT_CCtx* mtctx;
 #endif
+
+    /* Tracing */
+    int tracingEnabled;
 };
 
 typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 2536cf44b..c8a7701fb 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -58,6 +58,7 @@
 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
 #include "../common/cpu.h"         /* bmi2 */
 #include "../common/mem.h"         /* low level memory routines */
+#include "../common/zstd_trace.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
 #define HUF_STATIC_LINKING_ONLY
@@ -630,6 +631,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
 #endif
     dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
     if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
+    dctx->processedCSize += headerSize;
     return 0;
 }
 
@@ -784,6 +786,31 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
     return regenSize;
 }
 
+static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
+{
+#if ZSTD_TRACE
+    if (dctx->tracingEnabled) {
+        ZSTD_trace trace;
+        ZSTD_memset(&trace, 0, sizeof(trace));
+        trace.version = ZSTD_VERSION_NUMBER;
+        trace.streaming = streaming;
+        if (dctx->ddict) {
+            trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
+            trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
+            trace.dictionaryIsCold = dctx->ddictIsCold;
+        }
+        trace.uncompressedSize = (size_t)uncompressedSize;
+        trace.compressedSize = (size_t)compressedSize;
+        ZSTD_trace_decompress_end(dctx, &trace);
+    }
+#else
+    (void)dctx;
+    (void)uncompressedSize;
+    (void)compressedSize;
+    (void)streaming;
+#endif
+}
+
 
 /*! ZSTD_decompressFrame() :
  * @dctx must be properly initialized
@@ -793,7 +820,8 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
                                    void* dst, size_t dstCapacity,
                              const void** srcPtr, size_t *srcSizePtr)
 {
-    const BYTE* ip = (const BYTE*)(*srcPtr);
+    const BYTE* const istart = (const BYTE*)(*srcPtr);
+    const BYTE* ip = istart;
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
     BYTE* op = ostart;
@@ -869,7 +897,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         ip += 4;
         remainingSrcSize -= 4;
     }
-
+    ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
     /* Allow caller to get size read */
     *srcPtr = ip;
     *srcSizePtr = remainingSrcSize;
@@ -1075,6 +1103,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
     RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
     ZSTD_checkContinuity(dctx, dst, dstCapacity);
 
+    dctx->processedCSize += srcSize;
+
     switch (dctx->stage)
     {
     case ZSTDds_getFrameHeaderSize :
@@ -1178,6 +1208,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
                     dctx->expected = 4;
                     dctx->stage = ZSTDds_checkChecksum;
                 } else {
+                    ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
                     dctx->expected = 0;   /* ends here */
                     dctx->stage = ZSTDds_getFrameHeaderSize;
                 }
@@ -1197,6 +1228,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
                 DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
                 RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
             }
+            ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
             dctx->expected = 0;
             dctx->stage = ZSTDds_getFrameHeaderSize;
             return 0;
@@ -1350,8 +1382,12 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
 {
     assert(dctx != NULL);
+#if ZSTD_TRACE
+    dctx->tracingEnabled = ZSTD_trace_decompress_begin(dctx);
+#endif
     dctx->expected = ZSTD_startingInputLength(dctx->format);  /* dctx->format must be properly set */
     dctx->stage = ZSTDds_getFrameHeaderSize;
+    dctx->processedCSize = 0;
     dctx->decodedSize = 0;
     dctx->previousDstEnd = NULL;
     dctx->prefixStart = NULL;
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 72751c9ff..07c515977 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -120,6 +120,7 @@ struct ZSTD_DCtx_s
     const void* dictEnd;          /* end of previous segment */
     size_t expected;
     ZSTD_frameHeader fParams;
+    U64 processedCSize;
     U64 decodedSize;
     blockType_e bType;            /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
     ZSTD_dStage stage;
@@ -175,6 +176,9 @@ struct ZSTD_DCtx_s
     void const* dictContentBeginForFuzzing;
     void const* dictContentEndForFuzzing;
 #endif
+
+    /* Tracing */
+    int tracingEnabled;
 };  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 143db3541..230adee34 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1001,13 +1001,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
 /*! ZSTD_DCtx_refDDict() :
  *  Reference a prepared dictionary, to be used to decompress next frames.
  *  The dictionary remains active for decompression of future frames using same DCtx.
- * 
+ *
  *  If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
  *  will store the DDict references in a table, and the DDict used for decompression
  *  will be determined at decompression time, as per the dict ID in the frame.
  *  The memory for the table is allocated on the first call to refDDict, and can be
  *  freed with ZSTD_freeDCtx().
- * 
+ *
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Note 1 : Currently, only one dictionary can be managed.
  *           Referencing a new dictionary effectively "discards" any previous one.
@@ -1892,7 +1892,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_c
  * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  */
-ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
 
 /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
  *  Apply a set of ZSTD_CCtx_params to the compression context.
@@ -2027,15 +2027,15 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
  * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
  * store all references. At decompression time, the appropriate dictID is selected
  * from the set of DDicts based on the dictID in the frame.
- * 
+ *
  * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
- * 
+ *
  * Param has values of byte ZSTD_refMultipleDDicts_e
- * 
+ *
  * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
  * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
  * Memory is allocated as per ZSTD_DCtx::customMem.
- * 
+ *
  * Although this function allocates memory for the table, the user is still responsible for
  * memory management of the underlying ZSTD_DDict* themselves.
  */
diff --git a/programs/Makefile b/programs/Makefile
index b254549bc..31576ecc9 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -314,16 +314,16 @@ zstd-pgo :
 ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
 zstd-small: CFLAGS = -Os -s
 zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c
-	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE $^ -o $@$(EXT)
 
 zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c
-	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
 
 zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c
-	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
 
 ## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
-zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS
+zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE
 zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index b279287d4..c4e1b4792 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -42,6 +42,9 @@
 #ifndef ZSTD_NODICT
 #  include "dibio.h"  /* ZDICT_cover_params_t, DiB_trainFromFiles() */
 #endif
+#ifndef ZSTD_NOTRACE
+#  include "zstdcli_trace.h"
+#endif
 #include "../lib/zstd.h"  /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
 
 
@@ -186,6 +189,10 @@ static void usage_advanced(const char* programName)
     DISPLAYOUT( "--[no-]check : during decompression, ignore/validate checksums in compressed frame (default: validate).");
 #endif
 #endif /* ZSTD_NOCOMPRESS */
+
+#ifndef ZSTD_NOTRACE
+    DISPLAYOUT( "--trace FILE : log tracing information to FILE.");
+#endif
     DISPLAYOUT( "\n");
 
     DISPLAYOUT( "--      : All arguments after \"--\" are treated as files \n");
@@ -918,6 +925,9 @@ int main(int const argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--output-dir-flat")) { NEXT_FIELD(outDirName); continue; }
 #ifdef UTIL_HAS_MIRRORFILELIST
                 if (longCommandWArg(&argument, "--output-dir-mirror")) { NEXT_FIELD(outMirroredDirName); continue; }
+#endif
+#ifndef ZSTD_NOTRACE
+                if (longCommandWArg(&argument, "--trace")) { char const* traceFile; NEXT_FIELD(traceFile); TRACE_enable(traceFile); continue; }
 #endif
                 if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); continue; }
                 if (longCommandWArg(&argument, "--long")) {
@@ -1397,6 +1407,9 @@ _end:
     if (main_pause) waitEnter();
     UTIL_freeFileNamesTable(filenames);
     UTIL_freeFileNamesTable(file_of_names);
+#ifndef ZSTD_NOTRACE
+    TRACE_finish();
+#endif
 
     return operationResult;
 }
diff --git a/programs/zstdcli_trace.c b/programs/zstdcli_trace.c
new file mode 100644
index 000000000..aa90a2a02
--- /dev/null
+++ b/programs/zstdcli_trace.c
@@ -0,0 +1,192 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstdcli_trace.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "timefn.h"
+#include "util.h"
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "../lib/zstd.h"
+/* We depend on the trace header to avoid duplicating the ZSTD_trace struct.
+ * But, we check the version so it is compatible with dynamic linking.
+ */
+#include "../lib/common/zstd_trace.h"
+/* We only use macros from threading.h so it is compatible with dynamic linking */
+#include "../lib/common/threading.h"
+
+#if ZSTD_TRACE
+
+static FILE* g_traceFile = NULL;
+static int g_mutexInit = 0;
+static ZSTD_pthread_mutex_t g_mutex;
+
+void TRACE_enable(char const* filename)
+{
+    int const writeHeader = !UTIL_isRegularFile(filename);
+    if (g_traceFile)
+        fclose(g_traceFile);
+    g_traceFile = fopen(filename, "a");
+    if (g_traceFile && writeHeader) {
+        /* Fields:
+        * algorithm
+        * version
+        * method
+        * streaming
+        * level
+        * workers
+        * dictionary size
+        * uncompressed size
+        * compressed size
+        * duration nanos
+        * compression ratio
+        * speed MB/s
+        */
+        fprintf(g_traceFile, "Algorithm, Version, Method, Mode, Level, Workers, Dictionary Size, Uncompressed Size, Compressed Size, Duration Nanos, Compression Ratio, Speed MB/s\n");
+    }
+    if (!g_mutexInit) {
+        if (!ZSTD_pthread_mutex_init(&g_mutex, NULL)) {
+            g_mutexInit = 1;
+        } else {
+            TRACE_finish();
+        }
+    }
+}
+
+void TRACE_finish(void)
+{
+    if (g_traceFile) {
+        fclose(g_traceFile);
+    }
+    g_traceFile = NULL;
+    if (g_mutexInit) {
+        ZSTD_pthread_mutex_destroy(&g_mutex);
+        g_mutexInit = 0;
+    }
+}
+
+static void TRACE_log(char const* method, PTime duration, ZSTD_trace const* trace)
+{
+    int level = 0;
+    int workers = 0;
+    double const ratio = (double)trace->uncompressedSize / (double)trace->compressedSize;
+    double const speed = ((double)trace->uncompressedSize * 1000) / (double)duration;
+    if (trace->params) {
+        ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_compressionLevel, &level);
+        ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_nbWorkers, &workers);
+    }
+    assert(g_traceFile != NULL);
+
+    /* Fields:
+     * algorithm
+     * version
+     * method
+     * streaming
+     * level
+     * workers
+     * dictionary size
+     * uncompressed size
+     * compressed size
+     * duration nanos
+     * compression ratio
+     * speed MB/s
+     */
+    fprintf(g_traceFile,
+        "zstd, %u, %s, %s, %d, %d, %llu, %llu, %llu, %llu, %.2f, %.2f\n",
+        trace->version,
+        method,
+        trace->streaming ? "streaming" : "single-pass",
+        level,
+        workers,
+        (unsigned long long)trace->dictionarySize,
+        (unsigned long long)trace->uncompressedSize,
+        (unsigned long long)trace->compressedSize,
+        (unsigned long long)duration,
+        ratio,
+        speed);
+}
+
+static ZSTD_CCtx const* g_cctx;
+static ZSTD_DCtx const* g_dctx;
+static UTIL_time_t g_begin = UTIL_TIME_INITIALIZER;
+
+/**
+ * These symbols override the weak symbols provided by the library.
+ */
+
+int ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
+{
+    int enabled = 0;
+    if (g_traceFile == NULL)
+        return 0;
+    ZSTD_pthread_mutex_lock(&g_mutex);
+    if (g_cctx == NULL) {
+        g_cctx = cctx;
+        g_dctx = NULL;
+        g_begin = UTIL_getTime();
+        enabled = 1;
+    }
+    ZSTD_pthread_mutex_unlock(&g_mutex);
+    return enabled;
+}
+
+void ZSTD_trace_compress_end(ZSTD_CCtx const* cctx, ZSTD_trace const* trace)
+{
+    assert(g_traceFile != NULL);
+    ZSTD_pthread_mutex_lock(&g_mutex);
+    assert(g_cctx == cctx);
+    assert(g_dctx == NULL);
+    if (cctx == g_cctx && trace->version == ZSTD_VERSION_NUMBER)
+        TRACE_log("compress", UTIL_clockSpanNano(g_begin), trace);
+    g_cctx = NULL;
+    ZSTD_pthread_mutex_unlock(&g_mutex);
+}
+
+int ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
+{
+    int enabled = 0;
+    if (g_traceFile == NULL)
+        return 0;
+    ZSTD_pthread_mutex_lock(&g_mutex);
+    if (g_dctx == NULL) {
+        g_cctx = NULL;
+        g_dctx = dctx;
+        g_begin = UTIL_getTime();
+        enabled = 1;
+    }
+    ZSTD_pthread_mutex_unlock(&g_mutex);
+    return enabled;
+}
+
+void ZSTD_trace_decompress_end(ZSTD_DCtx const* dctx, ZSTD_trace const* trace)
+{
+    assert(g_traceFile != NULL);
+    ZSTD_pthread_mutex_lock(&g_mutex);
+    assert(g_cctx == NULL);
+    assert(g_dctx == dctx);
+    if (dctx == g_dctx && trace->version == ZSTD_VERSION_NUMBER)
+        TRACE_log("decompress", UTIL_clockSpanNano(g_begin), trace);
+    g_dctx = NULL;
+    ZSTD_pthread_mutex_unlock(&g_mutex);
+}
+
+#else /* ZSTD_TRACE */
+
+void TRACE_enable(char const* filename)
+{
+    (void)filename;
+}
+
+void TRACE_finish(void) {}
+
+#endif /* ZSTD_TRACE */
diff --git a/programs/zstdcli_trace.h b/programs/zstdcli_trace.h
new file mode 100644
index 000000000..6ed390808
--- /dev/null
+++ b/programs/zstdcli_trace.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTDCLI_TRACE_H
+#define ZSTDCLI_TRACE_H
+
+/**
+ * Enable tracing - log to filename.
+ */
+void TRACE_enable(char const* filename);
+
+/**
+ * Shut down the tracing library.
+ */
+void TRACE_finish(void);
+
+#endif /* ZSTDCLI_TRACE_H */
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 4d6abbcc0..20259bc2f 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -1322,6 +1322,21 @@ zstd -f --no-check tmp1
 zstd -l tmp1.zst
 zstd -lv tmp1.zst
 
+println "\n===>  zstd trace tests "
+zstd -f --trace tmp.trace tmp1
+zstd -f --trace tmp.trace tmp1 tmp2 tmp3
+zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null
+zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread
+zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null
+zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread
+zstd --trace tmp.trace -t tmp1.zst
+zstd --trace tmp.trace -t tmp1.zst tmp2.zst
+zstd -f --trace tmp.trace -d tmp1.zst
+zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst
+zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1
+zstd -b1e10i0 --trace tmp.trace tmp1
+zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3
+
 rm tmp*
 
 

From 2636f53619ac7832eda63ebf4f258ba46bf7290e Mon Sep 17 00:00:00 2001
From: "Issam E. Maghni" <issam.e.maghni@mailbox.org>
Date: Sun, 7 Feb 2021 19:38:47 -0500
Subject: [PATCH 57/80] CMake: Enable only C for lib and programs projects

---
 build/cmake/lib/CMakeLists.txt      | 2 +-
 build/cmake/programs/CMakeLists.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index f34c950a3..179c97624 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -7,7 +7,7 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(libzstd)
+project(libzstd C)
 
 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index c61fe56ea..f1d127746 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -7,7 +7,7 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(programs)
+project(programs C)
 
 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 

From e2ad174d730de3ed7655c3c6b550f4feb361c919 Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Mon, 8 Feb 2021 11:43:06 +0100
Subject: [PATCH 58/80] fix some compiler warnings

---
 lib/compress/zstd_ldm.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 9292aa4d0..314e4845f 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -264,7 +264,7 @@ void ZSTD_ldm_fillHashTable(
             if (ip + splits[n] >= istart + minMatchLength) {
                 BYTE const* const split = ip + splits[n] - minMatchLength;
                 U64 const xxhash = XXH64(split, minMatchLength, 0);
-                U64 const hash = xxhash & (((U32)1 << hBits) - 1);
+                U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
                 ldmEntry_t entry;
 
                 entry.offset = (U32)(split - base);
@@ -355,7 +355,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
         for (n = 0; n < numSplits; n++) {
             BYTE const* const split = ip + splits[n] - minMatchLength;
             U64 const xxhash = XXH64(split, minMatchLength, 0);
-            U64 const hash = xxhash & (((U32)1 << hBits) - 1);
+            U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
 
             candidates[n].split = split;
             candidates[n].hash = hash;
@@ -368,8 +368,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
             size_t forwardMatchLength = 0, backwardMatchLength = 0,
                    bestMatchLength = 0, mLength;
             BYTE const* const split = candidates[n].split;
-            U64 const checksum = candidates[n].checksum;
-            U64 const hash = candidates[n].hash;
+            U32 const checksum = candidates[n].checksum;
+            U32 const hash = candidates[n].hash;
             ldmEntry_t* const bucket = candidates[n].bucket;
             ldmEntry_t const* cur;
             ldmEntry_t const* bestEntry = NULL;

From e59c9459a57c68e9b478bf6f0a3c4e5f3052b4f7 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 9 Feb 2021 11:37:05 -0800
Subject: [PATCH 59/80] [trace] Keep track of a uint64_t tracing context

The most common information that you want to track between begin() and
end() is the timestamp of the begin function, so you can measure the
duration of the (de)compression call. Allow the tracing library to put
this information inside the `ZSTD_TraceCtx`, so it doesn't need to keep
a global map in this case. If a single uint64_t is not enough, the
tracing library can return a unique identifier (like the context
pointer) instead, and use it as a key in a map.

This keeps the simple case simple.
---
 lib/common/zstd_trace.c                   | 12 ++---
 lib/common/zstd_trace.h                   | 48 +++++++++++++----
 lib/compress/zstd_compress.c              | 14 ++---
 lib/compress/zstd_compress_internal.h     | 10 +++-
 lib/compress/zstdmt_compress.c            |  3 ++
 lib/decompress/zstd_decompress.c          |  9 ++--
 lib/decompress/zstd_decompress_internal.h |  5 +-
 programs/zstdcli_trace.c                  | 66 ++++++++---------------
 8 files changed, 95 insertions(+), 72 deletions(-)

diff --git a/lib/common/zstd_trace.c b/lib/common/zstd_trace.c
index 18c3cfb74..36f3b5d69 100644
--- a/lib/common/zstd_trace.c
+++ b/lib/common/zstd_trace.c
@@ -15,27 +15,27 @@
 
 #if ZSTD_TRACE && ZSTD_HAVE_WEAK_SYMBOLS
 
-ZSTD_WEAK_ATTR int ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
 {
     (void)cctx;
     return 0;
 }
 
-ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(ZSTD_CCtx const* cctx, ZSTD_trace const* trace)
+ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(ZSTD_TraceCtx ctx, ZSTD_Trace const* trace)
 {
-    (void)cctx;
+    (void)ctx;
     (void)trace;
 }
 
-ZSTD_WEAK_ATTR int ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
 {
     (void)dctx;
     return 0;
 }
 
-ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(ZSTD_DCtx const* dctx, ZSTD_trace const* trace)
+ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(ZSTD_TraceCtx ctx, ZSTD_Trace const* trace)
 {
-    (void)dctx;
+    (void)ctx;
     (void)trace;
 }
 
diff --git a/lib/common/zstd_trace.h b/lib/common/zstd_trace.h
index 1e28b1f83..f2fa7f79e 100644
--- a/lib/common/zstd_trace.h
+++ b/lib/common/zstd_trace.h
@@ -77,41 +77,67 @@ typedef struct {
      * The fully resolved CCtx parameters (NULL on decompression).
      */
     struct ZSTD_CCtx_params_s const* params;
-} ZSTD_trace;
+    /**
+     * The ZSTD_CCtx pointer (NULL on decompression).
+     */
+    struct ZSTD_CCtx_s const* cctx;
+    /**
+     * The ZSTD_DCtx pointer (NULL on compression).
+     */
+    struct ZSTD_DCtx_s const* dctx;
+} ZSTD_Trace;
+
+/**
+ * A tracing context. It must be 0 when tracing is disabled.
+ * Otherwise, any non-zero value returned by a tracing begin()
+ * function is presented to any subsequent calls to end().
+ *
+ * Any non-zero value is treated as tracing is enabled and not
+ * interpreted by the library.
+ *
+ * Two possible uses are:
+ * * A timestamp for when the begin() function was called.
+ * * A unique key identifying the (de)compression, like the
+ *   address of the [dc]ctx pointer if you need to track
+ *   more information than just a timestamp.
+ */
+typedef unsigned long long ZSTD_TraceCtx;
 
 /**
  * Trace the beginning of a compression call.
  * @param cctx The dctx pointer for the compression.
  *             It can be used as a key to map begin() to end().
- * @returns Non-zero if tracing is enabled.
+ * @returns Non-zero if tracing is enabled. The return value is
+ *          passed to ZSTD_trace_compress_end().
  */
-int ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx);
+ZSTD_TraceCtx ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx);
 
 /**
  * Trace the end of a compression call.
- * @param cctx The dctx pointer for the decompression.
+ * @param ctx The return value of ZSTD_trace_compress_begin().
  * @param trace The zstd tracing info.
  */
 void ZSTD_trace_compress_end(
-    struct ZSTD_CCtx_s const* cctx,
-    ZSTD_trace const* trace);
+    ZSTD_TraceCtx ctx,
+    ZSTD_Trace const* trace);
 
 /**
  * Trace the beginning of a decompression call.
  * @param dctx The dctx pointer for the decompression.
  *             It can be used as a key to map begin() to end().
- * @returns Non-zero if tracing is enabled.
+ * @returns Non-zero if tracing is enabled. The return value is
+ *          passed to ZSTD_trace_compress_end().
  */
-int ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx);
+ZSTD_TraceCtx ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx);
 
 /**
  * Trace the end of a decompression call.
- * @param dctx The dctx pointer for the decompression.
+ * @param ctx The return value of ZSTD_trace_decompress_begin().
  * @param trace The zstd tracing info.
  */
 void ZSTD_trace_decompress_end(
-    struct ZSTD_DCtx_s const* dctx,
-    ZSTD_trace const* trace);
+    ZSTD_TraceCtx ctx,
+    ZSTD_Trace const* trace);
 
 #endif /* ZSTD_TRACE */
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index eea724d48..444b4027c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3386,7 +3386,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                                     ZSTD_buffered_policy_e zbuff)
 {
 #if ZSTD_TRACE
-    cctx->tracingEnabled = ZSTD_trace_compress_begin(cctx);
+    cctx->traceCtx = ZSTD_trace_compress_begin(cctx);
 #endif
     DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog);
     /* params are supposed to be fully validated at this point */
@@ -3512,12 +3512,12 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
     return op-ostart;
 }
 
-static void ZSTD_CCtx_trace(ZSTD_CCtx const* cctx, size_t extraCSize)
+void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
 {
 #if ZSTD_TRACE
-    if (cctx->tracingEnabled) {
+    if (cctx->traceCtx) {
         int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0;
-        ZSTD_trace trace;
+        ZSTD_Trace trace;
         ZSTD_memset(&trace, 0, sizeof(trace));
         trace.version = ZSTD_VERSION_NUMBER;
         trace.streaming = streaming;
@@ -3526,8 +3526,10 @@ static void ZSTD_CCtx_trace(ZSTD_CCtx const* cctx, size_t extraCSize)
         trace.uncompressedSize = cctx->consumedSrcSize;
         trace.compressedSize = cctx->producedCSize + extraCSize;
         trace.params = &cctx->appliedParams;
-        ZSTD_trace_compress_end(cctx, &trace);
+        trace.cctx = cctx;
+        ZSTD_trace_compress_end(cctx->traceCtx, &trace);
     }
+    cctx->traceCtx = 0;
 #else
     (void)cctx;
     (void)extraCSize;
@@ -4441,7 +4443,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     }
     if (params.nbWorkers > 0) {
 #if ZSTD_TRACE
-        cctx->tracingEnabled = ZSTD_trace_compress_begin(cctx);
+        cctx->traceCtx = ZSTD_trace_compress_begin(cctx);
 #endif
         /* mt context creation */
         if (cctx->mtctx == NULL) {
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 3ee00269d..8c1e5f9cd 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -19,6 +19,7 @@
 *  Dependencies
 ***************************************/
 #include "../common/zstd_internal.h"
+#include "../common/zstd_trace.h"  /* ZSTD_TraceCtx */
 #include "zstd_cwksp.h"
 #ifdef ZSTD_MULTITHREAD
 #  include "zstdmt_compress.h"
@@ -324,7 +325,9 @@ struct ZSTD_CCtx_s {
 #endif
 
     /* Tracing */
-    int tracingEnabled;
+#if ZSTD_TRACE
+    ZSTD_TraceCtx traceCtx;
+#endif
 };
 
 typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
@@ -1204,4 +1207,9 @@ size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSe
  *  condition for correct operation : hashLog > 1 */
 U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
 
+/** ZSTD_CCtx_trace() :
+ *  Trace the end of a compression call.
+ */
+void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
+
 #endif /* ZSTD_COMPRESS_H */
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index c3ead0878..8a0f40ef9 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -683,6 +683,8 @@ static void ZSTDMT_compressionJob(void* jobDescription)
     if (job->jobID != 0) jobParams.fParams.checksumFlag = 0;
     /* Don't run LDM for the chunks, since we handle it externally */
     jobParams.ldmParams.enableLdm = 0;
+    /* Correct nbWorkers to 0. */
+    jobParams.nbWorkers = 0;
 
 
     /* init */
@@ -750,6 +752,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
             if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
             lastCBlockSize = cSize;
     }   }
+    ZSTD_CCtx_trace(cctx, 0);
 
 _endJob:
     ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize);
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index c8a7701fb..15139501b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -789,8 +789,8 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
 static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
 {
 #if ZSTD_TRACE
-    if (dctx->tracingEnabled) {
-        ZSTD_trace trace;
+    if (dctx->traceCtx) {
+        ZSTD_Trace trace;
         ZSTD_memset(&trace, 0, sizeof(trace));
         trace.version = ZSTD_VERSION_NUMBER;
         trace.streaming = streaming;
@@ -801,7 +801,8 @@ static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64
         }
         trace.uncompressedSize = (size_t)uncompressedSize;
         trace.compressedSize = (size_t)compressedSize;
-        ZSTD_trace_decompress_end(dctx, &trace);
+        trace.dctx = dctx;
+        ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
     }
 #else
     (void)dctx;
@@ -1383,7 +1384,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
 {
     assert(dctx != NULL);
 #if ZSTD_TRACE
-    dctx->tracingEnabled = ZSTD_trace_decompress_begin(dctx);
+    dctx->traceCtx = ZSTD_trace_decompress_begin(dctx);
 #endif
     dctx->expected = ZSTD_startingInputLength(dctx->format);  /* dctx->format must be properly set */
     dctx->stage = ZSTDds_getFrameHeaderSize;
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 07c515977..3fcec6c56 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -21,6 +21,7 @@
  *********************************************************/
 #include "../common/mem.h"             /* BYTE, U16, U32 */
 #include "../common/zstd_internal.h"   /* ZSTD_seqSymbol */
+#include "../common/zstd_trace.h"      /* ZSTD_TraceCtx */
 
 
 
@@ -178,7 +179,9 @@ struct ZSTD_DCtx_s
 #endif
 
     /* Tracing */
-    int tracingEnabled;
+#if ZSTD_TRACE
+    ZSTD_TraceCtx traceCtx;
+#endif
 };  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
 
diff --git a/programs/zstdcli_trace.c b/programs/zstdcli_trace.c
index aa90a2a02..cd220b92d 100644
--- a/programs/zstdcli_trace.c
+++ b/programs/zstdcli_trace.c
@@ -30,6 +30,7 @@
 static FILE* g_traceFile = NULL;
 static int g_mutexInit = 0;
 static ZSTD_pthread_mutex_t g_mutex;
+static UTIL_time_t g_enableTime = UTIL_TIME_INITIALIZER;
 
 void TRACE_enable(char const* filename)
 {
@@ -54,6 +55,7 @@ void TRACE_enable(char const* filename)
         */
         fprintf(g_traceFile, "Algorithm, Version, Method, Mode, Level, Workers, Dictionary Size, Uncompressed Size, Compressed Size, Duration Nanos, Compression Ratio, Speed MB/s\n");
     }
+    g_enableTime = UTIL_getTime();
     if (!g_mutexInit) {
         if (!ZSTD_pthread_mutex_init(&g_mutex, NULL)) {
             g_mutexInit = 1;
@@ -75,7 +77,7 @@ void TRACE_finish(void)
     }
 }
 
-static void TRACE_log(char const* method, PTime duration, ZSTD_trace const* trace)
+static void TRACE_log(char const* method, PTime duration, ZSTD_Trace const* trace)
 {
     int level = 0;
     int workers = 0;
@@ -87,6 +89,7 @@ static void TRACE_log(char const* method, PTime duration, ZSTD_trace const* trac
     }
     assert(g_traceFile != NULL);
 
+    ZSTD_pthread_mutex_lock(&g_mutex);
     /* Fields:
      * algorithm
      * version
@@ -114,70 +117,47 @@ static void TRACE_log(char const* method, PTime duration, ZSTD_trace const* trac
         (unsigned long long)duration,
         ratio,
         speed);
+    ZSTD_pthread_mutex_unlock(&g_mutex);
 }
 
-static ZSTD_CCtx const* g_cctx;
-static ZSTD_DCtx const* g_dctx;
-static UTIL_time_t g_begin = UTIL_TIME_INITIALIZER;
-
 /**
  * These symbols override the weak symbols provided by the library.
  */
 
-int ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
+ZSTD_TraceCtx ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
 {
-    int enabled = 0;
+    (void)cctx;
     if (g_traceFile == NULL)
         return 0;
-    ZSTD_pthread_mutex_lock(&g_mutex);
-    if (g_cctx == NULL) {
-        g_cctx = cctx;
-        g_dctx = NULL;
-        g_begin = UTIL_getTime();
-        enabled = 1;
-    }
-    ZSTD_pthread_mutex_unlock(&g_mutex);
-    return enabled;
+    return (ZSTD_TraceCtx)UTIL_clockSpanNano(g_enableTime);
 }
 
-void ZSTD_trace_compress_end(ZSTD_CCtx const* cctx, ZSTD_trace const* trace)
+void ZSTD_trace_compress_end(ZSTD_TraceCtx ctx, ZSTD_Trace const* trace)
 {
+    PTime const beginNanos = (PTime)ctx;
+    PTime const endNanos = UTIL_clockSpanNano(g_enableTime);
+    PTime const durationNanos = endNanos > beginNanos ? endNanos - beginNanos : 0;
     assert(g_traceFile != NULL);
-    ZSTD_pthread_mutex_lock(&g_mutex);
-    assert(g_cctx == cctx);
-    assert(g_dctx == NULL);
-    if (cctx == g_cctx && trace->version == ZSTD_VERSION_NUMBER)
-        TRACE_log("compress", UTIL_clockSpanNano(g_begin), trace);
-    g_cctx = NULL;
-    ZSTD_pthread_mutex_unlock(&g_mutex);
+    assert(trace->version == ZSTD_VERSION_NUMBER); /* CLI version must match. */
+    TRACE_log("compress", durationNanos, trace);
 }
 
-int ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
+ZSTD_TraceCtx ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
 {
-    int enabled = 0;
+    (void)dctx;
     if (g_traceFile == NULL)
         return 0;
-    ZSTD_pthread_mutex_lock(&g_mutex);
-    if (g_dctx == NULL) {
-        g_cctx = NULL;
-        g_dctx = dctx;
-        g_begin = UTIL_getTime();
-        enabled = 1;
-    }
-    ZSTD_pthread_mutex_unlock(&g_mutex);
-    return enabled;
+    return (ZSTD_TraceCtx)UTIL_clockSpanNano(g_enableTime);
 }
 
-void ZSTD_trace_decompress_end(ZSTD_DCtx const* dctx, ZSTD_trace const* trace)
+void ZSTD_trace_decompress_end(ZSTD_TraceCtx ctx, ZSTD_Trace const* trace)
 {
+    PTime const beginNanos = (PTime)ctx;
+    PTime const endNanos = UTIL_clockSpanNano(g_enableTime);
+    PTime const durationNanos = endNanos > beginNanos ? endNanos - beginNanos : 0;
     assert(g_traceFile != NULL);
-    ZSTD_pthread_mutex_lock(&g_mutex);
-    assert(g_cctx == NULL);
-    assert(g_dctx == dctx);
-    if (dctx == g_dctx && trace->version == ZSTD_VERSION_NUMBER)
-        TRACE_log("decompress", UTIL_clockSpanNano(g_begin), trace);
-    g_dctx = NULL;
-    ZSTD_pthread_mutex_unlock(&g_mutex);
+    assert(trace->version == ZSTD_VERSION_NUMBER); /* CLI version must match. */
+    TRACE_log("decompress", durationNanos, trace);
 }
 
 #else /* ZSTD_TRACE */

From 552efcac2d82e11165841eef80af0e09c76e934e Mon Sep 17 00:00:00 2001
From: Quentin Carbonneaux <quentin@c9x.me>
Date: Wed, 10 Feb 2021 16:16:54 +0100
Subject: [PATCH 60/80] relocate large arrays from the stack to ldmState_t

---
 lib/compress/zstd_compress.c          |  1 -
 lib/compress/zstd_compress_internal.h | 13 +++++++++++--
 lib/compress/zstd_ldm.c               | 18 ++++++------------
 lib/compress/zstdmt_compress.c        |  2 --
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c90fb320e..2e65a4da5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1627,7 +1627,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
         assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
         assert(params.ldmParams.hashRateLog < 32);
-        zc->ldmState.hashPower = ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
     }
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize));
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 10846ae50..57d3fa075 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -183,13 +183,22 @@ typedef struct {
     U32 checksum;
 } ldmEntry_t;
 
+typedef struct {
+    BYTE const* split;
+    U32 hash;
+    U32 checksum;
+    ldmEntry_t* bucket;
+} ldmMatchCandidate_t;
+
+#define LDM_BATCH_SIZE 64
+
 typedef struct {
     ZSTD_window_t window;   /* State for the window round buffer management */
     ldmEntry_t* hashTable;
     U32 loadedDictEnd;
     BYTE* bucketOffsets;    /* Next position in bucket to insert entry */
-    U64 hashPower;          /* Used to compute the rolling hash.
-                             * Depends on ldmParams.minMatchLength */
+    size_t splitIndices[LDM_BATCH_SIZE];
+    ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE];
 } ldmState_t;
 
 typedef struct {
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 314e4845f..fb4b8a0fd 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -19,7 +19,6 @@
 #define LDM_BUCKET_SIZE_LOG 3
 #define LDM_MIN_MATCH_LENGTH 64
 #define LDM_HASH_RLOG 7
-#define LDM_LOOKAHEAD_SPLITS 64
 
 typedef struct {
     U64 rolling;
@@ -62,8 +61,8 @@ static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const*
  *
  * Registers in the splits array all the split points found in the first
  * size bytes following the data pointer. This function terminates when
- * either all the data has been processed or LDM_LOOKAHEAD_SPLITS splits
- * are present in the splits array.
+ * either all the data has been processed or LDM_BATCH_SIZE splits are
+ * present in the splits array.
  *
  * Precondition: The splits array must not be full.
  * Returns: The number of bytes processed. */
@@ -84,7 +83,7 @@ static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state,
         if (UNLIKELY((hash & mask) == 0)) { \
             splits[*numSplits] = n; \
             *numSplits += 1; \
-            if (*numSplits == LDM_LOOKAHEAD_SPLITS) \
+            if (*numSplits == LDM_BATCH_SIZE) \
                 goto done; \
         } \
     } while (0)
@@ -247,7 +246,7 @@ void ZSTD_ldm_fillHashTable(
     BYTE const* const base = ldmState->window.base;
     BYTE const* const istart = ip;
     ldmRollingHashState_t hashState;
-    size_t splits[LDM_LOOKAHEAD_SPLITS];
+    size_t* const splits = ldmState->splitIndices;
     unsigned numSplits;
 
     DEBUGLOG(5, "ZSTD_ldm_fillHashTable");
@@ -319,13 +318,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
     /* Rolling hash state */
     ldmRollingHashState_t hashState;
     /* Arrays for staged-processing */
-    size_t splits[LDM_LOOKAHEAD_SPLITS];
-    struct {
-        BYTE const* split;
-        U32 hash;
-        U32 checksum;
-        ldmEntry_t* bucket;
-    } candidates[LDM_LOOKAHEAD_SPLITS];
+    size_t* const splits = ldmState->splitIndices;
+    ldmMatchCandidate_t* const candidates = ldmState->matchCandidates;
     unsigned numSplits;
 
     if (srcSize < minMatchLength)
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index c3ead0878..7ad4f6543 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -472,8 +472,6 @@ ZSTDMT_serialState_reset(serialState_t* serialState,
         ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
         assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
         assert(params.ldmParams.hashRateLog < 32);
-        serialState->ldmState.hashPower =
-                ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
     } else {
         ZSTD_memset(&params.ldmParams, 0, sizeof(params.ldmParams));
     }

From 74f85818a6ae3c609ddfb38f0de09dcdf552f88b Mon Sep 17 00:00:00 2001
From: nia <nia@netbsd.org>
Date: Thu, 11 Feb 2021 10:50:11 +0100
Subject: [PATCH 61/80] Use standard md5 tool on NetBSD.

This avoids a GNU coreutils dependency.

-n is used to match the output format of coreutils:
http://man.netbsd.org/md5.1
---
 lib/Makefile       | 2 ++
 programs/Makefile  | 2 ++
 tests/playTests.sh | 1 +
 3 files changed, 5 insertions(+)

diff --git a/lib/Makefile b/lib/Makefile
index 9a3cc0096..39ff304f5 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -179,6 +179,8 @@ ifeq ($(UNAME), Darwin)
   HASH ?= md5
 else ifeq ($(UNAME), FreeBSD)
   HASH ?= gmd5sum
+else ifeq ($(UNAME), NetBSD)
+  HASH ?= md5 -n
 else ifeq ($(UNAME), OpenBSD)
   HASH ?= md5
 endif
diff --git a/programs/Makefile b/programs/Makefile
index 31576ecc9..936f3c804 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -104,6 +104,8 @@ ifeq ($(UNAME), Darwin)
   HASH ?= md5
 else ifeq ($(UNAME), FreeBSD)
   HASH ?= gmd5sum
+else ifeq ($(UNAME), NetBSD)
+  HASH ?= md5 -n
 else ifeq ($(UNAME), OpenBSD)
   HASH ?= md5
 endif
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 20259bc2f..ec1692c21 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -114,6 +114,7 @@ esac
 case "$UNAME" in
   Darwin) MD5SUM="md5 -r" ;;
   FreeBSD) MD5SUM="gmd5sum" ;;
+  NetBSD) MD5SUM="md5 -n" ;;
   OpenBSD) MD5SUM="md5" ;;
   *) MD5SUM="md5sum" ;;
 esac

From f520f6dfbe158ab30c84721652a9100600f6dba0 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 11 Feb 2021 16:20:04 -0800
Subject: [PATCH 62/80] [trace] Minor fixes found during integration

* Mark `ZSTD_CCtx_getParameter()` as const
* Add `extern "C"` guards to `zstd_trace.h`
---
 lib/common/zstd_trace.h      | 8 ++++++++
 lib/compress/zstd_compress.c | 2 +-
 lib/zstd.h                   | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/common/zstd_trace.h b/lib/common/zstd_trace.h
index f2fa7f79e..487617cf6 100644
--- a/lib/common/zstd_trace.h
+++ b/lib/common/zstd_trace.h
@@ -11,6 +11,10 @@
 #ifndef ZSTD_TRACE_H
 #define ZSTD_TRACE_H
 
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
 #include <stddef.h>
 
 /* weak symbol support */
@@ -141,4 +145,8 @@ void ZSTD_trace_decompress_end(
 
 #endif /* ZSTD_TRACE */
 
+#if defined (__cplusplus)
+}
+#endif
+
 #endif /* ZSTD_TRACE_H */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ac11c3ddb..5b054a72e 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -797,7 +797,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
     }
 }
 
-size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value)
+size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value)
 {
     return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value);
 }
diff --git a/lib/zstd.h b/lib/zstd.h
index 230adee34..64265f682 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1837,7 +1837,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
  *  and store it into int* value.
  * @return : 0, or an error code (which can be tested with ZSTD_isError()).
  */
-ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
 
 
 /*! ZSTD_CCtx_params :

From c62eb05964702412ad467916b47ec76cff34b266 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 12 Feb 2021 14:53:07 -0800
Subject: [PATCH 63/80] [lib] Set appliedParams.compressionLevel correctly

Forward the correct compressionLevel to the appliedParams in all cases.
It was already correct for the advanced API, so only the old single-pass
functions needed to be fixed.

This compression level is unused by the library, but is set so that the
tracing framework can consume it.
---
 lib/compress/zstd_compress.c | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 5b054a72e..01f32a514 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -270,6 +270,8 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
     return 0;
 }
 
+#define ZSTD_NO_CLEVEL 0
+
 size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
 {
     RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
@@ -278,20 +280,25 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete
     assert(!ZSTD_checkCParams(params.cParams));
     cctxParams->cParams = params.cParams;
     cctxParams->fParams = params.fParams;
-    cctxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* should not matter, as all cParams are presumed properly defined */
+    cctxParams->compressionLevel = ZSTD_NO_CLEVEL;   /* should not matter, as all cParams are presumed properly defined */
     return 0;
 }
 
 /* ZSTD_assignParamsToCCtxParams() :
- * params is presumed valid at this stage */
+ * params is presumed valid at this stage.
+ * Set compressionLevel = ZSTD_NO_CLEVEL when there is no source compression level.
+ */
 static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(
-        const ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
+        const ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params, int compressionLevel)
 {
     ZSTD_CCtx_params ret = *cctxParams;
     assert(!ZSTD_checkCParams(params->cParams));
     ret.cParams = params->cParams;
     ret.fParams = params->fParams;
-    ret.compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* should not matter, as all cParams are presumed properly defined */
+    /* Should not matter, as all cParams are presumed properly defined.
+     * But, set it for tracing anyway.
+     */
+    ret.compressionLevel = compressionLevel;
     return ret;
 }
 
@@ -3446,7 +3453,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
                                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     ZSTD_CCtx_params const cctxParams =
-            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params);
+            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params, ZSTD_NO_CLEVEL);
     return ZSTD_compressBegin_advanced_internal(cctx,
                                             dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast,
                                             NULL /*cdict*/,
@@ -3457,7 +3464,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
 {
     ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
     ZSTD_CCtx_params const cctxParams =
-            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params);
+            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
     DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
     return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
                                        &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
@@ -3568,7 +3575,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
                                 const ZSTD_parameters* params)
 {
     ZSTD_CCtx_params const cctxParams =
-            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params);
+            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params, ZSTD_NO_CLEVEL);
     DEBUGLOG(4, "ZSTD_compress_internal");
     return ZSTD_compress_advanced_internal(cctx,
                                            dst, dstCapacity,
@@ -3614,7 +3621,7 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
                                int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
-    ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params);
+    ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
     DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
     assert(params.fParams.contentSizeFlag == 1);
     return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams);
@@ -3763,7 +3770,7 @@ static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
         assert(cdict != NULL);
         ZSTD_cwksp_move(&cdict->workspace, &ws);
         cdict->customMem = customMem;
-        cdict->compressionLevel = 0; /* signals advanced API usage */
+        cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
 
         return cdict;
     }
@@ -3949,6 +3956,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
     RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!");
     {   ZSTD_CCtx_params params = cctx->requestedParams;
+        params.compressionLevel = cdict->compressionLevel;
         params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
                         || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
                         || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
@@ -4136,7 +4144,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
     FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
     FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
     FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
-    zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, &params);
+    zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, &params, ZSTD_NO_CLEVEL);
     FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
     return 0;
 }

From 7736549beaee6408969dbcce478e2e3760483f3c Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 12 Feb 2021 18:52:08 -0800
Subject: [PATCH 64/80] [bug-fix] Make simple single-pass functions ignore
 advanced parameters

The simple compression functions are intended to ignore the advanced
parameters, but they were accidentally using them. All the
`ZSTD_parameters` were set correctly, but any extra parameters were
used as-is. E.g. `ZSTD_c_format`.

This PR makes all the simple single-pass functions listed below ignore
the advanced parameters, as intended.

* `ZSTD_compressCCtx()`
* `ZSTD_compress_usingDict()`
* `ZSTD_compress_usingCDict()`
* `ZSTD_compress_advanced()`
* `ZSTD_compress_usingCDict_advanced()`

It also adds a test case that ensures that each of these functions
ignore the advanced parameters.
---
 lib/compress/zstd_compress.c | 131 ++++++++++++++++++-----------------
 tests/fuzzer.c               |  35 ++++++++++
 2 files changed, 102 insertions(+), 64 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 01f32a514..93c4075c5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -272,34 +272,44 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
 
 #define ZSTD_NO_CLEVEL 0
 
+/**
+ * Initializes the cctxParams from params and compressionLevel.
+ * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL.
+ */
+static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel)
+{
+    assert(!ZSTD_checkCParams(params->cParams));
+    ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
+    cctxParams->cParams = params->cParams;
+    cctxParams->fParams = params->fParams;
+    /* Should not matter, as all cParams are presumed properly defined.
+     * But, set it for tracing anyway.
+     */
+    cctxParams->compressionLevel = compressionLevel;
+}
+
 size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
 {
     RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
     FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
-    ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
-    assert(!ZSTD_checkCParams(params.cParams));
-    cctxParams->cParams = params.cParams;
-    cctxParams->fParams = params.fParams;
-    cctxParams->compressionLevel = ZSTD_NO_CLEVEL;   /* should not matter, as all cParams are presumed properly defined */
+    ZSTD_CCtxParams_init_internal(cctxParams, &params, ZSTD_NO_CLEVEL);
     return 0;
 }
 
-/* ZSTD_assignParamsToCCtxParams() :
- * params is presumed valid at this stage.
- * Set compressionLevel = ZSTD_NO_CLEVEL when there is no source compression level.
+/**
+ * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone.
+ * @param param Validated zstd parameters.
  */
-static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(
-        const ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params, int compressionLevel)
+static void ZSTD_CCtxParams_setZstdParams(
+        ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
 {
-    ZSTD_CCtx_params ret = *cctxParams;
     assert(!ZSTD_checkCParams(params->cParams));
-    ret.cParams = params->cParams;
-    ret.fParams = params->fParams;
+    cctxParams->cParams = params->cParams;
+    cctxParams->fParams = params->fParams;
     /* Should not matter, as all cParams are presumed properly defined.
      * But, set it for tracing anyway.
      */
-    ret.compressionLevel = compressionLevel;
-    return ret;
+    cctxParams->compressionLevel = ZSTD_NO_CLEVEL;
 }
 
 ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
@@ -3452,8 +3462,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
                                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    ZSTD_CCtx_params const cctxParams =
-            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params, ZSTD_NO_CLEVEL);
+    ZSTD_CCtx_params cctxParams;
+    ZSTD_CCtxParams_init_internal(&cctxParams, &params, ZSTD_NO_CLEVEL);
     return ZSTD_compressBegin_advanced_internal(cctx,
                                             dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast,
                                             NULL /*cdict*/,
@@ -3462,9 +3472,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
-    ZSTD_CCtx_params const cctxParams =
-            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
+    ZSTD_CCtx_params cctxParams;
+    {
+        ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
+        ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
+    }
     DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
     return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
                                        &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
@@ -3568,35 +3580,21 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     return cSize + endResult;
 }
 
-static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
-                                      void* dst, size_t dstCapacity,
-                                const void* src, size_t srcSize,
-                                const void* dict,size_t dictSize,
-                                const ZSTD_parameters* params)
-{
-    ZSTD_CCtx_params const cctxParams =
-            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params, ZSTD_NO_CLEVEL);
-    DEBUGLOG(4, "ZSTD_compress_internal");
-    return ZSTD_compress_advanced_internal(cctx,
-                                           dst, dstCapacity,
-                                           src, srcSize,
-                                           dict, dictSize,
-                                           &cctxParams);
-}
-
 size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
                                void* dst, size_t dstCapacity,
                          const void* src, size_t srcSize,
                          const void* dict,size_t dictSize,
                                ZSTD_parameters params)
 {
+    ZSTD_CCtx_params cctxParams;
     DEBUGLOG(4, "ZSTD_compress_advanced");
     FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
-    return ZSTD_compress_internal(cctx,
-                                  dst, dstCapacity,
-                                  src, srcSize,
-                                  dict, dictSize,
-                                  &params);
+    ZSTD_CCtxParams_init_internal(&cctxParams, &params, ZSTD_NO_CLEVEL);
+    return ZSTD_compress_advanced_internal(cctx,
+                                           dst, dstCapacity,
+                                           src, srcSize,
+                                           dict, dictSize,
+                                           &cctxParams);
 }
 
 /* Internal */
@@ -3620,10 +3618,13 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
                          const void* dict, size_t dictSize,
                                int compressionLevel)
 {
-    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
-    ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
+    ZSTD_CCtx_params cctxParams;
+    {
+        ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
+        assert(params.fParams.contentSizeFlag == 1);
+        ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
+    }
     DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
-    assert(params.fParams.contentSizeFlag == 1);
     return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams);
 }
 
@@ -3953,35 +3954,37 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
     ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
+    ZSTD_CCtx_params cctxParams;
     DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
     RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!");
-    {   ZSTD_CCtx_params params = cctx->requestedParams;
-        params.compressionLevel = cdict->compressionLevel;
+    /* Initialize the cctxParams from the cdict */
+    {
+        ZSTD_parameters params;
+        params.fParams = fParams;
         params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
                         || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
                         || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
-                        || cdict->compressionLevel == 0 )
-                      && (params.attachDictPref != ZSTD_dictForceLoad) ?
+                        || cdict->compressionLevel == 0 ) ?
                 ZSTD_getCParamsFromCDict(cdict)
               : ZSTD_getCParams(cdict->compressionLevel,
                                 pledgedSrcSize,
                                 cdict->dictContentSize);
-        /* Increase window log to fit the entire dictionary and source if the
-         * source size is known. Limit the increase to 19, which is the
-         * window log for compression level 1 with the largest source size.
-         */
-        if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
-            U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
-            U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
-            params.cParams.windowLog = MAX(params.cParams.windowLog, limitedSrcLog);
-        }
-        params.fParams = fParams;
-        return ZSTD_compressBegin_internal(cctx,
-                                           NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast,
-                                           cdict,
-                                           &params, pledgedSrcSize,
-                                           ZSTDb_not_buffered);
+        ZSTD_CCtxParams_init_internal(&cctxParams, &params, cdict->compressionLevel);
     }
+    /* Increase window log to fit the entire dictionary and source if the
+     * source size is known. Limit the increase to 19, which is the
+     * window log for compression level 1 with the largest source size.
+     */
+    if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+        U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
+        U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
+        cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog);
+    }
+    return ZSTD_compressBegin_internal(cctx,
+                                        NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast,
+                                        cdict,
+                                        &cctxParams, pledgedSrcSize,
+                                        ZSTDb_not_buffered);
 }
 
 /* ZSTD_compressBegin_usingCDict() :
@@ -4144,7 +4147,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
     FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
     FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
     FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
-    zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, &params, ZSTD_NO_CLEVEL);
+    ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, &params);
     FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
     return 0;
 }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 157b798ac..5f707e027 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -675,6 +675,41 @@ static int basicUnitTests(U32 const seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    {
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
+        ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
+        CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
+        if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
+        CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
+        if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
+        CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
+        if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
+        CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
+        if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
+        CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
+        if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
+        DISPLAYLEVEL(3, "OK \n");
+
+        ZSTD_freeCDict(cdict);
+        ZSTD_freeCCtx(cctx);
+    }
+
     DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
     {
         ZSTD_CCtx* const cctx = ZSTD_createCCtx();

From 444c4650a0ddb23cf537c934031c3a2e036b16b7 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 17 Feb 2021 12:26:38 -0500
Subject: [PATCH 65/80] Add newline to end of cli help message

---
 programs/zstdcli.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index c4e1b4792..f263e969f 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -191,6 +191,7 @@ static void usage_advanced(const char* programName)
 #endif /* ZSTD_NOCOMPRESS */
 
 #ifndef ZSTD_NOTRACE
+    DISPLAYOUT( "\n");
     DISPLAYOUT( "--trace FILE : log tracing information to FILE.");
 #endif
     DISPLAYOUT( "\n");

From a774c5797399040af62db21d8a9b9769e005430e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 11 Feb 2021 15:50:13 -0500
Subject: [PATCH 66/80] Use umask() to Constrain Created File Permissions

This commit addresses #2491.

Note that a downside of this solution is that it is global: `umask()` affects
all file creation calls in the process. I believe this is safe since
`fileio.c` functions should only ever be used in the zstd binary, and these
are (almost) the only files ever created by zstd, and AIUI they're only
created in a single thread. So we can get away with messing with global state.

Note that this doesn't change the permissions of files created by `dibio.c`.
I'm not sure what those should be...
---
 programs/fileio.c | 9 +++------
 programs/util.c   | 9 +++++++++
 programs/util.h   | 7 ++++++-
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 51956f681..30a2879a7 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -679,14 +679,11 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
             FIO_removeFile(dstFileName);
     }   }
 
-    {   FILE* const f = fopen( dstFileName, "wb" );
+    {   const int old_umask = UTIL_umask(0177); /* u-x,go-rwx */
+        FILE* const f = fopen( dstFileName, "wb" );
+        UTIL_umask(old_umask);
         if (f == NULL) {
             DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
-        } else if (srcFileName != NULL
-               && strcmp (srcFileName, stdinmark)
-               && strcmp(dstFileName, nulmark) ) {
-            /* reduce rights on newly created dst file while compression is ongoing */
-            UTIL_chmod(dstFileName, NULL, 00600);
         }
         return f;
     }
diff --git a/programs/util.c b/programs/util.c
index 460d9bf11..7208d66d2 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -159,6 +159,15 @@ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
     return chmod(filename, permissions);
 }
 
+int UTIL_umask(int mode) {
+#if PLATFORM_POSIX_VERSION > 0
+    return umask(mode);
+#else
+    /* do nothing, fake return value */
+    return mode;
+#endif
+}
+
 int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
 {
     int res = 0;
diff --git a/programs/util.h b/programs/util.h
index d2077c9ac..0e696f003 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -22,7 +22,7 @@ extern "C" {
 #include "platform.h"     /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */
 #include <stddef.h>       /* size_t, ptrdiff_t */
 #include <sys/types.h>    /* stat, utime */
-#include <sys/stat.h>     /* stat, chmod */
+#include <sys/stat.h>     /* stat, chmod, umask */
 #include "../lib/common/mem.h"          /* U64 */
 
 
@@ -152,6 +152,11 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf);
  */
 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions);
 
+/**
+ * Wraps umask(). Does nothing when the platform doesn't have that concept.
+ */
+int UTIL_umask(int mode);
+
 /*
  * In the absence of a pre-existing stat result on the file in question, these
  * functions will do a stat() call internally and then use that result to

From 9b0772177c51a8296467e9b57a8fcdf4f38f4909 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 19 Feb 2021 10:42:05 -0800
Subject: [PATCH 67/80] fix ubsan test errors

allows recovering from pointer overflow
---
 .github/workflows/generic-dev.yml | 2 +-
 Makefile                          | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml
index 1632101c7..8d9fd839a 100644
--- a/.github/workflows/generic-dev.yml
+++ b/.github/workflows/generic-dev.yml
@@ -81,7 +81,7 @@ jobs:
         make libc6install
         CFLAGS="-Werror -m32" make -j all32
         make clean
-        make -j uasan-test-zstd32
+        make -j uasan-test-zstd32 V=1
 
   clang-msan-testzstd:
     runs-on: ubuntu-latest
diff --git a/Makefile b/Makefile
index 9f94baaa1..d2bc63027 100644
--- a/Makefile
+++ b/Makefile
@@ -309,10 +309,10 @@ asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests

From 91e6480458df7b90874cccf27c8a55e1db9311af Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 19 Feb 2021 13:11:29 -0800
Subject: [PATCH 68/80] [fuzz] Fix compiler detection & update ubsan flags

* Fix compiler version regex, which was broken for multi-digit
  versions.
* Fix compiler detection for gcc.
* Disable `pointer-overflow` instead of `integer-overflow` for gcc
  versions newer than 8.0.0.
---
 tests/fuzz/fuzz.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index b3aa2bdf6..05a778738 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -180,14 +180,15 @@ def compiler_version(cc, cxx):
     cxx_version_bytes = subprocess.check_output([cxx, "--version"])
     compiler = None
     version = None
+    print("{} --version:\n{}".format(cc, cc_version_bytes.decode('ascii')))
     if b'clang' in cc_version_bytes:
         assert(b'clang' in cxx_version_bytes)
         compiler = 'clang'
-    elif b'gcc' in cc_version_bytes:
+    elif b'gcc' in cc_version_bytes or b'GCC' in cc_version_bytes:
         assert(b'gcc' in cxx_version_bytes or b'g++' in cxx_version_bytes)
         compiler = 'gcc'
     if compiler is not None:
-        version_regex = b'([0-9])+\.([0-9])+\.([0-9])+'
+        version_regex = b'([0-9]+)\.([0-9]+)\.([0-9]+)'
         version_match = re.search(version_regex, cc_version_bytes)
         version = tuple(int(version_match.group(i)) for i in range(1, 4))
     return compiler, version
@@ -195,9 +196,9 @@ def compiler_version(cc, cxx):
 
 def overflow_ubsan_flags(cc, cxx):
     compiler, version = compiler_version(cc, cxx)
-    if compiler == 'gcc':
+    if compiler == 'gcc' and version < (8, 0, 0):
         return ['-fno-sanitize=signed-integer-overflow']
-    if compiler == 'clang' and version >= (5, 0, 0):
+    if compiler == 'gcc' or (compiler == 'clang' and version >= (5, 0, 0)):
         return ['-fno-sanitize=pointer-overflow']
     return []
 

From f2c0312889db7c467fc7a91fa8a70143de2ef208 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 19 Feb 2021 16:30:06 -0800
Subject: [PATCH 69/80] removing signed integer overflow exception from ubsan
 tests

---
 Makefile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index d2bc63027..8522d9d5e 100644
--- a/Makefile
+++ b/Makefile
@@ -286,12 +286,11 @@ uasanregressiontest:
 msanregressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
 
-# run UBsan with -fsanitize-recover=signed-integer-overflow
-# due to a bug in UBsan when doing pointer subtraction
-# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
+# run UBsan with -fsanitize-recover=pointer-overflow
+# this only works with recent compilers such as gcc 8+
 
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
 
 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
@@ -309,10 +308,10 @@ asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests

From 08fef036eca760ce04d47825db9480dc756455a6 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 23 Feb 2021 10:40:44 -0800
Subject: [PATCH 70/80] update FreeBSD image to latest 12.2

---
 .cirrus.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 8387ca1b1..fe17aacea 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -2,7 +2,7 @@ task:
   name: FreeBSD (shortest)
   freebsd_instance:
     matrix:
-      image_family: freebsd-12-1
+      image_family: freebsd-12-2
       # The stable 11.3 image causes "Agent is not responding" so use a snapshot
       image_family: freebsd-11-3-snap
   install_script: pkg install -y gmake coreutils

From adb54293ab6fce33afb1a01fe79f38ad3788026e Mon Sep 17 00:00:00 2001
From: Stephen Kitt <steve@sk2.org>
Date: Sat, 20 Feb 2021 17:28:19 +0100
Subject: [PATCH 71/80] Stop using deprecated reset?Stream functions

These are replaced by the corresponding context resets. When
converting resetCStream, CCtx_setPledgedSrcSize isn't called if the
source size is "unknown".

This helps reduce the reliance on "static only" symbols, as well as
reducing the use of deprecated functions.

Signed-off-by: Stephen Kitt <steve@sk2.org>
---
 contrib/pzstd/Pzstd.cpp                       |  4 ++--
 contrib/seekable_format/zstdseek_compress.c   |  2 +-
 contrib/seekable_format/zstdseek_decompress.c |  2 +-
 programs/fileio.c                             |  2 +-
 zlibWrapper/examples/zwrapbench.c             | 10 ++++++----
 zlibWrapper/zstd_zlibwrapper.c                | 14 ++++++++++----
 6 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index ce142ad84..2c09bda7a 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -274,7 +274,7 @@ static void compress(
     return;
   }
   {
-    auto err = ZSTD_resetCStream(ctx.get(), 0);
+    auto err = ZSTD_CCtx_reset(ctx.get(), ZSTD_reset_session_only);
     if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
       return;
     }
@@ -432,7 +432,7 @@ static void decompress(
     return;
   }
   {
-    auto err = ZSTD_resetDStream(ctx.get());
+    auto err = ZSTD_DCtx_reset(ctx.get(), ZSTD_reset_session_only);
     if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
       return;
     }
diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c
index 5a75714fa..3db291887 100644
--- a/contrib/seekable_format/zstdseek_compress.c
+++ b/contrib/seekable_format/zstdseek_compress.c
@@ -223,7 +223,7 @@ size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output
     zcs->frameCSize = 0;
     zcs->frameDSize = 0;
 
-    ZSTD_resetCStream(zcs->cstream, 0);
+    ZSTD_CCtx_reset(zcs->cstream, ZSTD_reset_session_only);
     if (zcs->framelog.checksumFlag)
         XXH64_reset(&zcs->xxhState, 0);
 
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index cc5c85950..4f76158e2 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -394,7 +394,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
                                   SEEK_SET));
             zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0};
             XXH64_reset(&zs->xxhState, 0);
-            ZSTD_resetDStream(zs->dstream);
+            ZSTD_DCtx_reset(zs->dstream, ZSTD_reset_session_only);
         }
 
         while (zs->decompressedOffset < offset + len) {
diff --git a/programs/fileio.c b/programs/fileio.c
index 30a2879a7..1f1cbb9e3 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -2109,7 +2109,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput,
         if (srcFileLength>20) srcFileName += srcFileLength-20;
     }
 
-    ZSTD_resetDStream(ress->dctx);
+    ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only);
 
     /* Header loading : ensures ZSTD_getFrameHeader() will succeed */
     {   size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX;
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index f30cad40c..127f6114b 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -270,8 +270,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                            rSize = ZSTD_resetCStream(zbc, blockTable[blockNb].srcSize);
-                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetCStream() failed : %s", ZSTD_getErrorName(rSize));
+                            rSize = ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only);
+                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_CCtx_reset() failed : %s", ZSTD_getErrorName(rSize));
+                            rSize = ZSTD_CCtx_setPledgedSrcSize(zbc, blockTable[blockNb].srcSize);
+                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_CCtx_setPledgedSrcSize() failed : %s", ZSTD_getErrorName(rSize));
                             inBuffer.src = blockTable[blockNb].srcPtr;
                             inBuffer.size = blockTable[blockNb].srcSize;
                             inBuffer.pos = 0;
@@ -418,8 +420,8 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                            rSize = ZSTD_resetDStream(zbd);
-                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetDStream() failed : %s", ZSTD_getErrorName(rSize));
+                            rSize = ZSTD_DCtx_reset(zbd, ZSTD_reset_session_only);
+                            if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_reset() failed : %s", ZSTD_getErrorName(rSize));
                             inBuffer.src = blockTable[blockNb].cPtr;
                             inBuffer.size = blockTable[blockNb].cSize;
                             inBuffer.pos = 0;
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index a4867c20b..b73bf5951 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -372,9 +372,15 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
     } else {
         if (zwc->totalInBytes == 0) {
             if (zwc->comprState == ZWRAP_useReset) {
-                size_t const resetErr = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
+                size_t resetErr = ZSTD_CCtx_reset(zwc->zbc, ZSTD_reset_session_only);
                 if (ZSTD_isError(resetErr)) {
-                    LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n",
+                    LOG_WRAPPERC("ERROR: ZSTD_CCtx_reset errorCode=%s\n",
+                                ZSTD_getErrorName(resetErr));
+                    return ZWRAPC_finishWithError(zwc, strm, 0);
+                }
+                resetErr = ZSTD_CCtx_setPledgedSrcSize(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
+                if (ZSTD_isError(resetErr)) {
+                    LOG_WRAPPERC("ERROR: ZSTD_CCtx_setPledgedSrcSize errorCode=%s\n",
                                 ZSTD_getErrorName(resetErr));
                     return ZWRAPC_finishWithError(zwc, strm, 0);
                 }
@@ -829,7 +835,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                     goto error;
                 }
             } else {
-                size_t const resetErr = ZSTD_resetDStream(zwd->zbd);
+                size_t const resetErr = ZSTD_DCtx_reset(zwd->zbd, ZSTD_reset_session_only);
                 if (ZSTD_isError(resetErr)) goto error;
             }
         } else {
@@ -849,7 +855,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                     goto error;
                 }
             } else {
-                size_t const resetErr = ZSTD_resetDStream(zwd->zbd);
+                size_t const resetErr = ZSTD_DCtx_reset(zwd->zbd, ZSTD_reset_session_only);
                 if (ZSTD_isError(resetErr)) goto error;
             }
 

From 3d91ab74a151ba12c38edb5cdcaf180e1fd38a80 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 23 Feb 2021 13:25:08 -0800
Subject: [PATCH 72/80] disabled icc tests on Github Actions

---
 .github/workflows/generic-dev.yml | 40 +++++++++++++++++--------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml
index 8d9fd839a..d6e9b44b4 100644
--- a/.github/workflows/generic-dev.yml
+++ b/.github/workflows/generic-dev.yml
@@ -190,21 +190,25 @@ jobs:
         tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
         shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
 
-  icc:
-    name: icc-check
-    runs-on: ubuntu-latest
-    steps:
-    - name: install icc
-      run: |
-        export DEBIAN_FRONTEND=noninteractive
-        sudo apt-get -qqq update
-        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
-        sudo apt-get update
-        sudo apt-get install -y intel-basekit intel-hpckit
-    - uses: actions/checkout@v2
-    - name: make check
-      run: |
-        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
+# For reference : icc tests
+# icc tests are currently failing on Github Actions, likely to issues during installation stage
+# To be fixed later
+#
+#  icc:
+#    name: icc-check
+#    runs-on: ubuntu-latest
+#    steps:
+#    - name: install icc
+#      run: |
+#        export DEBIAN_FRONTEND=noninteractive
+#        sudo apt-get -qqq update
+#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
+#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+#        sudo apt-get update
+#        sudo apt-get install -y intel-basekit intel-hpckit
+#    - uses: actions/checkout@v2
+#    - name: make check
+#      run: |
+#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check

From 59b2c596d7b227a7afd3cdfd374fd2c0baabcc19 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 23 Feb 2021 15:15:39 -0800
Subject: [PATCH 73/80] [regression] Update results.csv

9f327c02fd17a5aad2b24ae06b85d8226add1f93 changed the compression method
for LDM, so the results are slightly different.

I've re-tested LDM on some larger inputs and everything seems fine.
These ratio changes just seem to be noise. There is generally a 0.01%
swing in ratio, sometimes better sometimes worse, but never large.
---
 tests/regression/results.csv | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 0f818149b..6d5f3d376 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -111,9 +111,9 @@ silesia,                            level 9,                            zstdcli,
 silesia,                            level 13,                           zstdcli,                            4482183
 silesia,                            level 16,                           zstdcli,                            4377513
 silesia,                            level 19,                           zstdcli,                            4293378
-silesia,                            long distance mode,                 zstdcli,                            4839756
+silesia,                            long distance mode,                 zstdcli,                            4840792
 silesia,                            multithreaded,                      zstdcli,                            4849600
-silesia,                            multithreaded long distance mode,   zstdcli,                            4839756
+silesia,                            multithreaded long distance mode,   zstdcli,                            4840792
 silesia,                            small window log,                   zstdcli,                            7111012
 silesia,                            small hash log,                     zstdcli,                            6555069
 silesia,                            small chain log,                    zstdcli,                            4931196
@@ -137,9 +137,9 @@ silesia.tar,                        level 13,                           zstdcli,
 silesia.tar,                        level 16,                           zstdcli,                            4381336
 silesia.tar,                        level 19,                           zstdcli,                            4281609
 silesia.tar,                        no source size,                     zstdcli,                            4861508
-silesia.tar,                        long distance mode,                 zstdcli,                            4853190
+silesia.tar,                        long distance mode,                 zstdcli,                            4853153
 silesia.tar,                        multithreaded,                      zstdcli,                            4861512
-silesia.tar,                        multithreaded long distance mode,   zstdcli,                            4853190
+silesia.tar,                        multithreaded long distance mode,   zstdcli,                            4853153
 silesia.tar,                        small window log,                   zstdcli,                            7101576
 silesia.tar,                        small hash log,                     zstdcli,                            6587959
 silesia.tar,                        small chain log,                    zstdcli,                            4943310
@@ -217,9 +217,9 @@ github.tar,                         level 19,                           zstdcli,
 github.tar,                         level 19 with dict,                 zstdcli,                            32899
 github.tar,                         no source size,                     zstdcli,                            38442
 github.tar,                         no source size with dict,           zstdcli,                            38004
-github.tar,                         long distance mode,                 zstdcli,                            39680
+github.tar,                         long distance mode,                 zstdcli,                            39726
 github.tar,                         multithreaded,                      zstdcli,                            38445
-github.tar,                         multithreaded long distance mode,   zstdcli,                            39680
+github.tar,                         multithreaded long distance mode,   zstdcli,                            39726
 github.tar,                         small window log,                   zstdcli,                            199432
 github.tar,                         small hash log,                     zstdcli,                            129874
 github.tar,                         small chain log,                    zstdcli,                            41673

From 04139c3ff213fe84ba5c13b8f5ccce3c3086e72b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 24 Feb 2021 19:11:38 -0800
Subject: [PATCH 74/80] [regression] Update results.csv

Fixes the update from PR #2508. I had accidentally forgotten to rebuild
the library, and the regression test suite isn't hooked up to the new
fancy build system yet.

I've double checked that the results are deterministic.
---
 tests/regression/results.csv | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 6d5f3d376..0fb1d160a 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -243,9 +243,9 @@ silesia,                            level 13,                           advanced
 silesia,                            level 16,                           advanced one pass,                  4377465
 silesia,                            level 19,                           advanced one pass,                  4293330
 silesia,                            no source size,                     advanced one pass,                  4849552
-silesia,                            long distance mode,                 advanced one pass,                  4839708
+silesia,                            long distance mode,                 advanced one pass,                  4840744
 silesia,                            multithreaded,                      advanced one pass,                  4849552
-silesia,                            multithreaded long distance mode,   advanced one pass,                  4839708
+silesia,                            multithreaded long distance mode,   advanced one pass,                  4840744
 silesia,                            small window log,                   advanced one pass,                  7095919
 silesia,                            small hash log,                     advanced one pass,                  6555021
 silesia,                            small chain log,                    advanced one pass,                  4931148
@@ -269,9 +269,9 @@ silesia.tar,                        level 13,                           advanced
 silesia.tar,                        level 16,                           advanced one pass,                  4381332
 silesia.tar,                        level 19,                           advanced one pass,                  4281605
 silesia.tar,                        no source size,                     advanced one pass,                  4861425
-silesia.tar,                        long distance mode,                 advanced one pass,                  4848098
+silesia.tar,                        long distance mode,                 advanced one pass,                  4847735
 silesia.tar,                        multithreaded,                      advanced one pass,                  4861508
-silesia.tar,                        multithreaded long distance mode,   advanced one pass,                  4853186
+silesia.tar,                        multithreaded long distance mode,   advanced one pass,                  4853149
 silesia.tar,                        small window log,                   advanced one pass,                  7101530
 silesia.tar,                        small hash log,                     advanced one pass,                  6587951
 silesia.tar,                        small chain log,                    advanced one pass,                  4943307
@@ -351,9 +351,9 @@ github.tar,                         level 19,                           advanced
 github.tar,                         level 19 with dict,                 advanced one pass,                  32895
 github.tar,                         no source size,                     advanced one pass,                  38441
 github.tar,                         no source size with dict,           advanced one pass,                  37995
-github.tar,                         long distance mode,                 advanced one pass,                  39676
+github.tar,                         long distance mode,                 advanced one pass,                  39722
 github.tar,                         multithreaded,                      advanced one pass,                  38441
-github.tar,                         multithreaded long distance mode,   advanced one pass,                  39676
+github.tar,                         multithreaded long distance mode,   advanced one pass,                  39722
 github.tar,                         small window log,                   advanced one pass,                  198540
 github.tar,                         small hash log,                     advanced one pass,                  129870
 github.tar,                         small chain log,                    advanced one pass,                  41669
@@ -377,9 +377,9 @@ silesia,                            level 13,                           advanced
 silesia,                            level 16,                           advanced one pass small out,        4377465
 silesia,                            level 19,                           advanced one pass small out,        4293330
 silesia,                            no source size,                     advanced one pass small out,        4849552
-silesia,                            long distance mode,                 advanced one pass small out,        4839708
+silesia,                            long distance mode,                 advanced one pass small out,        4840744
 silesia,                            multithreaded,                      advanced one pass small out,        4849552
-silesia,                            multithreaded long distance mode,   advanced one pass small out,        4839708
+silesia,                            multithreaded long distance mode,   advanced one pass small out,        4840744
 silesia,                            small window log,                   advanced one pass small out,        7095919
 silesia,                            small hash log,                     advanced one pass small out,        6555021
 silesia,                            small chain log,                    advanced one pass small out,        4931148
@@ -403,9 +403,9 @@ silesia.tar,                        level 13,                           advanced
 silesia.tar,                        level 16,                           advanced one pass small out,        4381332
 silesia.tar,                        level 19,                           advanced one pass small out,        4281605
 silesia.tar,                        no source size,                     advanced one pass small out,        4861425
-silesia.tar,                        long distance mode,                 advanced one pass small out,        4848098
+silesia.tar,                        long distance mode,                 advanced one pass small out,        4847735
 silesia.tar,                        multithreaded,                      advanced one pass small out,        4861508
-silesia.tar,                        multithreaded long distance mode,   advanced one pass small out,        4853186
+silesia.tar,                        multithreaded long distance mode,   advanced one pass small out,        4853149
 silesia.tar,                        small window log,                   advanced one pass small out,        7101530
 silesia.tar,                        small hash log,                     advanced one pass small out,        6587951
 silesia.tar,                        small chain log,                    advanced one pass small out,        4943307
@@ -485,9 +485,9 @@ github.tar,                         level 19,                           advanced
 github.tar,                         level 19 with dict,                 advanced one pass small out,        32895
 github.tar,                         no source size,                     advanced one pass small out,        38441
 github.tar,                         no source size with dict,           advanced one pass small out,        37995
-github.tar,                         long distance mode,                 advanced one pass small out,        39676
+github.tar,                         long distance mode,                 advanced one pass small out,        39722
 github.tar,                         multithreaded,                      advanced one pass small out,        38441
-github.tar,                         multithreaded long distance mode,   advanced one pass small out,        39676
+github.tar,                         multithreaded long distance mode,   advanced one pass small out,        39722
 github.tar,                         small window log,                   advanced one pass small out,        198540
 github.tar,                         small hash log,                     advanced one pass small out,        129870
 github.tar,                         small chain log,                    advanced one pass small out,        41669
@@ -511,9 +511,9 @@ silesia,                            level 13,                           advanced
 silesia,                            level 16,                           advanced streaming,                 4377465
 silesia,                            level 19,                           advanced streaming,                 4293330
 silesia,                            no source size,                     advanced streaming,                 4849516
-silesia,                            long distance mode,                 advanced streaming,                 4839708
+silesia,                            long distance mode,                 advanced streaming,                 4840744
 silesia,                            multithreaded,                      advanced streaming,                 4849552
-silesia,                            multithreaded long distance mode,   advanced streaming,                 4839708
+silesia,                            multithreaded long distance mode,   advanced streaming,                 4840744
 silesia,                            small window log,                   advanced streaming,                 7112062
 silesia,                            small hash log,                     advanced streaming,                 6555021
 silesia,                            small chain log,                    advanced streaming,                 4931148
@@ -537,9 +537,9 @@ silesia.tar,                        level 13,                           advanced
 silesia.tar,                        level 16,                           advanced streaming,                 4381350
 silesia.tar,                        level 19,                           advanced streaming,                 4281562
 silesia.tar,                        no source size,                     advanced streaming,                 4861423
-silesia.tar,                        long distance mode,                 advanced streaming,                 4848098
+silesia.tar,                        long distance mode,                 advanced streaming,                 4847735
 silesia.tar,                        multithreaded,                      advanced streaming,                 4861508
-silesia.tar,                        multithreaded long distance mode,   advanced streaming,                 4853186
+silesia.tar,                        multithreaded long distance mode,   advanced streaming,                 4853149
 silesia.tar,                        small window log,                   advanced streaming,                 7118769
 silesia.tar,                        small hash log,                     advanced streaming,                 6587952
 silesia.tar,                        small chain log,                    advanced streaming,                 4943312
@@ -619,9 +619,9 @@ github.tar,                         level 19,                           advanced
 github.tar,                         level 19 with dict,                 advanced streaming,                 32895
 github.tar,                         no source size,                     advanced streaming,                 38438
 github.tar,                         no source size with dict,           advanced streaming,                 38000
-github.tar,                         long distance mode,                 advanced streaming,                 39676
+github.tar,                         long distance mode,                 advanced streaming,                 39722
 github.tar,                         multithreaded,                      advanced streaming,                 38441
-github.tar,                         multithreaded long distance mode,   advanced streaming,                 39676
+github.tar,                         multithreaded long distance mode,   advanced streaming,                 39722
 github.tar,                         small window log,                   advanced streaming,                 199558
 github.tar,                         small hash log,                     advanced streaming,                 129870
 github.tar,                         small chain log,                    advanced streaming,                 41669

From 61db590ad857ef145de45fbfdd5513eb45077b37 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Feb 2021 12:29:42 -0500
Subject: [PATCH 75/80] Detect `..` in Paths Correctly

This commit addresses #2509.
---
 programs/util.c | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/programs/util.c b/programs/util.c
index 7208d66d2..3fd4cd17e 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -679,7 +679,27 @@ const char* UTIL_getFileExtension(const char* infilename)
 
 static int pathnameHas2Dots(const char *pathname)
 {
-    return NULL != strstr(pathname, "..");
+    /* We need to figure out whether any ".." present in the path is a whole
+     * path token, which is the case if it is bordered on both sides by either
+     * the beginning/end of the path or by a directory separator.
+     */
+    const char *needle = pathname;
+    while (1) {
+        needle = strstr(needle, "..");
+
+        if (needle == NULL) {
+            return 0;
+        }
+
+        if ((needle == pathname || needle[-1] == PATH_SEP)
+         && (needle[2] == '\0' || needle[2] == PATH_SEP)) {
+            return 1;
+        }
+
+        /* increment so we search for the next match */
+        needle++;
+    };
+    return 0;
 }
 
 static int isFileNameValidForMirroredOutput(const char *filename)

From 9b7f9d26d5cefe2f9208950e15866b69525e9e46 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Feb 2021 13:01:20 -0500
Subject: [PATCH 76/80] Cover These Edge Cases in Tests

---
 tests/playTests.sh | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index ec1692c21..8550badeb 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -486,23 +486,29 @@ rm -rf tmp*
 if [ "$isWindows" = false ] ; then
     println "\n===>  compress multiple files into an output directory and mirror input folder, --output-dir-mirror"
     println "test --output-dir-mirror" > tmp1
-    mkdir -p tmpInputTestDir/we/must/go/deeper
-    println cool > tmpInputTestDir/we/must/go/deeper/tmp2
+    mkdir -p tmpInputTestDir/we/.../..must/go/deeper..
+    println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2
     zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir
     test -f tmpOutDir/tmp1.zst
-    test -f tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2.zst
+    test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst
 
     println "test: compress input dir will be ignored if it has '..'"
-    zstd  -r tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'"
+    zstd  -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'"
+    zstd  -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'"
+    zstd  -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'"
     test ! -d non-exist
 
+    println "test: compress input dir should succeed with benign uses of '..'"
+    zstd  -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout
+    test -d tmpout
+
     println "test : decompress multiple files into an output directory, --output-dir-mirror"
     zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp
     test -f tmpOutDirDecomp/tmpOutDir/tmp1
-    test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2
+    test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2
 
     println "test: decompress input dir will be ignored if it has '..'"
-    zstd  -r tmpOutDir/tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'"
+    zstd  -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'"
     test ! -d non-exist
 
     rm -rf tmp*

From 221e4659cdda91fa71869d9f740c391acfdb09e5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Feb 2021 13:05:39 -0500
Subject: [PATCH 77/80] Avoid Using `stat -c` on NetBSD

Addresses #2493. I think. I don't have a NetBSD system to test on.
---
 tests/playTests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index ec1692c21..b99fd80f0 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -121,7 +121,7 @@ esac
 
 MTIME="stat -c %Y"
 case "$UNAME" in
-    Darwin | FreeBSD | OpenBSD) MTIME="stat -f %m" ;;
+    Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
 esac
 
 DIFF="diff"

From d7db928f728743197e259240fe984b3143f1ddb5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 1 Mar 2021 17:45:30 -0500
Subject: [PATCH 78/80] Bump Library Version 1.4.8 -> 1.4.9

---
 lib/zstd.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 64265f682..222339d71 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -72,7 +72,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    4
-#define ZSTD_VERSION_RELEASE  8
+#define ZSTD_VERSION_RELEASE  9
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 
 /*! ZSTD_versionNumber() :

From 0f1a52b34948fb28cccf27745bdb3086d91f846d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 1 Mar 2021 17:57:02 -0500
Subject: [PATCH 79/80] Documentation Rebuild

---
 doc/zstd_manual.html | 45 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 315488844..450f19bc6 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.4.8 Manual</title>
+<title>zstd 1.4.9 Manual</title>
 </head>
 <body>
-<h1>zstd 1.4.8 Manual</h1>
+<h1>zstd 1.4.9 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -473,12 +473,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      * ZSTD_d_format
      * ZSTD_d_stableOutBuffer
      * ZSTD_d_forceIgnoreChecksum
+     * ZSTD_d_refMultipleDDicts
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
      ZSTD_d_experimentalParam2=1001,
-     ZSTD_d_experimentalParam3=1002
+     ZSTD_d_experimentalParam3=1002,
+     ZSTD_d_experimentalParam4=1003
 
 } ZSTD_dParameter;
 </b></pre><BR>
@@ -816,7 +818,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 </b><p>  Reference a prepared dictionary, to be used for all next compressed frames.
   Note that compression parameters are enforced from within CDict,
   and supersede any compression parameter previously set within CCtx.
-  The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
   The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
   The dictionary will remain valid for future compressed frames using same CCtx.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
@@ -867,6 +869,13 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 <pre><b>size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
 </b><p>  Reference a prepared dictionary, to be used to decompress next frames.
   The dictionary remains active for decompression of future frames using same DCtx.
+
+  If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
+  will store the DDict references in a table, and the DDict used for decompression
+  will be determined at decompression time, as per the dict ID in the frame.
+  The memory for the table is allocated on the first call to refDDict, and can be
+  freed with ZSTD_freeDCtx().
+
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Note 1 : Currently, only one dictionary can be managed.
            Referencing a new dictionary effectively "discards" any previous one.
@@ -995,6 +1004,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
     ZSTD_d_ignoreChecksum = 1
 } ZSTD_forceIgnoreChecksum_e;
 </b></pre><BR>
+<pre><b>typedef enum {
+    </b>/* Note: this enum controls ZSTD_d_refMultipleDDicts */<b>
+    ZSTD_rmd_refSingleDDict = 0,
+    ZSTD_rmd_refMultipleDDicts = 1
+} ZSTD_refMultipleDDicts_e;
+</b></pre><BR>
 <pre><b>typedef enum {
     </b>/* Note: this enum and the behavior it controls are effectively internal<b>
      * implementation details of the compressor. They are expected to continue
@@ -1073,7 +1088,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
   `srcSize` must be the _exact_ size of this series
        (i.e. there should be a frame boundary at `src + srcSize`)
   @return : - upper-bound for the decompressed size of all data in all successive frames
-            - if an error occured: ZSTD_CONTENTSIZE_ERROR
+            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
 
   note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame.
   note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
@@ -1155,6 +1170,22 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 </p></pre><BR>
 
+<pre><b>size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+                                const void* src, size_t srcSize, unsigned magicVariant);
+</b><p> Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
+
+ Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
+ ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+
+ Returns an error if destination buffer is not large enough, if the source size is not representable
+ with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
+
+ @return : number of bytes written or a ZSTD error.
+ 
+</p></pre><BR>
+
 <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
 
 <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
@@ -1328,7 +1359,7 @@ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this con
   how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) 
 </p></pre><BR>
 
-<pre><b>size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+<pre><b>size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
 </b><p>  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
   and store it into int* value.
  @return : 0, or an error code (which can be tested with ZSTD_isError()).
@@ -1382,7 +1413,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
  
 </p></pre><BR>
 
-<pre><b>size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+<pre><b>size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
 </b><p> Similar to ZSTD_CCtx_getParameter.
  Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).

From 3835957b2d7b012ef0a721c27967d5e56a3d91ca Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 1 Mar 2021 18:00:10 -0500
Subject: [PATCH 80/80] Update CHANGELOG

---
 CHANGELOG | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 860925631..9f3c8f753 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,29 @@
+v1.4.9  (Mar 1, 2021)
+bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
+bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)
+api: Add (De)Compression Tracing Functionality (#2482, @terrelln)
+api: Support References to Multiple DDicts (#2446, @senhuang42)
+api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
+perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
+perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
+perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
+cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte)
+cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
+cli: Improve Help Message (#2500, @senhuang42)
+tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
+tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat)
+tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte)
+build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973)
+build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt)
+build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime)
+build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka)
+build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln)
+build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg)
+build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973)
+contrib: Add `recover_directory` Program (#2473, @terrelln)
+doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42)
+doc: Fix Typos (#2459, @ThomasWaldmann)
+
 v1.4.8  (Dec 18, 2020)
 hotfix: wrong alignment of an internal buffer