initial commit

contrib/diagnose_corruption/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/diagnose_corruption/check_flipped_bits.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/externalMatchfinder/.gitignore

@@ -0,0 +1,2 @@
+# build artifacts
+externalMatchfinder

contrib/externalMatchfinder/Makefile

@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) Yann Collet, Meta Platforms, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common
+
+CFLAGS  ?= -O3
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+default: externalMatchfinder
+
+all: externalMatchfinder
+
+externalMatchfinder: matchfinder.c main.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) externalMatchfinder

contrib/externalMatchfinder/README.md

@@ -0,0 +1,14 @@
+externalMatchfinder
+=====================
+
+`externalMatchfinder` is a test tool for the external matchfinder API.
+It demonstrates how to use the API to perform a simple round-trip test.
+
+A sample matchfinder is provided in matchfinder.c, but the user can swap
+this out with a different one if desired. The sample matchfinder implements
+LZ compression with a 1KB hashtable. Dictionary compression is not currently supported.
+
+Command line :
+```
+externalMatchfinder filename
+```

contrib/externalMatchfinder/main.c

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#include "zstd_errors.h"
+#include "matchfinder.h" // simpleExternalMatchFinder
+
+#define CHECK(res)                                      \
+do {                                                    \
+    if (ZSTD_isError(res)) {                            \
+        printf("ERROR: %s\n", ZSTD_getErrorName(res));  \
+        return 1;                                       \
+    }                                                   \
+} while (0)                                             \
+
+int main(int argc, char *argv[]) {
+    if (argc != 2) {
+        printf("Usage: exampleMatchfinder <file>\n");
+        return 1;
+    }
+
+    ZSTD_CCtx* const zc = ZSTD_createCCtx();
+
+    int simpleExternalMatchState = 0xdeadbeef;
+
+    // Here is the crucial bit of code!
+    ZSTD_registerExternalMatchFinder(
+        zc,
+        &simpleExternalMatchState,
+        simpleExternalMatchFinder
+    );
+
+    {
+        size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 1);
+        CHECK(res);
+    }
+
+    FILE *f = fopen(argv[1], "rb");
+    assert(f);
+    {
+        int const ret = fseek(f, 0, SEEK_END);
+        assert(ret == 0);
+    }
+    size_t const srcSize = ftell(f);
+    {
+        int const ret = fseek(f, 0, SEEK_SET);
+        assert(ret == 0);
+    }
+
+    char* const src = malloc(srcSize + 1);
+    assert(src);
+    {
+        size_t const ret = fread(src, srcSize, 1, f);
+        assert(ret == 1);
+        int const ret2 = fclose(f);
+        assert(ret2 == 0);
+    }
+
+    size_t const dstSize = ZSTD_compressBound(srcSize);
+    char* const dst = malloc(dstSize);
+    assert(dst);
+
+    size_t const cSize = ZSTD_compress2(zc, dst, dstSize, src, srcSize);
+    CHECK(cSize);
+
+    char* const val = malloc(srcSize);
+    assert(val);
+
+    {
+        size_t const res = ZSTD_decompress(val, srcSize, dst, cSize);
+        CHECK(res);
+    }
+
+    if (memcmp(src, val, srcSize) == 0) {
+        printf("Compression and decompression were successful!\n");
+        printf("Original size: %lu\n", srcSize);
+        printf("Compressed size: %lu\n", cSize);
+    } else {
+        printf("ERROR: input and validation buffers don't match!\n");
+        for (size_t i = 0; i < srcSize; i++) {
+            if (src[i] != val[i]) {
+                printf("First bad index: %zu\n", i);
+                break;
+            }
+        }
+        return 1;
+    }
+
+    ZSTD_freeCCtx(zc);
+    free(src);
+    free(dst);
+    free(val);
+    return 0;
+}

contrib/externalMatchfinder/matchfinder.c

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "matchfinder.h"
+
+#define HSIZE 1024
+static U32 const HLOG = 10;
+static U32 const MLS = 4;
+static U32 const BADIDX = 0xffffffff;
+
+size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    size_t seqCount = 0;
+    U32 hashTable[HSIZE];
+
+    (void)externalMatchState;
+    (void)dict;
+    (void)dictSize;
+    (void)outSeqsCapacity;
+    (void)compressionLevel;
+
+    {   int i;
+        for (i=0; i < HSIZE; i++) {
+            hashTable[i] = BADIDX;
+    }   }
+
+    while (ip + MLS < iend) {
+        size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS);
+        U32 const matchIndex = hashTable[hash];
+        hashTable[hash] = (U32)(ip - istart);
+
+        if (matchIndex != BADIDX) {
+            const BYTE* const match = istart + matchIndex;
+            U32 const matchLen = (U32)ZSTD_count(ip, match, iend);
+            if (matchLen >= ZSTD_MINMATCH_MIN) {
+                U32 const litLen = (U32)(ip - anchor);
+                U32 const offset = (U32)(ip - match);
+                ZSTD_Sequence const seq = {
+                    offset, litLen, matchLen, 0
+                };
+
+                /* Note: it's crucial to stay within the window size! */
+                if (offset <= windowSize) {
+                    outSeqs[seqCount++] = seq;
+                    ip += matchLen;
+                    anchor = ip;
+                    continue;
+                }
+            }
+        }
+
+        ip++;
+    }
+
+    {   ZSTD_Sequence const finalSeq = {
+            0, (U32)(iend - anchor), 0, 0
+        };
+        outSeqs[seqCount++] = finalSeq;
+    }
+
+    return seqCount;
+}

contrib/externalMatchfinder/matchfinder.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MATCHFINDER_H
+#define MATCHFINDER_H
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+#endif

contrib/freestanding_lib/freestanding.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) 2021-2021, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/gen_html/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/gen_html/gen_html.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/largeNbDicts/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/largeNbDicts/largeNbDicts.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -856,7 +856,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary,
     CONTROL(cTotalSizeNoDict != 0);
     DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f  (%u bytes) \n",
                     clevel,
-                    (double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
+                    (double)totalSrcSlicesSize / (double)cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
 
     size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
     CONTROL(cSizes != NULL);
@@ -865,7 +865,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary,
     CONTROL(cTotalSize != 0);
     DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f  (%u bytes) \n",
                     (unsigned)dictBuffer.size,
-                    (double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
+                    (double)totalSrcSlicesSize / (double)cTotalSize, (unsigned)cTotalSize);
 
     /* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
     shrinkSizes(dstSlices, cSizes);
@@ -985,7 +985,7 @@ int usage(const char* exeName)
     DISPLAY ("-#          : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
     DISPLAY ("-D #        : use # as a dictionary (default: create one) \n");
     DISPLAY ("-i#         : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S);
-    DISPLAY ("-p#         : print speed for all rounds 0=fastest 1=median (default: 0)");
+    DISPLAY ("-p#         : print speed for all rounds 0=fastest 1=median (default: 0) \n");
     DISPLAY ("--nbBlocks=#: use # blocks for bench (default: one per file) \n");
     DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n");
     DISPLAY ("-h          : help (this text) \n");

contrib/linux-kernel/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -32,9 +32,7 @@ libzstd:
 		-DZSTD_LEGACY_SUPPORT=0 \
 		-DZSTD_STATIC_LINKING_ONLY \
 		-DFSE_STATIC_LINKING_ONLY \
-		-DHUF_STATIC_LINKING_ONLY \
 		-DXXH_STATIC_LINKING_ONLY \
-		-DMEM_FORCE_MEMORY_ACCESS=0 \
 		-D__GNUC__ \
 		-D__linux__=1 \
 		-DSTATIC_BMI2=0 \
@@ -50,8 +48,8 @@ libzstd:
 		-UZSTD_MULTITHREAD \
 		-U_MSC_VER \
 		-U_WIN32 \
-		-RZSTDLIB_VISIBILITY= \
-		-RZSTDERRORLIB_VISIBILITY= \
+		-RZSTDLIB_VISIBLE= \
+		-RZSTDERRORLIB_VISIBLE= \
 		-RZSTD_FALLTHROUGH=fallthrough \
 		-DZSTD_HAVE_WEAK_SYMBOLS=0 \
 		-DZSTD_TRACE=0 \

contrib/linux-kernel/decompress_sources.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/linux.mk

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/linux_zstd.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/mem.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/compiler.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/errno.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/kernel.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/limits.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/math64.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/module.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/printk.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/stddef.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/swab.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/types.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/include/linux/xxhash.h

@@ -2,7 +2,7 @@
  * xxHash - Extremely Fast Hash algorithm
  * Copyright (C) 2012-2016, Yann Collet.
  *
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -260,7 +260,7 @@ XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state
  * xxHash - Extremely Fast Hash algorithm
  * Copyright (C) 2012-2016, Yann Collet.
  *
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are

contrib/linux-kernel/test/static_test.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/test/test.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) {
   asm volatile("" : "+r"(x));
 }
 
+static void __attribute__((noinline)) fill_stack(void) {
+  memset(g_stack, 0x33, 8192);
+}
+
 static void __attribute__((noinline)) set_stack(void) {
 
   char stack[8192];
   g_stack = stack;
-  memset(g_stack, 0x33, 8192);
   use(g_stack);
 }
 
@@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) {
 
 static void test_stack_usage(test_data_t const *data) {
   set_stack();
+  fill_stack();
   test_f2fs();
   test_btrfs(data);
   test_decompress_unzstd(data);

contrib/linux-kernel/zstd_common_module.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/zstd_compress_module.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/zstd_decompress_module.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/linux-kernel/zstd_deps.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/match_finders/zstd_edist.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/match_finders/zstd_edist.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/ErrorHolder.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/Logging.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/Options.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/Options.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/Pzstd.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -10,11 +10,13 @@
 #include "Pzstd.h"
 #include "SkippableFrame.h"
 #include "utils/FileSystem.h"
+#include "utils/Portability.h"
 #include "utils/Range.h"
 #include "utils/ScopeGuard.h"
 #include "utils/ThreadPool.h"
 #include "utils/WorkQueue.h"
 
+#include <algorithm>
 #include <chrono>
 #include <cinttypes>
 #include <cstddef>
@@ -336,6 +338,10 @@ static size_t calculateStep(
     const ZSTD_parameters &params) {
   (void)size;
   (void)numThreads;
+  // Not validated to work correctly for window logs > 23.
+  // It will definitely fail if windowLog + 2 is >= 4GB because
+  // the skippable frame can only store sizes up to 4GB.
+  assert(params.cParams.windowLog <= 23);
   return size_t{1} << (params.cParams.windowLog + 2);
 }
 
@@ -587,7 +593,8 @@ std::uint64_t writeFile(
       // start writing before compression is done because we need to know the
       // compressed size.
       // Wait for the compressed size to be available and write skippable frame
-      SkippableFrame frame(out->size());
+      assert(uint64_t(out->size()) < uint64_t(1) << 32);
+      SkippableFrame frame(uint32_t(out->size()));
       if (!writeData(frame.data(), outputFd)) {
         errorHolder.setError("Failed to write output");
         return bytesWritten;

contrib/pzstd/Pzstd.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/README.md

@@ -31,7 +31,7 @@ If this number is not suitable, during compilation you can define `PZSTD_NUM_THR
 
 ## Benchmarks
 
-As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia).
+As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia).
 
 Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads
 ------------------------------------------|-----------------------------------

contrib/pzstd/SkippableFrame.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/SkippableFrame.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/main.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/test/OptionsTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/test/PzstdTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/test/RoundTrip.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/test/RoundTripTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/Buffer.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/FileSystem.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -8,16 +8,18 @@
  */
 #pragma once
 
+#include "utils/Portability.h"
 #include "utils/Range.h"
 
 #include <sys/stat.h>
 #include <cerrno>
 #include <cstdint>
+#include <limits>
 #include <system_error>
 
 // A small subset of `std::filesystem`.
 // `std::filesystem` should be a drop in replacement.
-// See http://en.cppreference.com/w/cpp/filesystem for documentation.
+// See https://en.cppreference.com/w/cpp/filesystem for documentation.
 
 namespace pzstd {
 
@@ -28,7 +30,7 @@ typedef struct ::_stat64 file_status;
 typedef struct ::stat file_status;
 #endif
 
-/// http://en.cppreference.com/w/cpp/filesystem/status
+/// https://en.cppreference.com/w/cpp/filesystem/status
 inline file_status status(StringPiece path, std::error_code& ec) noexcept {
   file_status status;
 #if defined(_MSC_VER)
@@ -44,7 +46,7 @@ inline file_status status(StringPiece path, std::error_code& ec) noexcept {
   return status;
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
+/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file
 inline bool is_regular_file(file_status status) noexcept {
 #if defined(S_ISREG)
   return S_ISREG(status.st_mode);
@@ -55,12 +57,12 @@ inline bool is_regular_file(file_status status) noexcept {
 #endif
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
+/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file
 inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept {
   return is_regular_file(status(path, ec));
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+/// https://en.cppreference.com/w/cpp/filesystem/is_directory
 inline bool is_directory(file_status status) noexcept {
 #if defined(S_ISDIR)
   return S_ISDIR(status.st_mode);
@@ -71,22 +73,22 @@ inline bool is_directory(file_status status) noexcept {
 #endif
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+/// https://en.cppreference.com/w/cpp/filesystem/is_directory
 inline bool is_directory(StringPiece path, std::error_code& ec) noexcept {
   return is_directory(status(path, ec));
 }
 
-/// http://en.cppreference.com/w/cpp/filesystem/file_size
+/// https://en.cppreference.com/w/cpp/filesystem/file_size
 inline std::uintmax_t file_size(
     StringPiece path,
     std::error_code& ec) noexcept {
   auto stat = status(path, ec);
   if (ec) {
-    return -1;
+    return std::numeric_limits<uintmax_t>::max();
   }
   if (!is_regular_file(stat)) {
     ec.assign(ENOTSUP, std::generic_category());
-    return -1;
+    return std::numeric_limits<uintmax_t>::max();
   }
   ec.clear();
   return stat.st_size;

contrib/pzstd/utils/Likely.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/Portability.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+#pragma once
+
+#include <algorithm>
+
+// Required for windows, which defines min/max, but we want the std:: version.
+#undef min
+#undef max

contrib/pzstd/utils/Range.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,9 @@
 #pragma once
 
 #include "utils/Likely.h"
+#include "utils/Portability.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <cstring>
 #include <stdexcept>

contrib/pzstd/utils/ResourcePool.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/ScopeGuard.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/ThreadPool.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/WorkQueue.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/test/BufferTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/test/RangeTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/test/ResourcePoolTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/test/ScopeGuardTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/test/ThreadPoolTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/pzstd/utils/test/WorkQueueTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/recovery/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2019-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/recovery/recover_directory.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/examples/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/examples/parallel_compression.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/examples/parallel_processing.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/examples/seekable_compression.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/examples/seekable_decompression.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/examples/seekable_decompression_mem.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/tests/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2017-present, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the

contrib/seekable_format/tests/seekable_tests.c

@@ -186,6 +186,40 @@ int main(int argc, const char** argv)
     }
     printf("Success!\n");
 
+
+    printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++);
+    { // compressing empty string should return a zstd header
+        size_t const capacity = 255;
+        char* inBuffer = malloc(capacity);
+        assert(inBuffer != NULL);
+        inBuffer[0] = '\0';
+        void* const outBuffer = malloc(capacity);
+        assert(outBuffer != NULL);
+
+        ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream();
+        ZSTD_seekable_initCStream(s, 1, 1, 255);
+
+        ZSTD_inBuffer input = { .src=inBuffer, .pos=0, .size=0 };
+        ZSTD_outBuffer output = { .dst=outBuffer, .pos=0, .size=capacity };
+
+        ZSTD_seekable_compressStream(s, &output, &input);
+        ZSTD_seekable_endStream(s, &output);
+
+        if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) {
+            printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] );
+
+            free(inBuffer);
+            free(outBuffer);
+            ZSTD_seekable_freeCStream(s);
+            goto _test_error;
+        }
+
+        free(inBuffer);
+        free(outBuffer);
+        ZSTD_seekable_freeCStream(s);
+    }
+    printf("Success!\n");
+
     /* TODO: Add more tests */
     printf("Finished tests\n");
     return 0;

contrib/seekable_format/zstd_seekable_compression_format.md

@@ -2,7 +2,7 @@
 
 ### Notices
 
-Copyright (c) 2017-present Facebook, Inc.
+Copyright (c) Meta Platforms, Inc. and affiliates.
 
 Permission is granted to copy and distribute this document
 for any purpose and without charge,

contrib/seekable_format/zstdseek_compress.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -350,7 +350,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
 
 size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)
 {
-    if (!zcs->writingSeekTable && zcs->frameDSize) {
+    if (!zcs->writingSeekTable) {
         const size_t endFrame = ZSTD_seekable_endFrame(zcs, output);
         if (ZSTD_isError(endFrame)) return endFrame;
         /* return an accurate size hint */

contrib/seekable_format/zstdseek_decompress.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -38,7 +38,7 @@
       * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */
 #    define PLATFORM_POSIX_VERSION 200112L
 
-/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
+/* try to determine posix version through official unistd.h's _POSIX_VERSION (https://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
  * note : there is no simple way to know in advance if <unistd.h> is present or not on target system,
  * Posix specification mandates its presence and its content, but target system must respect this spec.
  * It's necessary to _not_ #include <unistd.h> whenever target OS is not unix-like
@@ -517,9 +517,9 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
             size_t forwardProgress;
             if (zs->decompressedOffset < offset) {
                 /* dummy decompressions until we get to the target offset */
-                outTmp = (ZSTD_outBuffer){zs->outBuff, MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset), 0};
+                outTmp = (ZSTD_outBuffer){zs->outBuff, (size_t) (MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset)), 0};
             } else {
-                outTmp = (ZSTD_outBuffer){dst, len, zs->decompressedOffset - offset};
+                outTmp = (ZSTD_outBuffer){dst, len, (size_t) (zs->decompressedOffset - offset)};
             }
 
             prevOutPos = outTmp.pos;

contrib/seqBench/Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the