Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev10

.gitignore

@@ -1,6 +1,7 @@
 # Object files
 *.o
 *.ko
+*.dSYM
 
 # Libraries
 *.lib
@@ -30,3 +31,4 @@ _zstdbench/
 *.idea
 *.swp
 .DS_Store
+googletest/

.travis.yml

@@ -9,7 +9,17 @@ matrix:
       env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest"
+      language: cpp
+      install:
+        - export CXX="g++-4.8" CC="gcc-4.8"
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - gcc-4.8
+            - g++-4.8
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
     - os: linux
       sudo: false
       env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"

NEWS

@@ -1,3 +1,10 @@
+v1.0.1
+New : contrib/pzstd, parallel version of zstd, by Nick Terrell
+Fixed : CLI -d output to stdout by default when input is stdin (#322)
+Fixed : CLI correctly detects console on Mac OS-X
+Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
+Fixed : zstd-pgo, reported by octoploid (#329)
+
 v1.0.0
 Change Licensing, all project is now BSD, Copyright Facebook
 Small decompression speed improvement
@@ -29,6 +36,9 @@ Modified : minor compression level adaptations
 Updated : compression format specification to v0.2.0
 changed : zstd.h moved to /lib directory
 
+v0.7.5
+Transition version, supporting decoding of v0.8.x
+
 v0.7.4
 Added : homebrew for Mac, by Daniel Cade
 Added : more examples
@@ -179,6 +189,6 @@ frame concatenation support
 v0.1.1
 fix compression bug
 detects write-flush errors
-git@github.com:Cyan4973/zstd.git
+
 v0.1.0
 first release

contrib/pzstd/.gitignore

@@ -0,0 +1,2 @@
+# compilation result
+pzstd

contrib/pzstd/ErrorHolder.h

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <stdexcept>
+#include <string>
+
+namespace pzstd {
+
+// Coordinates graceful shutdown of the pzstd pipeline
+class ErrorHolder {
+  std::atomic<bool> error_;
+  std::string message_;
+
+ public:
+  ErrorHolder() : error_(false) {}
+
+  bool hasError() noexcept {
+    return error_.load();
+  }
+
+  void setError(std::string message) noexcept {
+    // Given multiple possibly concurrent calls, exactly one will ever succeed.
+    bool expected = false;
+    if (error_.compare_exchange_strong(expected, true)) {
+      message_ = std::move(message);
+    }
+  }
+
+  bool check(bool predicate, std::string message) noexcept {
+    if (!predicate) {
+      setError(std::move(message));
+    }
+    return !hasError();
+  }
+
+  std::string getError() noexcept {
+    error_.store(false);
+    return std::move(message_);
+  }
+
+  ~ErrorHolder() {
+    assert(!hasError());
+  }
+};
+}

contrib/pzstd/Makefile

@@ -0,0 +1,76 @@
+# ##########################################################################
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ##########################################################################
+
+ZSTDDIR = ../../lib
+PROGDIR = ../../programs
+
+CPPFLAGS = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+CXXFLAGS  ?= -O3
+CXXFLAGS  += -std=c++11
+CXXFLAGS  += $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
+
+
+ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
+ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
+ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+.PHONY: default all test clean
+
+default: pzstd
+
+all: pzstd
+
+
+libzstd.a: $(ZSTD_FILES)
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	@cp $(ZSTDDIR)/libzstd.a .
+
+
+Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
+	$(CXX) $(FLAGS) -c Pzstd.cpp -o $@
+
+SkippableFrame.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
+	$(CXX) $(FLAGS) -c SkippableFrame.cpp -o $@
+
+Options.o: Options.h Options.cpp
+	$(CXX) $(FLAGS) -c Options.cpp -o $@
+
+main.o: main.cpp *.h utils/*.h
+	$(CXX) $(FLAGS) -c main.cpp -o $@
+
+pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a 
+	$(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread
+
+googletest:
+	@git clone https://github.com/google/googletest
+	@mkdir -p googletest/build
+	@cd googletest/build && cmake .. && make
+
+test: libzstd.a Pzstd.o Options.o SkippableFrame.o
+	$(MAKE) -C utils/test clean
+	$(MAKE) -C utils/test test
+	$(MAKE) -C test clean
+	$(MAKE) -C test test
+
+clean:
+	$(MAKE) -C $(ZSTDDIR) clean
+	$(MAKE) -C utils/test clean
+	$(MAKE) -C test clean
+	@$(RM) -rf libzstd.a *.o pzstd$(EXT)
+	@echo Cleaning completed

contrib/pzstd/Options.cpp

@@ -0,0 +1,181 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "Options.h"
+
+#include <cstdio>
+#include <cstring>
+
+namespace pzstd {
+
+namespace {
+unsigned parseUnsigned(const char* arg) {
+  unsigned result = 0;
+  while (*arg >= '0' && *arg <= '9') {
+    result *= 10;
+    result += *arg - '0';
+    ++arg;
+  }
+  return result;
+}
+
+const std::string zstdExtension = ".zst";
+constexpr unsigned defaultCompressionLevel = 3;
+constexpr unsigned maxNonUltraCompressionLevel = 19;
+
+void usage() {
+  std::fprintf(stderr, "Usage:\n");
+  std::fprintf(stderr, "\tpzstd [args] FILE\n");
+  std::fprintf(stderr, "Parallel ZSTD options:\n");
+  std::fprintf(stderr, "\t-n/--num-threads #: Number of threads to spawn\n");
+  std::fprintf(stderr, "\t-p/--pzstd-headers: Write pzstd headers to enable parallel decompression\n");
+
+  std::fprintf(stderr, "ZSTD options:\n");
+  std::fprintf(stderr, "\t-u/--ultra        : enable levels beyond %i, up to %i (requires more memory)\n", maxNonUltraCompressionLevel, ZSTD_maxCLevel());
+  std::fprintf(stderr, "\t-h/--help         : display help and exit\n");
+  std::fprintf(stderr, "\t-V/--version      : display version number and exit\n");
+  std::fprintf(stderr, "\t-d/--decompress   : decompression\n");
+  std::fprintf(stderr, "\t-f/--force        : overwrite output\n");
+  std::fprintf(stderr, "\t-o/--output file  : result stored into `file`\n");
+  std::fprintf(stderr, "\t-c/--stdout       : write output to standard output\n");
+  std::fprintf(stderr, "\t-#                : # compression level (1-%d, default:%d)\n", maxNonUltraCompressionLevel, defaultCompressionLevel);
+}
+} // anonymous namespace
+
+Options::Options()
+    : numThreads(0),
+      maxWindowLog(23),
+      compressionLevel(defaultCompressionLevel),
+      decompress(false),
+      overwrite(false),
+      pzstdHeaders(false) {}
+
+bool Options::parse(int argc, const char** argv) {
+  bool ultra = false;
+  for (int i = 1; i < argc; ++i) {
+    const char* arg = argv[i];
+    // Arguments with a short option
+    char option = 0;
+    if (!std::strcmp(arg, "--num-threads")) {
+      option = 'n';
+    } else if (!std::strcmp(arg, "--pzstd-headers")) {
+      option = 'p';
+    } else if (!std::strcmp(arg, "--ultra")) {
+      option = 'u';
+    } else if (!std::strcmp(arg, "--version")) {
+      option = 'V';
+    } else if (!std::strcmp(arg, "--help")) {
+      option = 'h';
+    } else if (!std::strcmp(arg, "--decompress")) {
+      option = 'd';
+    } else if (!std::strcmp(arg, "--force")) {
+      option = 'f';
+    } else if (!std::strcmp(arg, "--output")) {
+      option = 'o';
+    }  else if (!std::strcmp(arg, "--stdout")) {
+      option = 'c';
+    }else if (arg[0] == '-' && arg[1] != 0) {
+      // Parse the compression level or short option
+      if (arg[1] >= '0' && arg[1] <= '9') {
+        compressionLevel = parseUnsigned(arg + 1);
+        continue;
+      }
+      option = arg[1];
+    } else if (inputFile.empty()) {
+      inputFile = arg;
+      continue;
+    } else {
+      std::fprintf(stderr, "Invalid argument: %s.\n", arg);
+      return false;
+    }
+
+    switch (option) {
+      case 'n':
+        if (++i == argc) {
+          std::fprintf(stderr, "Invalid argument: -n requires an argument.\n");
+          return false;
+        }
+        numThreads = parseUnsigned(argv[i]);
+        if (numThreads == 0) {
+          std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n");
+        }
+        break;
+      case 'p':
+        pzstdHeaders = true;
+        break;
+      case 'u':
+        ultra = true;
+        maxWindowLog = 0;
+        break;
+      case 'V':
+        std::fprintf(stderr, "ZSTD version: %s.\n", ZSTD_VERSION_STRING);
+        return false;
+      case 'h':
+        usage();
+        return false;
+      case 'd':
+        decompress = true;
+        break;
+      case 'f':
+        overwrite = true;
+        break;
+      case 'o':
+        if (++i == argc) {
+          std::fprintf(stderr, "Invalid argument: -o requires an argument.\n");
+          return false;
+        }
+        outputFile = argv[i];
+        break;
+      case 'c':
+        outputFile = '-';
+        break;
+      default:
+        std::fprintf(stderr, "Invalid argument: %s.\n", arg);
+        return false;
+    }
+  }
+  // Determine input file if not specified
+  if (inputFile.empty()) {
+    inputFile = "-";
+  }
+  // Determine output file if not specified
+  if (outputFile.empty()) {
+    if (inputFile == "-") {
+      outputFile = "-";
+    } else {
+      // Attempt to add/remove zstd extension from the input file
+      if (decompress) {
+        int stemSize = inputFile.size() - zstdExtension.size();
+        if (stemSize > 0 && inputFile.substr(stemSize) == zstdExtension) {
+          outputFile = inputFile.substr(0, stemSize);
+        } else {
+          std::fprintf(
+              stderr, "Invalid argument: Unable to determine output file.\n");
+          return false;
+        }
+      } else {
+        outputFile = inputFile + zstdExtension;
+      }
+    }
+  }
+  // Check compression level
+  {
+    unsigned maxCLevel = ultra ? ZSTD_maxCLevel() : maxNonUltraCompressionLevel;
+    if (compressionLevel > maxCLevel) {
+      std::fprintf(
+          stderr, "Invalid compression level %u.\n", compressionLevel);
+    }
+  }
+  // Check that numThreads is set
+  if (numThreads == 0) {
+    std::fprintf(stderr, "Invalid arguments: # of threads not specified.\n");
+    return false;
+  }
+  return true;
+}
+}

contrib/pzstd/Options.h

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#undef ZSTD_STATIC_LINKING_ONLY
+
+#include <cstdint>
+#include <string>
+
+namespace pzstd {
+
+struct Options {
+  unsigned numThreads;
+  unsigned maxWindowLog;
+  unsigned compressionLevel;
+  bool decompress;
+  std::string inputFile;
+  std::string outputFile;
+  bool overwrite;
+  bool pzstdHeaders;
+
+  Options();
+  Options(
+      unsigned numThreads,
+      unsigned maxWindowLog,
+      unsigned compressionLevel,
+      bool decompress,
+      const std::string& inputFile,
+      const std::string& outputFile,
+      bool overwrite,
+      bool pzstdHeaders)
+      : numThreads(numThreads),
+        maxWindowLog(maxWindowLog),
+        compressionLevel(compressionLevel),
+        decompress(decompress),
+        inputFile(inputFile),
+        outputFile(outputFile),
+        overwrite(overwrite),
+        pzstdHeaders(pzstdHeaders) {}
+
+  bool parse(int argc, const char** argv);
+
+  ZSTD_parameters determineParameters() const {
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
+    if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) {
+      params.cParams.windowLog = maxWindowLog;
+      params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0);
+    }
+    return params;
+  }
+};
+}

contrib/pzstd/Pzstd.cpp

@@ -0,0 +1,480 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "Pzstd.h"
+#include "SkippableFrame.h"
+#include "utils/FileSystem.h"
+#include "utils/Range.h"
+#include "utils/ScopeGuard.h"
+#include "utils/ThreadPool.h"
+#include "utils/WorkQueue.h"
+
+#include <cstddef>
+#include <cstdio>
+#include <memory>
+#include <string>
+
+namespace pzstd {
+
+namespace {
+#ifdef _WIN32
+const std::string nullOutput = "nul";
+#else
+const std::string nullOutput = "/dev/null";
+#endif
+}
+
+using std::size_t;
+
+size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) {
+  // Open the input file and attempt to determine its size
+  FILE* inputFd = stdin;
+  size_t inputSize = 0;
+  if (options.inputFile != "-") {
+    inputFd = std::fopen(options.inputFile.c_str(), "rb");
+    if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
+      return 0;
+    }
+    std::error_code ec;
+    inputSize = file_size(options.inputFile, ec);
+    if (ec) {
+      inputSize = 0;
+    }
+  }
+  auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); });
+
+  // Check if the output file exists and then open it
+  FILE* outputFd = stdout;
+  if (options.outputFile != "-") {
+    if (!options.overwrite && options.outputFile != nullOutput) {
+      outputFd = std::fopen(options.outputFile.c_str(), "rb");
+      if (!errorHolder.check(outputFd == nullptr, "Output file exists")) {
+        return 0;
+      }
+    }
+    outputFd = std::fopen(options.outputFile.c_str(), "wb");
+    if (!errorHolder.check(
+            outputFd != nullptr, "Failed to open output file")) {
+      return 0;
+    }
+  }
+  auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); });
+
+  // WorkQueue outlives ThreadPool so in the case of error we are certain
+  // we don't accidently try to call push() on it after it is destroyed.
+  WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{2 * options.numThreads};
+  size_t bytesWritten;
+  {
+    // Initialize the thread pool with numThreads + 1
+    // We add one because the read thread spends most of its time waiting.
+    // This also sets the minimum number of threads to 2, so the algorithm
+    // doesn't deadlock.
+    ThreadPool executor(options.numThreads + 1);
+    if (!options.decompress) {
+      // Add a job that reads the input and starts all the compression jobs
+      executor.add(
+          [&errorHolder, &outs, &executor, inputFd, inputSize, &options] {
+            asyncCompressChunks(
+                errorHolder,
+                outs,
+                executor,
+                inputFd,
+                inputSize,
+                options.numThreads,
+                options.determineParameters());
+          });
+      // Start writing
+      bytesWritten =
+          writeFile(errorHolder, outs, outputFd, options.pzstdHeaders);
+    } else {
+      // Add a job that reads the input and starts all the decompression jobs
+      executor.add([&errorHolder, &outs, &executor, inputFd] {
+        asyncDecompressFrames(errorHolder, outs, executor, inputFd);
+      });
+      // Start writing
+      bytesWritten = writeFile(
+          errorHolder, outs, outputFd, /* writeSkippableFrames */ false);
+    }
+  }
+  return bytesWritten;
+}
+
+/// Construct a `ZSTD_inBuffer` that points to the data in `buffer`.
+static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) {
+  return ZSTD_inBuffer{buffer.data(), buffer.size(), 0};
+}
+
+/**
+ * Advance `buffer` and `inBuffer` by the amount of data read, as indicated by
+ * `inBuffer.pos`.
+ */
+void advance(Buffer& buffer, ZSTD_inBuffer& inBuffer) {
+  auto pos = inBuffer.pos;
+  inBuffer.src = static_cast<const unsigned char*>(inBuffer.src) + pos;
+  inBuffer.size -= pos;
+  inBuffer.pos = 0;
+  return buffer.advance(pos);
+}
+
+/// Construct a `ZSTD_outBuffer` that points to the data in `buffer`.
+static ZSTD_outBuffer makeZstdOutBuffer(Buffer& buffer) {
+  return ZSTD_outBuffer{buffer.data(), buffer.size(), 0};
+}
+
+/**
+ * Split `buffer` and advance `outBuffer` by the amount of data written, as
+ * indicated by `outBuffer.pos`.
+ */
+Buffer split(Buffer& buffer, ZSTD_outBuffer& outBuffer) {
+  auto pos = outBuffer.pos;
+  outBuffer.dst = static_cast<unsigned char*>(outBuffer.dst) + pos;
+  outBuffer.size -= pos;
+  outBuffer.pos = 0;
+  return buffer.splitAt(pos);
+}
+
+/**
+ * Stream chunks of input from `in`, compress it, and stream it out to `out`.
+ *
+ * @param errorHolder Used to report errors and check if an error occured
+ * @param in           Queue that we `pop()` input buffers from
+ * @param out          Queue that we `push()` compressed output buffers to
+ * @param maxInputSize An upper bound on the size of the input
+ * @param parameters   The zstd parameters to use for compression
+ */
+static void compress(
+    ErrorHolder& errorHolder,
+    std::shared_ptr<BufferWorkQueue> in,
+    std::shared_ptr<BufferWorkQueue> out,
+    size_t maxInputSize,
+    ZSTD_parameters parameters) {
+  auto guard = makeScopeGuard([&] { out->finish(); });
+  // Initialize the CCtx
+  std::unique_ptr<ZSTD_CStream, size_t (&)(ZSTD_CStream*)> ctx(
+      ZSTD_createCStream(), ZSTD_freeCStream);
+  if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
+    return;
+  }
+  {
+    auto err = ZSTD_initCStream_advanced(ctx.get(), nullptr, 0, parameters, 0);
+    if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
+      return;
+    }
+  }
+
+  // Allocate space for the result
+  auto outBuffer = Buffer(ZSTD_compressBound(maxInputSize));
+  auto zstdOutBuffer = makeZstdOutBuffer(outBuffer);
+  {
+    Buffer inBuffer;
+    // Read a buffer in from the input queue
+    while (in->pop(inBuffer) && !errorHolder.hasError()) {
+      auto zstdInBuffer = makeZstdInBuffer(inBuffer);
+      // Compress the whole buffer and send it to the output queue
+      while (!inBuffer.empty() && !errorHolder.hasError()) {
+        if (!errorHolder.check(
+                !outBuffer.empty(), "ZSTD_compressBound() was too small")) {
+          return;
+        }
+        // Compress
+        auto err =
+            ZSTD_compressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer);
+        if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
+          return;
+        }
+        // Split the compressed data off outBuffer and pass to the output queue
+        out->push(split(outBuffer, zstdOutBuffer));
+        // Forget about the data we already compressed
+        advance(inBuffer, zstdInBuffer);
+      }
+    }
+  }
+  // Write the epilog
+  size_t bytesLeft;
+  do {
+    if (!errorHolder.check(
+            !outBuffer.empty(), "ZSTD_compressBound() was too small")) {
+      return;
+    }
+    bytesLeft = ZSTD_endStream(ctx.get(), &zstdOutBuffer);
+    if (!errorHolder.check(
+            !ZSTD_isError(bytesLeft), ZSTD_getErrorName(bytesLeft))) {
+      return;
+    }
+    out->push(split(outBuffer, zstdOutBuffer));
+  } while (bytesLeft != 0 && !errorHolder.hasError());
+}
+
+/**
+ * Calculates how large each independently compressed frame should be.
+ *
+ * @param size       The size of the source if known, 0 otherwise
+ * @param numThreads The number of threads available to run compression jobs on
+ * @param params     The zstd parameters to be used for compression
+ */
+static size_t
+calculateStep(size_t size, size_t numThreads, const ZSTD_parameters& params) {
+  size_t step = 1ul << (params.cParams.windowLog + 2);
+  // If file size is known, see if a smaller step will spread work more evenly
+  if (size != 0) {
+    size_t newStep = size / numThreads;
+    if (newStep != 0) {
+      step = std::min(step, newStep);
+    }
+  }
+  return step;
+}
+
+namespace {
+enum class FileStatus { Continue, Done, Error };
+/// Determines the status of the file descriptor `fd`.
+FileStatus fileStatus(FILE* fd) {
+  if (std::feof(fd)) {
+    return FileStatus::Done;
+  } else if (std::ferror(fd)) {
+    return FileStatus::Error;
+  }
+  return FileStatus::Continue;
+}
+} // anonymous namespace
+
+/**
+ * Reads `size` data in chunks of `chunkSize` and puts it into `queue`.
+ * Will read less if an error or EOF occurs.
+ * Returns the status of the file after all of the reads have occurred.
+ */
+static FileStatus
+readData(BufferWorkQueue& queue, size_t chunkSize, size_t size, FILE* fd) {
+  Buffer buffer(size);
+  while (!buffer.empty()) {
+    auto bytesRead =
+        std::fread(buffer.data(), 1, std::min(chunkSize, buffer.size()), fd);
+    queue.push(buffer.splitAt(bytesRead));
+    auto status = fileStatus(fd);
+    if (status != FileStatus::Continue) {
+      return status;
+    }
+  }
+  return FileStatus::Continue;
+}
+
+void asyncCompressChunks(
+    ErrorHolder& errorHolder,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
+    ThreadPool& executor,
+    FILE* fd,
+    size_t size,
+    size_t numThreads,
+    ZSTD_parameters params) {
+  auto chunksGuard = makeScopeGuard([&] { chunks.finish(); });
+
+  // Break the input up into chunks of size `step` and compress each chunk
+  // independently.
+  size_t step = calculateStep(size, numThreads, params);
+  auto status = FileStatus::Continue;
+  while (status == FileStatus::Continue && !errorHolder.hasError()) {
+    // Make a new input queue that we will put the chunk's input data into.
+    auto in = std::make_shared<BufferWorkQueue>();
+    auto inGuard = makeScopeGuard([&] { in->finish(); });
+    // Make a new output queue that compress will put the compressed data into.
+    auto out = std::make_shared<BufferWorkQueue>();
+    // Start compression in the thread pool
+    executor.add([&errorHolder, in, out, step, params] {
+      return compress(
+          errorHolder, std::move(in), std::move(out), step, params);
+    });
+    // Pass the output queue to the writer thread.
+    chunks.push(std::move(out));
+    // Fill the input queue for the compression job we just started
+    status = readData(*in, ZSTD_CStreamInSize(), step, fd);
+  }
+  errorHolder.check(status != FileStatus::Error, "Error reading input");
+}
+
+/**
+ * Decompress a frame, whose data is streamed into `in`, and stream the output
+ * to `out`.
+ *
+ * @param errorHolder Used to report errors and check if an error occured
+ * @param in           Queue that we `pop()` input buffers from. It contains
+ *                      exactly one compressed frame.
+ * @param out          Queue that we `push()` decompressed output buffers to
+ */
+static void decompress(
+    ErrorHolder& errorHolder,
+    std::shared_ptr<BufferWorkQueue> in,
+    std::shared_ptr<BufferWorkQueue> out) {
+  auto guard = makeScopeGuard([&] { out->finish(); });
+  // Initialize the DCtx
+  std::unique_ptr<ZSTD_DStream, size_t (&)(ZSTD_DStream*)> ctx(
+      ZSTD_createDStream(), ZSTD_freeDStream);
+  if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
+    return;
+  }
+  {
+    auto err = ZSTD_initDStream(ctx.get());
+    if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
+      return;
+    }
+  }
+
+  const size_t outSize = ZSTD_DStreamOutSize();
+  Buffer inBuffer;
+  size_t returnCode = 0;
+  // Read a buffer in from the input queue
+  while (in->pop(inBuffer) && !errorHolder.hasError()) {
+    auto zstdInBuffer = makeZstdInBuffer(inBuffer);
+    // Decompress the whole buffer and send it to the output queue
+    while (!inBuffer.empty() && !errorHolder.hasError()) {
+      // Allocate a buffer with at least outSize bytes.
+      Buffer outBuffer(outSize);
+      auto zstdOutBuffer = makeZstdOutBuffer(outBuffer);
+      // Decompress
+      returnCode =
+          ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer);
+      if (!errorHolder.check(
+              !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) {
+        return;
+      }
+      // Pass the buffer with the decompressed data to the output queue
+      out->push(split(outBuffer, zstdOutBuffer));
+      // Advance past the input we already read
+      advance(inBuffer, zstdInBuffer);
+      if (returnCode == 0) {
+        // The frame is over, prepare to (maybe) start a new frame
+        ZSTD_initDStream(ctx.get());
+      }
+    }
+  }
+  if (!errorHolder.check(returnCode <= 1, "Incomplete block")) {
+    return;
+  }
+  // We've given ZSTD_decompressStream all of our data, but there may still
+  // be data to read.
+  while (returnCode == 1) {
+    // Allocate a buffer with at least outSize bytes.
+    Buffer outBuffer(outSize);
+    auto zstdOutBuffer = makeZstdOutBuffer(outBuffer);
+    // Pass in no input.
+    ZSTD_inBuffer zstdInBuffer{nullptr, 0, 0};
+    // Decompress
+    returnCode =
+        ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer);
+    if (!errorHolder.check(
+            !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) {
+      return;
+    }
+    // Pass the buffer with the decompressed data to the output queue
+    out->push(split(outBuffer, zstdOutBuffer));
+  }
+}
+
+void asyncDecompressFrames(
+    ErrorHolder& errorHolder,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& frames,
+    ThreadPool& executor,
+    FILE* fd) {
+  auto framesGuard = makeScopeGuard([&] { frames.finish(); });
+  // Split the source up into its component frames.
+  // If we find our recognized skippable frame we know the next frames size
+  // which means that we can decompress each standard frame in independently.
+  // Otherwise, we will decompress using only one decompression task.
+  const size_t chunkSize = ZSTD_DStreamInSize();
+  auto status = FileStatus::Continue;
+  while (status == FileStatus::Continue && !errorHolder.hasError()) {
+    // Make a new input queue that we will put the frames's bytes into.
+    auto in = std::make_shared<BufferWorkQueue>();
+    auto inGuard = makeScopeGuard([&] { in->finish(); });
+    // Make a output queue that decompress will put the decompressed data into
+    auto out = std::make_shared<BufferWorkQueue>();
+
+    size_t frameSize;
+    {
+      // Calculate the size of the next frame.
+      // frameSize is 0 if the frame info can't be decoded.
+      Buffer buffer(SkippableFrame::kSize);
+      auto bytesRead = std::fread(buffer.data(), 1, buffer.size(), fd);
+      status = fileStatus(fd);
+      if (bytesRead == 0 && status != FileStatus::Continue) {
+        break;
+      }
+      buffer.subtract(buffer.size() - bytesRead);
+      frameSize = SkippableFrame::tryRead(buffer.range());
+      in->push(std::move(buffer));
+    }
+    if (frameSize == 0) {
+      // We hit a non SkippableFrame, so this will be the last job.
+      // Make sure that we don't use too much memory
+      in->setMaxSize(64);
+      out->setMaxSize(64);
+    }
+    // Start decompression in the thread pool
+    executor.add([&errorHolder, in, out] {
+      return decompress(errorHolder, std::move(in), std::move(out));
+    });
+    // Pass the output queue to the writer thread
+    frames.push(std::move(out));
+    if (frameSize == 0) {
+      // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted
+      // Pass the rest of the source to this decompression task
+      while (status == FileStatus::Continue && !errorHolder.hasError()) {
+        status = readData(*in, chunkSize, chunkSize, fd);
+      }
+      break;
+    }
+    // Fill the input queue for the decompression job we just started
+    status = readData(*in, chunkSize, frameSize, fd);
+  }
+  errorHolder.check(status != FileStatus::Error, "Error reading input");
+}
+
+/// Write `data` to `fd`, returns true iff success.
+static bool writeData(ByteRange data, FILE* fd) {
+  while (!data.empty()) {
+    data.advance(std::fwrite(data.begin(), 1, data.size(), fd));
+    if (std::ferror(fd)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+size_t writeFile(
+    ErrorHolder& errorHolder,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
+    FILE* outputFd,
+    bool writeSkippableFrames) {
+  size_t bytesWritten = 0;
+  std::shared_ptr<BufferWorkQueue> out;
+  // Grab the output queue for each decompression job (in order).
+  while (outs.pop(out) && !errorHolder.hasError()) {
+    if (writeSkippableFrames) {
+      // If we are compressing and want to write skippable frames we can't
+      // start writing before compression is done because we need to know the
+      // compressed size.
+      // Wait for the compressed size to be available and write skippable frame
+      SkippableFrame frame(out->size());
+      if (!writeData(frame.data(), outputFd)) {
+        errorHolder.setError("Failed to write output");
+        return bytesWritten;
+      }
+      bytesWritten += frame.kSize;
+    }
+    // For each chunk of the frame: Pop it from the queue and write it
+    Buffer buffer;
+    while (out->pop(buffer) && !errorHolder.hasError()) {
+      if (!writeData(buffer.range(), outputFd)) {
+        errorHolder.setError("Failed to write output");
+        return bytesWritten;
+      }
+      bytesWritten += buffer.size();
+    }
+  }
+  return bytesWritten;
+}
+}

contrib/pzstd/Pzstd.h

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include "ErrorHolder.h"
+#include "Options.h"
+#include "utils/Buffer.h"
+#include "utils/Range.h"
+#include "utils/ThreadPool.h"
+#include "utils/WorkQueue.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#undef ZSTD_STATIC_LINKING_ONLY
+
+#include <cstddef>
+#include <memory>
+
+namespace pzstd {
+/**
+ * Runs pzstd with `options` and returns the number of bytes written.
+ * An error occurred if `errorHandler.hasError()`.
+ *
+ * @param options      The pzstd options to use for (de)compression
+ * @param errorHolder  Used to report errors and coordinate early shutdown
+ *                      if an error occured
+ * @returns            The number of bytes written.
+ */
+std::size_t pzstdMain(const Options& options, ErrorHolder& errorHolder);
+
+/**
+ * Streams input from `fd`, breaks input up into chunks, and compresses each
+ * chunk independently.  Output of each chunk gets streamed to a queue, and
+ * the output queues get put into `chunks` in order.
+ *
+ * @param errorHolder  Used to report errors and coordinate early shutdown
+ * @param chunks       Each compression jobs output queue gets `pushed()` here
+ *                      as soon as it is available
+ * @param executor     The thread pool to run compression jobs in
+ * @param fd           The input file descriptor
+ * @param size         The size of the input file if known, 0 otherwise
+ * @param numThreads   The number of threads in the thread pool
+ * @param parameters   The zstd parameters to use for compression
+ */
+void asyncCompressChunks(
+    ErrorHolder& errorHolder,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
+    ThreadPool& executor,
+    FILE* fd,
+    std::size_t size,
+    std::size_t numThreads,
+    ZSTD_parameters parameters);
+
+/**
+ * Streams input from `fd`.  If pzstd headers are available it breaks the input
+ * up into independent frames.  It sends each frame to an independent
+ * decompression job.  Output of each frame gets streamed to a queue, and
+ * the output queues get put into `frames` in order.
+ *
+ * @param errorHolder  Used to report errors and coordinate early shutdown
+ * @param frames       Each decompression jobs output queue gets `pushed()` here
+ *                      as soon as it is available
+ * @param executor     The thread pool to run compression jobs in
+ * @param fd           The input file descriptor
+ */
+void asyncDecompressFrames(
+    ErrorHolder& errorHolder,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& frames,
+    ThreadPool& executor,
+    FILE* fd);
+
+/**
+ * Streams input in from each queue in `outs` in order, and writes the data to
+ * `outputFd`.
+ *
+ * @param errorHolder          Used to report errors and coordinate early exit
+ * @param outs                 A queue of output queues, one for each
+ *                              (de)compression job.
+ * @param outputFd             The file descriptor to write to
+ * @param writeSkippableFrames Should we write pzstd headers?
+ * @returns                    The number of bytes written
+ */
+std::size_t writeFile(
+    ErrorHolder& errorHolder,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
+    FILE* outputFd,
+    bool writeSkippableFrames);
+}

contrib/pzstd/README.md

@@ -0,0 +1,48 @@
+# Parallel Zstandard (PZstandard)
+
+Parallel Zstandard is a Pigz-like tool for Zstandard.
+It provides Zstandard format compatible compression and decompression that is able to utilize multiple cores.
+It breaks the input up into equal sized chunks and compresses each chunk independently into a Zstandard frame.
+It then concatenates the frames together to produce the final compressed output.
+Optionally, with the `-p` option, PZstandard will write a 12 byte header for each frame that is a skippable frame in the Zstandard format, which tells PZstandard the size of the next compressed frame.
+When `-p` is specified for compression, PZstandard can decompress the output in parallel.
+
+## Usage
+
+Basic usage
+
+    pzstd input-file -o output-file -n num-threads [ -p ] -#   # Compression
+    pzstd -d input-file -o output-file -n num-threads          # Decompression
+
+PZstandard also supports piping and fifo pipes
+
+    cat input-file | pzstd -n num-threads [ -p ] -# -c > /dev/null
+
+For more options
+
+    pzstd --help
+
+## Benchmarks
+
+As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia).
+
+Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads
+------------------------------------------|-----------------------------------
+![Compression Speed vs Ratio](images/Cspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed.png "Decompression Speed")
+
+The test procedure was to run each of the following commands 2 times for each compression level, and take the minimum time.
+
+    time pzstd -# -n 4 -p -c silesia.tar     > silesia.tar.zst
+    time pzstd -d -n 4    -c silesia.tar.zst > /dev/null
+
+    time pigz  -# -p 4 -k -c silesia.tar     > silesia.tar.gz
+    time pigz  -d -p 4 -k -c silesia.tar.gz  > /dev/null
+
+PZstandard was tested using compression levels 1-19, and Pigz was tested using compression levels 1-9.
+Pigz cannot do parallel decompression, it simply does each of reading, decompression, and writing on separate threads.
+
+## Tests
+
+Tests require that you have [gtest](https://github.com/google/googletest) installed.
+Modify `GTEST_INC` and `GTEST_LIB` in `test/Makefile` and `utils/test/Makefile` to work for your install of gtest.
+Then run `make test` in the `contrib/pzstd` directory.

contrib/pzstd/SkippableFrame.cpp

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "SkippableFrame.h"
+#include "mem.h"
+#include "utils/Range.h"
+
+#include <cstdio>
+
+using namespace pzstd;
+
+SkippableFrame::SkippableFrame(std::uint32_t size) : frameSize_(size) {
+  MEM_writeLE32(data_.data(), kSkippableFrameMagicNumber);
+  MEM_writeLE32(data_.data() + 4, kFrameContentsSize);
+  MEM_writeLE32(data_.data() + 8, frameSize_);
+}
+
+/* static */ std::size_t SkippableFrame::tryRead(ByteRange bytes) {
+  if (bytes.size() < SkippableFrame::kSize ||
+      MEM_readLE32(bytes.begin()) != kSkippableFrameMagicNumber ||
+      MEM_readLE32(bytes.begin() + 4) != kFrameContentsSize) {
+    return 0;
+  }
+  return MEM_readLE32(bytes.begin() + 8);
+}

contrib/pzstd/SkippableFrame.h

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include "utils/Range.h"
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+namespace pzstd {
+/**
+ * We put a skippable frame before each frame.
+ * It contains a skippable frame magic number, the size of the skippable frame,
+ * and the size of the next frame.
+ * Each skippable frame is exactly 12 bytes in little endian format.
+ * The first 8 bytes are for compatibility with the ZSTD format.
+ * If we have N threads, the output will look like
+ *
+ * [0x184D2A50|4|size1] [frame1 of size size1]
+ * [0x184D2A50|4|size2] [frame2 of size size2]
+ * ...
+ * [0x184D2A50|4|sizeN] [frameN of size sizeN]
+ *
+ * Each sizeX is 4 bytes.
+ *
+ * These skippable frames should allow us to skip through the compressed file
+ * and only load at most N pages.
+ */
+class SkippableFrame {
+ public:
+  static constexpr std::size_t kSize = 12;
+
+ private:
+  std::uint32_t frameSize_;
+  std::array<std::uint8_t, kSize> data_;
+  static constexpr std::uint32_t kSkippableFrameMagicNumber = 0x184D2A50;
+  // Could be improved if the size fits in less bytes
+  static constexpr std::uint32_t kFrameContentsSize = kSize - 8;
+
+ public:
+   // Write the skippable frame to data_ in LE format.
+  explicit SkippableFrame(std::uint32_t size);
+
+  // Read the skippable frame from bytes in LE format.
+  static std::size_t tryRead(ByteRange bytes);
+
+  ByteRange data() const {
+    return {data_.data(), data_.size()};
+  }
+
+  // Size of the next frame.
+  std::size_t frameSize() const {
+    return frameSize_;
+  }
+};
+}

contrib/pzstd/main.cpp

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "ErrorHolder.h"
+#include "Options.h"
+#include "Pzstd.h"
+#include "utils/FileSystem.h"
+#include "utils/Range.h"
+#include "utils/ScopeGuard.h"
+#include "utils/ThreadPool.h"
+#include "utils/WorkQueue.h"
+
+using namespace pzstd;
+
+int main(int argc, const char** argv) {
+  Options options;
+  if (!options.parse(argc, argv)) {
+    return 1;
+  }
+
+  ErrorHolder errorHolder;
+  pzstdMain(options, errorHolder);
+
+  if (errorHolder.hasError()) {
+    std::fprintf(stderr, "Error: %s.\n", errorHolder.getError().c_str());
+    return 1;
+  }
+  return 0;
+}

contrib/pzstd/test/Makefile

@@ -0,0 +1,48 @@
+# ##########################################################################
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ##########################################################################
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+PZSTDDIR = ..
+PROGDIR = ../../../programs
+ZSTDDIR = ../../../lib
+
+# Set GTEST_INC and GTEST_LIB to work with your install of gtest
+GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
+GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
+
+CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+
+CXXFLAGS  ?= -O3
+CXXFLAGS  += -std=c++11
+CXXFLAGS  += $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
+
+datagen.o: $(PROGDIR)/datagen.*
+	$(CXX) $(FLAGS) $(PROGDIR)/datagen.c -c -o $@
+
+%: %.cpp *.h datagen.o
+	$(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) -lgtest -lgtest_main -lpthread
+
+.PHONY: test clean
+
+test: OptionsTest PzstdTest
+	@./OptionsTest$(EXT)
+	@./PzstdTest$(EXT)
+
+roundtrip: RoundTripTest
+	@./RoundTripTest$(EXT)
+
+clean:
+	@rm -f datagen.o OptionsTest PzstdTest RoundTripTest

contrib/pzstd/test/OptionsTest.cpp

@@ -0,0 +1,179 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "Options.h"
+
+#include <gtest/gtest.h>
+#include <array>
+
+using namespace pzstd;
+
+namespace pzstd {
+bool operator==(const Options& lhs, const Options& rhs) {
+  return lhs.numThreads == rhs.numThreads &&
+      lhs.maxWindowLog == rhs.maxWindowLog &&
+      lhs.compressionLevel == rhs.compressionLevel &&
+      lhs.decompress == rhs.decompress && lhs.inputFile == rhs.inputFile &&
+      lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite &&
+      lhs.pzstdHeaders == rhs.pzstdHeaders;
+}
+}
+
+TEST(Options, ValidInputs) {
+  {
+    Options options;
+    std::array<const char*, 6> args = {
+        {nullptr, "--num-threads", "5", "-o", "-", "-f"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {5, 23, 3, false, "-", "-", true, false};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    std::array<const char*, 6> args = {
+        {nullptr, "-n", "1", "input", "-19", "-p"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {1, 23, 19, false, "input", "input.zst", false, true};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    std::array<const char*, 10> args = {{nullptr,
+                                         "--ultra",
+                                         "-22",
+                                         "-n",
+                                         "1",
+                                         "--output",
+                                         "x",
+                                         "-d",
+                                         "x.zst",
+                                         "-f"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {1, 0, 22, true, "x.zst", "x", true, false};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    std::array<const char*, 6> args = {{nullptr,
+                                        "--num-threads",
+                                        "100",
+                                        "hello.zst",
+                                        "--decompress",
+                                        "--force"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {100, 23, 3, true, "hello.zst", "hello", true, false};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    std::array<const char*, 5> args = {{nullptr, "-", "-n", "1", "-c"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {1, 23, 3, false, "-", "-", false, false};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    std::array<const char*, 5> args = {{nullptr, "-", "-n", "1", "--stdout"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {1, 23, 3, false, "-", "-", false, false};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    std::array<const char*, 10> args = {{nullptr,
+                                         "-n",
+                                         "1",
+                                         "-",
+                                         "-5",
+                                         "-o",
+                                         "-",
+                                         "-u",
+                                         "-d",
+                                         "--pzstd-headers"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {1, 0, 5, true, "-", "-", false, true};
+  }
+  {
+    Options options;
+    std::array<const char*, 6> args = {
+        {nullptr, "silesia.tar", "-o", "silesia.tar.pzstd", "-n", "2"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {
+        2, 23, 3, false, "silesia.tar", "silesia.tar.pzstd", false, false};
+  }
+  {
+    Options options;
+    std::array<const char*, 3> args = {{nullptr, "-n", "1"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    std::array<const char*, 4> args = {{nullptr, "-", "-n", "1"}};
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, BadNumThreads) {
+  {
+    Options options;
+    std::array<const char*, 3> args = {{nullptr, "-o", "-"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    std::array<const char*, 5> args = {{nullptr, "-n", "0", "-o", "-"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    std::array<const char*, 4> args = {{nullptr, "-n", "-o", "-"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, BadCompressionLevel) {
+  {
+    Options options;
+    std::array<const char*, 3> args = {{nullptr, "x", "-20"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    std::array<const char*, 4> args = {{nullptr, "x", "-u", "-23"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, InvalidOption) {
+  {
+    Options options;
+    std::array<const char*, 3> args = {{nullptr, "x", "-x"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, BadOutputFile) {
+  {
+    Options options;
+    std::array<const char*, 5> args = {{nullptr, "notzst", "-d", "-n", "1"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, Extras) {
+  {
+    Options options;
+    std::array<const char*, 2> args = {{nullptr, "-h"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    std::array<const char*, 2> args = {{nullptr, "-V"}};
+    EXPECT_FALSE(options.parse(args.size(), args.data()));
+  }
+}

contrib/pzstd/test/PzstdTest.cpp

@@ -0,0 +1,121 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "datagen.h"
+#include "Pzstd.h"
+#include "test/RoundTrip.h"
+#include "utils/ScopeGuard.h"
+
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdio>
+#include <memory>
+#include <random>
+
+using namespace std;
+using namespace pzstd;
+
+TEST(Pzstd, SmallSizes) {
+  unsigned seed = std::random_device{}();
+  std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed);
+  std::mt19937 gen(seed);
+
+  for (unsigned len = 1; len < 1028; ++len) {
+    std::string inputFile = std::tmpnam(nullptr);
+    auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+    {
+      static uint8_t buf[1028];
+      RDG_genBuffer(buf, len, 0.5, 0.0, gen());
+      auto fd = std::fopen(inputFile.c_str(), "wb");
+      auto written = std::fwrite(buf, 1, len, fd);
+      std::fclose(fd);
+      ASSERT_EQ(written, len);
+    }
+    for (unsigned headers = 0; headers <= 1; ++headers) {
+      for (unsigned numThreads = 1; numThreads <= 4; numThreads *= 2) {
+        for (unsigned level = 1; level <= 8; level *= 8) {
+          auto errorGuard = makeScopeGuard([&] {
+            guard.dismiss();
+            std::fprintf(stderr, "file: %s\n", inputFile.c_str());
+            std::fprintf(stderr, "pzstd headers: %u\n", headers);
+            std::fprintf(stderr, "# threads: %u\n", numThreads);
+            std::fprintf(stderr, "compression level: %u\n", level);
+          });
+          Options options;
+          options.pzstdHeaders = headers;
+          options.overwrite = true;
+          options.inputFile = inputFile;
+          options.numThreads = numThreads;
+          options.compressionLevel = level;
+          ASSERT_TRUE(roundTrip(options));
+          errorGuard.dismiss();
+        }
+      }
+    }
+  }
+}
+
+TEST(Pzstd, LargeSizes) {
+  unsigned seed = std::random_device{}();
+  std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed);
+  std::mt19937 gen(seed);
+
+  for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) {
+    std::string inputFile = std::tmpnam(nullptr);
+    auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+    {
+      std::unique_ptr<uint8_t[]> buf(new uint8_t[len]);
+      RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen());
+      auto fd = std::fopen(inputFile.c_str(), "wb");
+      auto written = std::fwrite(buf.get(), 1, len, fd);
+      std::fclose(fd);
+      ASSERT_EQ(written, len);
+    }
+    for (unsigned headers = 0; headers <= 1; ++headers) {
+      for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
+        for (unsigned level = 1; level <= 4; level *= 2) {
+          auto errorGuard = makeScopeGuard([&] {
+            guard.dismiss();
+            std::fprintf(stderr, "file: %s\n", inputFile.c_str());
+            std::fprintf(stderr, "pzstd headers: %u\n", headers);
+            std::fprintf(stderr, "# threads: %u\n", numThreads);
+            std::fprintf(stderr, "compression level: %u\n", level);
+          });
+          Options options;
+          options.pzstdHeaders = headers;
+          options.overwrite = true;
+          options.inputFile = inputFile;
+          options.numThreads = numThreads;
+          options.compressionLevel = level;
+          ASSERT_TRUE(roundTrip(options));
+          errorGuard.dismiss();
+        }
+      }
+    }
+  }
+}
+
+TEST(Pzstd, ExtremelyCompressible) {
+  std::string inputFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+  {
+    std::unique_ptr<uint8_t[]> buf(new uint8_t[10000]);
+    std::memset(buf.get(), 'a', 10000);
+    auto fd = std::fopen(inputFile.c_str(), "wb");
+    auto written = std::fwrite(buf.get(), 1, 10000, fd);
+    std::fclose(fd);
+    ASSERT_EQ(written, 10000);
+  }
+  Options options;
+  options.pzstdHeaders = false;
+  options.overwrite = true;
+  options.inputFile = inputFile;
+  options.numThreads = 1;
+  options.compressionLevel = 1;
+  ASSERT_TRUE(roundTrip(options));
+}

contrib/pzstd/test/RoundTrip.h

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include "Options.h"
+#include "Pzstd.h"
+#include "utils/ScopeGuard.h"
+
+#include <cstdio>
+#include <string>
+#include <cstdint>
+#include <memory>
+
+namespace pzstd {
+
+inline bool check(std::string source, std::string decompressed) {
+  std::unique_ptr<std::uint8_t[]> sBuf(new std::uint8_t[1024]);
+  std::unique_ptr<std::uint8_t[]> dBuf(new std::uint8_t[1024]);
+
+  auto sFd = std::fopen(source.c_str(), "rb");
+  auto dFd = std::fopen(decompressed.c_str(), "rb");
+  auto guard = makeScopeGuard([&] {
+    std::fclose(sFd);
+    std::fclose(dFd);
+  });
+
+  size_t sRead, dRead;
+
+  do {
+    sRead = std::fread(sBuf.get(), 1, 1024, sFd);
+    dRead = std::fread(dBuf.get(), 1, 1024, dFd);
+    if (std::ferror(sFd) || std::ferror(dFd)) {
+      return false;
+    }
+    if (sRead != dRead) {
+      return false;
+    }
+
+    for (size_t i = 0; i < sRead; ++i) {
+      if (sBuf.get()[i] != dBuf.get()[i]) {
+        return false;
+      }
+    }
+  } while (sRead == 1024);
+  if (!std::feof(sFd) || !std::feof(dFd)) {
+    return false;
+  }
+  return true;
+}
+
+inline bool roundTrip(Options& options) {
+  std::string source = options.inputFile;
+  std::string compressedFile = std::tmpnam(nullptr);
+  std::string decompressedFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] {
+    std::remove(compressedFile.c_str());
+    std::remove(decompressedFile.c_str());
+  });
+
+  {
+    options.outputFile = compressedFile;
+    options.decompress = false;
+    ErrorHolder errorHolder;
+    pzstdMain(options, errorHolder);
+    if (errorHolder.hasError()) {
+      errorHolder.getError();
+      return false;
+    }
+  }
+  {
+    options.decompress = true;
+    options.inputFile = compressedFile;
+    options.outputFile = decompressedFile;
+    ErrorHolder errorHolder;
+    pzstdMain(options, errorHolder);
+    if (errorHolder.hasError()) {
+      errorHolder.getError();
+      return false;
+    }
+  }
+  return check(source, decompressedFile);
+}
+}

contrib/pzstd/test/RoundTripTest.cpp

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "datagen.h"
+#include "Options.h"
+#include "test/RoundTrip.h"
+#include "utils/ScopeGuard.h"
+
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+#include <random>
+
+using namespace std;
+using namespace pzstd;
+
+namespace {
+string
+writeData(size_t size, double matchProba, double litProba, unsigned seed) {
+  std::unique_ptr<uint8_t[]> buf(new uint8_t[size]);
+  RDG_genBuffer(buf.get(), size, matchProba, litProba, seed);
+  string file = tmpnam(nullptr);
+  auto fd = std::fopen(file.c_str(), "wb");
+  auto guard = makeScopeGuard([&] { std::fclose(fd); });
+  auto bytesWritten = std::fwrite(buf.get(), 1, size, fd);
+  if (bytesWritten != size) {
+    std::abort();
+  }
+  return file;
+}
+
+template <typename Generator>
+string generateInputFile(Generator& gen) {
+  // Use inputs ranging from 1 Byte to 2^16 Bytes
+  std::uniform_int_distribution<size_t> size{1, 1 << 16};
+  std::uniform_real_distribution<> prob{0, 1};
+  return writeData(size(gen), prob(gen), prob(gen), gen());
+}
+
+template <typename Generator>
+Options generateOptions(Generator& gen, const string& inputFile) {
+  Options options;
+  options.inputFile = inputFile;
+  options.overwrite = true;
+
+  std::bernoulli_distribution pzstdHeaders{0.75};
+  std::uniform_int_distribution<unsigned> numThreads{1, 32};
+  std::uniform_int_distribution<unsigned> compressionLevel{1, 10};
+
+  options.pzstdHeaders = pzstdHeaders(gen);
+  options.numThreads = numThreads(gen);
+  options.compressionLevel = compressionLevel(gen);
+
+  return options;
+}
+}
+
+int main(int argc, char** argv) {
+  std::mt19937 gen(std::random_device{}());
+
+  auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); });
+  for (unsigned i = 0; i < 10000; ++i) {
+    if (i % 100 == 0) {
+      std::fprintf(stderr, "Progress: %u%%\r", i / 100);
+    }
+    auto inputFile = generateInputFile(gen);
+    auto inputGuard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+    for (unsigned i = 0; i < 10; ++i) {
+      auto options = generateOptions(gen, inputFile);
+      if (!roundTrip(options)) {
+        std::fprintf(stderr, "numThreads: %u\n", options.numThreads);
+        std::fprintf(stderr, "level: %u\n", options.compressionLevel);
+        std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress);
+        std::fprintf(
+            stderr, "pzstd headers? %u\n", (unsigned)options.pzstdHeaders);
+        std::fprintf(stderr, "file: %s\n", inputFile.c_str());
+        return 1;
+      }
+    }
+  }
+  return 0;
+}

contrib/pzstd/utils/Buffer.h

@@ -0,0 +1,99 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include "utils/Range.h"
+
+#include <array>
+#include <cstddef>
+#include <memory>
+
+namespace pzstd {
+
+/**
+ * A `Buffer` has a pointer to a shared buffer, and a range of the buffer that
+ * it owns.
+ * The idea is that you can allocate one buffer, and write chunks into it
+ * and break off those chunks.
+ * The underlying buffer is reference counted, and will be destroyed when all
+ * `Buffer`s that reference it are destroyed.
+ */
+class Buffer {
+  std::shared_ptr<unsigned char> buffer_;
+  MutableByteRange range_;
+
+  static void delete_buffer(unsigned char* buffer) {
+    delete[] buffer;
+  }
+
+ public:
+  /// Construct an empty buffer that owns no data.
+  explicit Buffer() {}
+
+  /// Construct a `Buffer` that owns a new underlying buffer of size `size`.
+  explicit Buffer(std::size_t size)
+      : buffer_(new unsigned char[size], delete_buffer),
+        range_(buffer_.get(), buffer_.get() + size) {}
+
+  explicit Buffer(std::shared_ptr<unsigned char> buffer, MutableByteRange data)
+      : buffer_(buffer), range_(data) {}
+
+  Buffer(Buffer&&) = default;
+  Buffer& operator=(Buffer&&) & = default;
+
+  /**
+   * Splits the data into two pieces: [begin, begin + n), [begin + n, end).
+   * Their data both points into the same underlying buffer.
+   * Modifies the original `Buffer` to point to only [begin + n, end).
+   *
+   * @param n  The offset to split at.
+   * @returns  A buffer that owns the data [begin, begin + n).
+   */
+  Buffer splitAt(std::size_t n) {
+    auto firstPiece = range_.subpiece(0, n);
+    range_.advance(n);
+    return Buffer(buffer_, firstPiece);
+  }
+
+  /// Modifies the buffer to point to the range [begin + n, end).
+  void advance(std::size_t n) {
+    range_.advance(n);
+  }
+
+  /// Modifies the buffer to point to the range [begin, end - n).
+  void subtract(std::size_t n) {
+    range_.subtract(n);
+  }
+
+  /// Returns a read only `Range` pointing to the `Buffer`s data.
+  ByteRange range() const {
+    return range_;
+  }
+  /// Returns a mutable `Range` pointing to the `Buffer`s data.
+  MutableByteRange range() {
+    return range_;
+  }
+
+  const unsigned char* data() const {
+    return range_.data();
+  }
+
+  unsigned char* data() {
+    return range_.data();
+  }
+
+  std::size_t size() const {
+    return range_.size();
+  }
+
+  bool empty() const {
+    return range_.empty();
+  }
+};
+}

contrib/pzstd/utils/FileSystem.h

@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include "utils/Range.h"
+
+#include <sys/stat.h>
+#include <cstdint>
+#include <system_error>
+
+// A small subset of `std::filesystem`.
+// `std::filesystem` should be a drop in replacement.
+// See http://en.cppreference.com/w/cpp/filesystem for documentation.
+
+namespace pzstd {
+
+using file_status = struct stat;
+
+/// http://en.cppreference.com/w/cpp/filesystem/status
+inline file_status status(StringPiece path, std::error_code& ec) noexcept {
+  file_status status;
+  if (stat(path.data(), &status)) {
+    ec.assign(errno, std::generic_category());
+  } else {
+    ec.clear();
+  }
+  return status;
+}
+
+/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
+inline bool is_regular_file(file_status status) noexcept {
+  return S_ISREG(status.st_mode);
+}
+
+/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
+inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept {
+  return is_regular_file(status(path, ec));
+}
+
+/// http://en.cppreference.com/w/cpp/filesystem/file_size
+inline std::uintmax_t file_size(
+    StringPiece path,
+    std::error_code& ec) noexcept {
+  auto stat = status(path, ec);
+  if (ec) {
+    return -1;
+  }
+  if (!is_regular_file(stat)) {
+    ec.assign(ENOTSUP, std::generic_category());
+    return -1;
+  }
+  ec.clear();
+  return stat.st_size;
+}
+}

contrib/pzstd/utils/Likely.h

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+ 
+/**
+ * Compiler hints to indicate the fast path of an "if" branch: whether
+ * the if condition is likely to be true or false.
+ *
+ * @author Tudor Bosman (tudorb@fb.com)
+ */
+
+#pragma once
+
+#undef LIKELY
+#undef UNLIKELY
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+#define LIKELY(x) (__builtin_expect((x), 1))
+#define UNLIKELY(x) (__builtin_expect((x), 0))
+#else
+#define LIKELY(x) (x)
+#define UNLIKELY(x) (x)
+#endif

contrib/pzstd/utils/Range.h

@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+ 
+/**
+ * A subset of `folly/Range.h`.
+ * All code copied verbatiam modulo formatting
+ */
+#pragma once
+
+#include "utils/Likely.h"
+
+#include <cstddef>
+#include <cstring>
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+
+namespace pzstd {
+
+namespace detail {
+/*
+ *Use IsCharPointer<T>::type to enable const char* or char*.
+ *Use IsCharPointer<T>::const_type to enable only const char*.
+*/
+template <class T>
+struct IsCharPointer {};
+
+template <>
+struct IsCharPointer<char*> {
+  typedef int type;
+};
+
+template <>
+struct IsCharPointer<const char*> {
+  typedef int const_type;
+  typedef int type;
+};
+
+} // namespace detail
+
+template <typename Iter>
+class Range {
+  Iter b_;
+  Iter e_;
+
+ public:
+  using size_type = std::size_t;
+  using iterator = Iter;
+  using const_iterator = Iter;
+  using value_type = typename std::remove_reference<
+      typename std::iterator_traits<Iter>::reference>::type;
+  using reference = typename std::iterator_traits<Iter>::reference;
+
+  constexpr Range() : b_(), e_() {}
+  constexpr Range(Iter begin, Iter end) : b_(begin), e_(end) {}
+
+  constexpr Range(Iter begin, size_type size) : b_(begin), e_(begin + size) {}
+
+  template <class T = Iter, typename detail::IsCharPointer<T>::type = 0>
+  /* implicit */ Range(Iter str) : b_(str), e_(str + std::strlen(str)) {}
+
+  template <class T = Iter, typename detail::IsCharPointer<T>::const_type = 0>
+  /* implicit */ Range(const std::string& str)
+      : b_(str.data()), e_(b_ + str.size()) {}
+
+  // Allow implicit conversion from Range<From> to Range<To> if From is
+  // implicitly convertible to To.
+  template <
+      class OtherIter,
+      typename std::enable_if<
+          (!std::is_same<Iter, OtherIter>::value &&
+           std::is_convertible<OtherIter, Iter>::value),
+          int>::type = 0>
+  constexpr /* implicit */ Range(const Range<OtherIter>& other)
+      : b_(other.begin()), e_(other.end()) {}
+
+  Range(const Range&) = default;
+  Range(Range&&) = default;
+
+  Range& operator=(const Range&) & = default;
+  Range& operator=(Range&&) & = default;
+
+  constexpr size_type size() const {
+    return e_ - b_;
+  }
+  bool empty() const {
+    return b_ == e_;
+  }
+  Iter data() const {
+    return b_;
+  }
+  Iter begin() const {
+    return b_;
+  }
+  Iter end() const {
+    return e_;
+  }
+
+  void advance(size_type n) {
+    if (UNLIKELY(n > size())) {
+      throw std::out_of_range("index out of range");
+    }
+    b_ += n;
+  }
+
+  void subtract(size_type n) {
+    if (UNLIKELY(n > size())) {
+      throw std::out_of_range("index out of range");
+    }
+    e_ -= n;
+  }
+
+  Range subpiece(size_type first, size_type length = std::string::npos) const {
+    if (UNLIKELY(first > size())) {
+      throw std::out_of_range("index out of range");
+    }
+
+    return Range(b_ + first, std::min(length, size() - first));
+  }
+};
+
+using ByteRange = Range<const unsigned char*>;
+using MutableByteRange = Range<unsigned char*>;
+using StringPiece = Range<const char*>;
+}

contrib/pzstd/utils/ScopeGuard.h

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include <utility>
+
+namespace pzstd {
+
+/**
+ * Dismissable scope guard.
+ * `Function` must be callable and take no parameters.
+ * Unless `dissmiss()` is called, the callable is executed upon destruction of
+ * `ScopeGuard`.
+ *
+ * Example:
+ *
+ *   auto guard = makeScopeGuard([&] { cleanup(); });
+ */
+template <typename Function>
+class ScopeGuard {
+  Function function;
+  bool dismissed;
+
+ public:
+  explicit ScopeGuard(Function&& function)
+      : function(std::move(function)), dismissed(false) {}
+
+  void dismiss() {
+    dismissed = true;
+  }
+
+  ~ScopeGuard() noexcept {
+    if (!dismissed) {
+      function();
+    }
+  }
+};
+
+/// Creates a scope guard from `function`.
+template <typename Function>
+ScopeGuard<Function> makeScopeGuard(Function&& function) {
+  return ScopeGuard<Function>(std::forward<Function>(function));
+}
+}

contrib/pzstd/utils/ThreadPool.h

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include "utils/WorkQueue.h"
+
+#include <cstddef>
+#include <functional>
+#include <thread>
+#include <vector>
+
+namespace pzstd {
+/// A simple thread pool that pulls tasks off its queue in FIFO order.
+class ThreadPool {
+  std::vector<std::thread> threads_;
+
+  WorkQueue<std::function<void()>> tasks_;
+
+ public:
+  /// Constructs a thread pool with `numThreads` threads.
+  explicit ThreadPool(std::size_t numThreads) {
+    threads_.reserve(numThreads);
+    for (std::size_t i = 0; i < numThreads; ++i) {
+      threads_.emplace_back([&] {
+        std::function<void()> task;
+        while (tasks_.pop(task)) {
+          task();
+        }
+      });
+    }
+  }
+
+  /// Finishes all tasks currently in the queue.
+  ~ThreadPool() {
+    tasks_.finish();
+    for (auto& thread : threads_) {
+      thread.join();
+    }
+  }
+
+  /**
+   * Adds `task` to the queue of tasks to execute. Since `task` is a
+   * `std::function<>`, it cannot be a move only type. So any lambda passed must
+   * not capture move only types (like `std::unique_ptr`).
+   *
+   * @param task  The task to execute.
+   */
+  void add(std::function<void()> task) {
+    tasks_.push(std::move(task));
+  }
+};
+}

contrib/pzstd/utils/WorkQueue.h

@@ -0,0 +1,184 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include "utils/Buffer.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstddef>
+#include <condition_variable>
+#include <cstddef>
+#include <functional>
+#include <mutex>
+#include <queue>
+
+namespace pzstd {
+
+/// Unbounded thread-safe work queue.
+template <typename T>
+class WorkQueue {
+  // Protects all member variable access
+  std::mutex mutex_;
+  std::condition_variable readerCv_;
+  std::condition_variable writerCv_;
+
+  std::queue<T> queue_;
+  bool done_;
+  std::size_t maxSize_;
+
+  // Must have lock to call this function
+  bool full() const {
+    if (maxSize_ == 0) {
+      return false;
+    }
+    return queue_.size() >= maxSize_;
+  }
+
+ public:
+  /**
+   * Constructs an empty work queue with an optional max size.
+   * If `maxSize == 0` the queue size is unbounded.
+   *
+   * @param maxSize The maximum allowed size of the work queue.
+   */
+  WorkQueue(std::size_t maxSize = 0) : done_(false), maxSize_(maxSize) {}
+
+  /**
+   * Push an item onto the work queue.  Notify a single thread that work is
+   * available.  If `finish()` has been called, do nothing and return false.
+   *
+   * @param item  Item to push onto the queue.
+   * @returns     True upon success, false if `finish()` has been called.  An
+   *               item was pushed iff `push()` returns true.
+   */
+  bool push(T item) {
+    {
+      std::unique_lock<std::mutex> lock(mutex_);
+      while (full() && !done_) {
+        writerCv_.wait(lock);
+      }
+      if (done_) {
+        return false;
+      }
+      queue_.push(std::move(item));
+    }
+    readerCv_.notify_one();
+    return true;
+  }
+
+  /**
+   * Attempts to pop an item off the work queue.  It will block until data is
+   * available or `finish()` has been called.
+   *
+   * @param[out] item  If `pop` returns `true`, it contains the popped item.
+   *                    If `pop` returns `false`, it is unmodified.
+   * @returns          True upon success.  False if the queue is empty and
+   *                    `finish()` has been called.
+   */
+  bool pop(T& item) {
+    {
+      std::unique_lock<std::mutex> lock(mutex_);
+      while (queue_.empty() && !done_) {
+        readerCv_.wait(lock);
+      }
+      if (queue_.empty()) {
+        assert(done_);
+        return false;
+      }
+      item = std::move(queue_.front());
+      queue_.pop();
+    }
+    writerCv_.notify_one();
+    return true;
+  }
+
+  /**
+   * Sets the maximum queue size.  If `maxSize == 0` then it is unbounded.
+   *
+   * @param maxSize The new maximum queue size.
+   */
+  void setMaxSize(std::size_t maxSize) {
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      maxSize_ = maxSize;
+    }
+    writerCv_.notify_all();
+  }
+
+  /**
+   * Promise that `push()` won't be called again, so once the queue is empty
+   * there will never any more work.
+   */
+  void finish() {
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      assert(!done_);
+      done_ = true;
+    }
+    readerCv_.notify_all();
+    writerCv_.notify_all();
+  }
+
+  /// Blocks until `finish()` has been called (but the queue may not be empty).
+  void waitUntilFinished() {
+    std::unique_lock<std::mutex> lock(mutex_);
+    while (!done_) {
+      readerCv_.wait(lock);
+      // If we were woken by a push, we need to wake a thread waiting on pop().
+      if (!done_) {
+        lock.unlock();
+        readerCv_.notify_one();
+        lock.lock();
+      }
+    }
+  }
+};
+
+/// Work queue for `Buffer`s that knows the total number of bytes in the queue.
+class BufferWorkQueue {
+  WorkQueue<Buffer> queue_;
+  std::atomic<std::size_t> size_;
+
+ public:
+  BufferWorkQueue(std::size_t maxSize = 0) : queue_(maxSize), size_(0) {}
+
+  void push(Buffer buffer) {
+    size_.fetch_add(buffer.size());
+    queue_.push(std::move(buffer));
+  }
+
+  bool pop(Buffer& buffer) {
+    bool result = queue_.pop(buffer);
+    if (result) {
+      size_.fetch_sub(buffer.size());
+    }
+    return result;
+  }
+
+  void setMaxSize(std::size_t maxSize) {
+    queue_.setMaxSize(maxSize);
+  }
+
+  void finish() {
+    queue_.finish();
+  }
+
+  /**
+   * Blocks until `finish()` has been called.
+   *
+   * @returns The total number of bytes of all the `Buffer`s currently in the
+   *           queue.
+   */
+  std::size_t size() {
+    queue_.waitUntilFinished();
+    return size_.load();
+  }
+};
+}

contrib/pzstd/utils/test/BufferTest.cpp

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "utils/Buffer.h"
+#include "utils/Range.h"
+
+#include <gtest/gtest.h>
+#include <memory>
+
+using namespace pzstd;
+
+namespace {
+void deleter(const unsigned char* buf) {
+  delete[] buf;
+}
+}
+
+TEST(Buffer, Constructors) {
+  Buffer empty;
+  EXPECT_TRUE(empty.empty());
+  EXPECT_EQ(0, empty.size());
+
+  Buffer sized(5);
+  EXPECT_FALSE(sized.empty());
+  EXPECT_EQ(5, sized.size());
+
+  Buffer moved(std::move(sized));
+  EXPECT_FALSE(sized.empty());
+  EXPECT_EQ(5, sized.size());
+
+  Buffer assigned;
+  assigned = std::move(moved);
+  EXPECT_FALSE(sized.empty());
+  EXPECT_EQ(5, sized.size());
+}
+
+TEST(Buffer, BufferManagement) {
+  std::shared_ptr<unsigned char> buf(new unsigned char[10], deleter);
+  {
+    Buffer acquired(buf, MutableByteRange(buf.get(), buf.get() + 10));
+    EXPECT_EQ(2, buf.use_count());
+    Buffer moved(std::move(acquired));
+    EXPECT_EQ(2, buf.use_count());
+    Buffer assigned;
+    assigned = std::move(moved);
+    EXPECT_EQ(2, buf.use_count());
+
+    Buffer split = assigned.splitAt(5);
+    EXPECT_EQ(3, buf.use_count());
+
+    split.advance(1);
+    assigned.subtract(1);
+    EXPECT_EQ(3, buf.use_count());
+  }
+  EXPECT_EQ(1, buf.use_count());
+}
+
+TEST(Buffer, Modifiers) {
+  Buffer buf(10);
+  {
+    unsigned char i = 0;
+    for (auto& byte : buf.range()) {
+      byte = i++;
+    }
+  }
+
+  auto prefix = buf.splitAt(2);
+
+  ASSERT_EQ(2, prefix.size());
+  EXPECT_EQ(0, *prefix.data());
+
+  ASSERT_EQ(8, buf.size());
+  EXPECT_EQ(2, *buf.data());
+
+  buf.advance(2);
+  EXPECT_EQ(4, *buf.data());
+
+  EXPECT_EQ(9, *(buf.range().end() - 1));
+
+  buf.subtract(2);
+  EXPECT_EQ(7, *(buf.range().end() - 1));
+
+  EXPECT_EQ(4, buf.size());
+}

contrib/pzstd/utils/test/Makefile

@@ -0,0 +1,42 @@
+# ##########################################################################
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ##########################################################################
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+PZSTDDIR = ../..
+
+# Set GTEST_INC and GTEST_LIB to work with your install of gtest
+GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
+GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
+
+CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB)
+CXXFLAGS  ?= -O3
+CXXFLAGS  += -std=c++11
+CXXFLAGS  += $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
+
+%: %.cpp
+	$(CXX) $(FLAGS) $^ -o $@$(EXT) -lgtest -lgtest_main -lpthread
+
+.PHONY: test clean
+
+test: BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest
+	@./BufferTest$(EXT)
+	@./RangeTest$(EXT)
+	@./ScopeGuardTest$(EXT)
+	@./ThreadPoolTest$(EXT)
+	@./WorkQueueTest$(EXT)
+
+clean:
+	@rm -f BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest

contrib/pzstd/utils/test/RangeTest.cpp

@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "utils/Range.h"
+
+#include <gtest/gtest.h>
+#include <string>
+
+using namespace pzstd;
+
+// Range is directly copied from folly.
+// Just some sanity tests to make sure everything seems to work.
+
+TEST(Range, Constructors) {
+  StringPiece empty;
+  EXPECT_TRUE(empty.empty());
+  EXPECT_EQ(0, empty.size());
+
+  std::string str = "hello";
+  {
+    Range<std::string::const_iterator> piece(str.begin(), str.end());
+    EXPECT_EQ(5, piece.size());
+    EXPECT_EQ('h', *piece.data());
+    EXPECT_EQ('o', *(piece.end() - 1));
+  }
+
+  {
+    StringPiece piece(str.data(), str.size());
+    EXPECT_EQ(5, piece.size());
+    EXPECT_EQ('h', *piece.data());
+    EXPECT_EQ('o', *(piece.end() - 1));
+  }
+
+  {
+    StringPiece piece(str);
+    EXPECT_EQ(5, piece.size());
+    EXPECT_EQ('h', *piece.data());
+    EXPECT_EQ('o', *(piece.end() - 1));
+  }
+
+  {
+    StringPiece piece(str.c_str());
+    EXPECT_EQ(5, piece.size());
+    EXPECT_EQ('h', *piece.data());
+    EXPECT_EQ('o', *(piece.end() - 1));
+  }
+}
+
+TEST(Range, Modifiers) {
+  StringPiece range("hello world");
+  ASSERT_EQ(11, range.size());
+
+  {
+    auto hello = range.subpiece(0, 5);
+    EXPECT_EQ(5, hello.size());
+    EXPECT_EQ('h', *hello.data());
+    EXPECT_EQ('o', *(hello.end() - 1));
+  }
+  {
+    auto hello = range;
+    hello.subtract(6);
+    EXPECT_EQ(5, hello.size());
+    EXPECT_EQ('h', *hello.data());
+    EXPECT_EQ('o', *(hello.end() - 1));
+  }
+  {
+    auto world = range;
+    world.advance(6);
+    EXPECT_EQ(5, world.size());
+    EXPECT_EQ('w', *world.data());
+    EXPECT_EQ('d', *(world.end() - 1));
+  }
+
+  std::string expected = "hello world";
+  EXPECT_EQ(expected, std::string(range.begin(), range.end()));
+  EXPECT_EQ(expected, std::string(range.data(), range.size()));
+}

contrib/pzstd/utils/test/ScopeGuardTest.cpp

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "utils/ScopeGuard.h"
+
+#include <gtest/gtest.h>
+
+using namespace pzstd;
+
+TEST(ScopeGuard, Dismiss) {
+  {
+    auto guard = makeScopeGuard([&] { EXPECT_TRUE(false); });
+    guard.dismiss();
+  }
+}
+
+TEST(ScopeGuard, Executes) {
+  bool executed = false;
+  {
+    auto guard = makeScopeGuard([&] { executed = true; });
+  }
+  EXPECT_TRUE(executed);
+}

contrib/pzstd/utils/test/ThreadPoolTest.cpp

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "utils/ThreadPool.h"
+
+#include <gtest/gtest.h>
+#include <atomic>
+#include <thread>
+#include <vector>
+
+using namespace pzstd;
+
+TEST(ThreadPool, Ordering) {
+  std::vector<int> results;
+
+  {
+    ThreadPool executor(1);
+    for (int i = 0; i < 100; ++i) {
+      executor.add([ &results, i ] { results.push_back(i); });
+    }
+  }
+
+  for (int i = 0; i < 100; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(ThreadPool, AllJobsFinished) {
+  std::atomic<unsigned> numFinished{0};
+  std::atomic<bool> start{false};
+  {
+    ThreadPool executor(5);
+    for (int i = 0; i < 1000; ++i) {
+      executor.add([ &numFinished, &start ] {
+        while (!start.load()) {
+          // spin
+        }
+        ++numFinished;
+      });
+    }
+    start.store(true);
+  }
+  EXPECT_EQ(1000, numFinished.load());
+}
+
+TEST(ThreadPool, AddJobWhileJoining) {
+  std::atomic<bool> done{false};
+  {
+    ThreadPool executor(1);
+    executor.add([&executor, &done] {
+      while (!done.load()) {
+        std::this_thread::yield();
+      }
+      // Sleep for a second to be sure that we are joining
+      std::this_thread::sleep_for(std::chrono::seconds(1));
+      executor.add([] {
+        EXPECT_TRUE(false);
+      });
+    });
+    done.store(true);
+  }
+}

contrib/pzstd/utils/test/WorkQueueTest.cpp

@@ -0,0 +1,262 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "utils/Buffer.h"
+#include "utils/WorkQueue.h"
+
+#include <gtest/gtest.h>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+using namespace pzstd;
+
+namespace {
+struct Popper {
+  WorkQueue<int>* queue;
+  int* results;
+  std::mutex* mutex;
+
+  void operator()() {
+    int result;
+    while (queue->pop(result)) {
+      std::lock_guard<std::mutex> lock(*mutex);
+      results[result] = result;
+    }
+  }
+};
+}
+
+TEST(WorkQueue, SingleThreaded) {
+  WorkQueue<int> queue;
+  int result;
+
+  queue.push(5);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(5, result);
+
+  queue.push(1);
+  queue.push(2);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(1, result);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(2, result);
+
+  queue.push(1);
+  queue.push(2);
+  queue.finish();
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(1, result);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(2, result);
+  EXPECT_FALSE(queue.pop(result));
+
+  queue.waitUntilFinished();
+}
+
+TEST(WorkQueue, SPSC) {
+  WorkQueue<int> queue;
+  const int max = 100;
+
+  for (int i = 0; i < 10; ++i) {
+    queue.push(i);
+  }
+
+  std::thread thread([ &queue, max ] {
+    int result;
+    for (int i = 0;; ++i) {
+      if (!queue.pop(result)) {
+        EXPECT_EQ(i, max);
+        break;
+      }
+      EXPECT_EQ(i, result);
+    }
+  });
+
+  std::this_thread::yield();
+  for (int i = 10; i < max; ++i) {
+    queue.push(i);
+  }
+  queue.finish();
+
+  thread.join();
+}
+
+TEST(WorkQueue, SPMC) {
+  WorkQueue<int> queue;
+  std::vector<int> results(10000, -1);
+  std::mutex mutex;
+  std::vector<std::thread> threads;
+  for (int i = 0; i < 100; ++i) {
+    threads.emplace_back(Popper{&queue, results.data(), &mutex});
+  }
+
+  for (int i = 0; i < 10000; ++i) {
+    queue.push(i);
+  }
+  queue.finish();
+
+  for (auto& thread : threads) {
+    thread.join();
+  }
+
+  for (int i = 0; i < 10000; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(WorkQueue, MPMC) {
+  WorkQueue<int> queue;
+  std::vector<int> results(10000, -1);
+  std::mutex mutex;
+  std::vector<std::thread> popperThreads;
+  for (int i = 0; i < 100; ++i) {
+    popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
+  }
+
+  std::vector<std::thread> pusherThreads;
+  for (int i = 0; i < 10; ++i) {
+    auto min = i * 1000;
+    auto max = (i + 1) * 1000;
+    pusherThreads.emplace_back(
+        [ &queue, min, max ] {
+          for (int i = min; i < max; ++i) {
+            queue.push(i);
+          }
+        });
+  }
+
+  for (auto& thread : pusherThreads) {
+    thread.join();
+  }
+  queue.finish();
+
+  for (auto& thread : popperThreads) {
+    thread.join();
+  }
+
+  for (int i = 0; i < 10000; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(WorkQueue, BoundedSizeWorks) {
+  WorkQueue<int> queue(1);
+  int result;
+  queue.push(5);
+  queue.pop(result);
+  queue.push(5);
+  queue.pop(result);
+  queue.push(5);
+  queue.finish();
+  queue.pop(result);
+  EXPECT_EQ(5, result);
+}
+
+TEST(WorkQueue, BoundedSizePushAfterFinish) {
+  WorkQueue<int> queue(1);
+  int result;
+  queue.push(5);
+  std::thread pusher([&queue] {
+    queue.push(6);
+  });
+  // Dirtily try and make sure that pusher has run.
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  queue.finish();
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(5, result);
+  EXPECT_FALSE(queue.pop(result));
+
+  pusher.join();
+}
+
+TEST(WorkQueue, SetMaxSize) {
+  WorkQueue<int> queue(2);
+  int result;
+  queue.push(5);
+  queue.push(6);
+  queue.setMaxSize(1);
+  std::thread pusher([&queue] {
+    queue.push(7);
+  });
+  // Dirtily try and make sure that pusher has run.
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  queue.finish();
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(5, result);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(6, result);
+  EXPECT_FALSE(queue.pop(result));
+
+  pusher.join();
+}
+
+TEST(WorkQueue, BoundedSizeMPMC) {
+  WorkQueue<int> queue(100);
+  std::vector<int> results(10000, -1);
+  std::mutex mutex;
+  std::vector<std::thread> popperThreads;
+  for (int i = 0; i < 10; ++i) {
+    popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
+  }
+
+  std::vector<std::thread> pusherThreads;
+  for (int i = 0; i < 100; ++i) {
+    auto min = i * 100;
+    auto max = (i + 1) * 100;
+    pusherThreads.emplace_back(
+        [ &queue, min, max ] {
+          for (int i = min; i < max; ++i) {
+            queue.push(i);
+          }
+        });
+  }
+
+  for (auto& thread : pusherThreads) {
+    thread.join();
+  }
+  queue.finish();
+
+  for (auto& thread : popperThreads) {
+    thread.join();
+  }
+
+  for (int i = 0; i < 10000; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(BufferWorkQueue, SizeCalculatedCorrectly) {
+  {
+    BufferWorkQueue queue;
+    queue.finish();
+    EXPECT_EQ(0, queue.size());
+  }
+  {
+    BufferWorkQueue queue;
+    queue.push(Buffer(10));
+    queue.finish();
+    EXPECT_EQ(10, queue.size());
+  }
+  {
+    BufferWorkQueue queue;
+    queue.push(Buffer(10));
+    queue.push(Buffer(5));
+    queue.finish();
+    EXPECT_EQ(15, queue.size());
+  }
+  {
+    BufferWorkQueue queue;
+    queue.push(Buffer(10));
+    queue.push(Buffer(5));
+    queue.finish();
+    Buffer buffer;
+    queue.pop(buffer);
+    EXPECT_EQ(5, queue.size());
+  }
+}

lib/common/fse_decompress.c

@@ -42,12 +42,15 @@
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 

lib/common/xxhash.c

@@ -115,7 +115,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
 #  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
 #  define FORCE_INLINE static __forceinline
 #else
-#  if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #    ifdef __GNUC__
 #      define FORCE_INLINE static inline __attribute__((always_inline))
 #    else

lib/compress/fse_compress.c

@@ -41,12 +41,15 @@
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 

lib/compress/huf_compress.c

@@ -35,24 +35,8 @@
 /* **************************************************************
 *  Compiler specifics
 ****************************************************************/
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-/* inline is defined */
-#elif defined(_MSC_VER)
-#  define inline __inline
-#else
-#  define inline /* disable inline */
-#endif
-
-
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/compress/zstd_compress.c

@@ -8,7 +8,6 @@
  */
 
 
-
 /*-*******************************************************
 *  Compiler specifics
 *********************************************************/
@@ -17,11 +16,15 @@
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 

lib/decompress/huf_decompress.c

@@ -43,16 +43,8 @@
 #  define inline /* disable inline */
 #endif
 
-
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/decompress/zstd_decompress.c

@@ -67,11 +67,15 @@
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
 #  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
 #else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 

lib/dictBuilder/zdict.c

@@ -71,26 +71,18 @@ static const size_t g_min_fast_dictContent = 192;
 *  Console display
 ***************************************/
 #define DISPLAY(...)         { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
-#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-static unsigned g_displayLevel = 0;   /* 0 : no display;   1: errors;   2: default;  4: full information */
-
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if (ZDICT_clockSpan(g_time) > refreshRate)  \
-            { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 3 / 10;
-static clock_t g_time = 0;
+#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); }    /* 0 : no display;   1: errors;   2: default;  3: details;  4: debug */
 
 static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
 
-static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
+static void ZDICT_printHex(const void* ptr, size_t length)
 {
     const BYTE* const b = (const BYTE*)ptr;
     size_t u;
     for (u=0; u<length; u++) {
         BYTE c = b[u];
         if (c<32 || c>126) c = '.';   /* non-printable char */
-        DISPLAYLEVEL(dlevel, "%c", c);
+        DISPLAY("%c", c);
     }
 }
 
@@ -211,7 +203,7 @@ static void ZDICT_initDictItem(dictItem* d)
 static dictItem ZDICT_analyzePos(
                        BYTE* doneMarks,
                        const int* suffix, U32 start,
-                       const void* buffer, U32 minRatio)
+                       const void* buffer, U32 minRatio, U32 notificationLevel)
 {
     U32 lengthList[LLIMIT] = {0};
     U32 cumulLength[LLIMIT] = {0};
@@ -473,7 +465,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList)
 static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
                             const void* const buffer, size_t bufferSize,   /* buffer must end with noisy guard band */
                             const size_t* fileSizes, unsigned nbFiles,
-                            U32 minRatio)
+                            U32 minRatio, U32 notificationLevel)
 {
     int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0));
     int* const suffix = suffix0+1;
@@ -481,6 +473,13 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
     BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks));   /* +16 for overflow security */
     U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos));
     size_t result = 0;
+    clock_t displayClock = 0;
+    clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
+
+#   define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
+            if (ZDICT_clockSpan(displayClock) > refreshRate)  \
+            { displayClock = clock(); DISPLAY(__VA_ARGS__); \
+            if (notificationLevel>=4) fflush(stdout); } }
 
     /* init */
     DISPLAYLEVEL(2, "\r%70s\r", "");   /* clean display line */
@@ -518,7 +517,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
     {   U32 cursor; for (cursor=0; cursor < bufferSize; ) {
             dictItem solution;
             if (doneMarks[cursor]) { cursor++; continue; }
-            solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio);
+            solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel);
             if (solution.length==0) { cursor++; continue; }
             ZDICT_insertDictItem(dictList, dictListSize, solution);
             cursor += solution.length;
@@ -558,15 +557,15 @@ typedef struct
 
 static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
                             U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
-                            const void* src, size_t srcSize)
+                            const void* src, size_t srcSize, U32 notificationLevel)
 {
     size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog);
     size_t cSize;
 
     if (srcSize > blockSizeMax) srcSize = blockSizeMax;   /* protection vs large samples */
-	{	size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
-		if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
-	}
+    {  size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
+            if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
+    }
     cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
     if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
 
@@ -647,9 +646,10 @@ static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val,
 
 #define OFFCODE_MAX 30  /* only applicable to first block */
 static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
-                                 unsigned compressionLevel,
-                           const void*  srcBuffer, const size_t* fileSizes, unsigned nbFiles,
-                           const void* dictBuffer, size_t  dictBufferSize)
+                                   unsigned compressionLevel,
+                             const void*  srcBuffer, const size_t* fileSizes, unsigned nbFiles,
+                             const void* dictBuffer, size_t  dictBufferSize,
+                                   unsigned notificationLevel)
 {
     U32 countLit[256];
     HUF_CREATE_STATIC_CTABLE(hufTable, 255);
@@ -690,18 +690,19 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     memset(bestRepOffset, 0, sizeof(bestRepOffset));
     if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
     params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
-	{	size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
-		if (ZSTD_isError(beginResult)) {
-			eSize = ERROR(GENERIC);
-			DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
-			goto _cleanup;
-	}	}
+    {   size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
+            if (ZSTD_isError(beginResult)) {
+            eSize = ERROR(GENERIC);
+            DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
+            goto _cleanup;
+    }   }
 
     /* collect stats on all files */
     for (u=0; u<nbFiles; u++) {
         ZDICT_countEStats(esr, params,
-                        countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
-           (const char*)srcBuffer + pos, fileSizes[u]);
+                          countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
+                         (const char*)srcBuffer + pos, fileSizes[u],
+                          notificationLevel);
         pos += fileSizes[u];
     }
 
@@ -748,7 +749,6 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     }
     llLog = (U32)errorCode;
 
-
     /* write result to buffer */
     {   size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
         if (HUF_isError(hhSize)) {
@@ -828,6 +828,7 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
 {
     size_t hSize;
     int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
+    U32 const notificationLevel = params.notificationLevel;
 
     /* dictionary header */
     MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
@@ -844,7 +845,8 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
     {   size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
                                   compressionLevel,
                                   samplesBuffer, samplesSizes, nbSamples,
-                                  (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
+                                  (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize,
+                                  notificationLevel);
         if (ZDICT_isError(eSize)) return eSize;
         hSize += eSize;
     }
@@ -872,6 +874,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
     size_t const targetDictSize = maxDictSize;
     size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples);
     size_t dictSize = 0;
+    U32 const notificationLevel = params.notificationLevel;
 
     /* checks */
     if (!dictList) return ERROR(memory_allocation);
@@ -880,16 +883,15 @@ size_t ZDICT_trainFromBuffer_unsafe(
 
     /* init */
     ZDICT_initDictItem(dictList);
-    g_displayLevel = params.notificationLevel;
 
     /* build dictionary */
     ZDICT_trainBuffer(dictList, dictListSize,
                     samplesBuffer, samplesBuffSize,
                     samplesSizes, nbSamples,
-                    minRep);
+                    minRep, notificationLevel);
 
     /* display best matches */
-    if (g_displayLevel>= 3) {
+    if (params.notificationLevel>= 3) {
         U32 const nb = MIN(25, dictList[0].pos);
         U32 const dictContentSize = ZDICT_dictSize(dictList);
         U32 u;
@@ -901,7 +903,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
             U32 printedLength = MIN(40, length);
             DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
                          u, length, pos, dictList[u].savings);
-            ZDICT_printHex(3, (const char*)samplesBuffer+pos, printedLength);
+            ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);
             DISPLAYLEVEL(3, "| \n");
     }   }
 

lib/dictBuilder/zdict.h

@@ -68,7 +68,7 @@ typedef struct {
     int      compressionLevel;   /* 0 means default; target a specific zstd compression level */
     unsigned notificationLevel;  /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
     unsigned dictID;             /* 0 means auto mode (32-bits random value); other : force dictID value */
-    unsigned reserved[2];        /* space for future parameters */
+    unsigned reserved[2];        /* reserved space for future parameters */
 } ZDICT_params_t;
 
 

lib/legacy/zstd_v01.c

@@ -139,11 +139,15 @@ typedef struct
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
 #  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -1300,17 +1304,9 @@ typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes;   /* expo
 #endif
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
-#else
-#  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/legacy/zstd_v02.c

@@ -1122,12 +1122,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -1595,15 +1598,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
 
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 
@@ -2790,17 +2785,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
 #endif
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
-#else
-#  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/legacy/zstd_v03.c

@@ -1122,12 +1122,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -1588,25 +1591,13 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 /* inline is defined */
 #elif defined(_MSC_VER)
+#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  define inline __inline
 #else
 #  define inline /* disable inline */
 #endif
 
 
-#ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
-#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
-#endif
-
-
 /****************************************************************
 *  Includes
 ****************************************************************/
@@ -2432,17 +2423,11 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
 #endif
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
 #else
 #  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/legacy/zstd_v04.c

@@ -1380,12 +1380,15 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -2021,15 +2024,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const
 
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 
@@ -2837,17 +2832,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
 *  Compiler specifics
 *********************************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
-#else
-#  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/legacy/zstd_v05.c

@@ -1380,12 +1380,15 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -2002,14 +2005,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
 
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 
@@ -2858,17 +2854,9 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
 *  Compiler specifics
 *********************************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
-#else
-#  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/legacy/zstd_v06.c

@@ -1665,12 +1665,15 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -2238,14 +2241,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
 
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 
@@ -3054,16 +3050,9 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
 *  Compiler specifics
 *********************************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/legacy/zstd_v07.c

@@ -1680,17 +1680,18 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #else
-#  ifdef __GNUC__
-#    define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
 #  else
-#    define FORCE_INLINE static inline
-#  endif
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
-
-
 /* **************************************************************
 *  Error Management
 ****************************************************************/
@@ -2006,14 +2007,7 @@ size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
 
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 
@@ -3176,17 +3170,10 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
 *  Compiler specifics
 *********************************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
 #  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
 #endif
 
 

lib/libzstd.pc.in

@@ -8,7 +8,7 @@ includedir=@INCLUDEDIR@
 
 Name: zstd
 Description: lossless compression algorithm library
-URL: https://github.com/Cyan4973/zstd
+URL: https://github.com/facebook/zstd
 Version: @VERSION@
 Libs: -L@LIBDIR@ -lzstd
 Cflags: -I@INCLUDEDIR@

lib/zstd.h

@@ -33,7 +33,7 @@ extern "C" {
 /*=======   Version   =======*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    0
-#define ZSTD_VERSION_RELEASE  0
+#define ZSTD_VERSION_RELEASE  1
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str

programs/.gitignore

@@ -7,6 +7,7 @@ zstd-decompress
 # Object files
 *.o
 *.ko
+default.profraw
 
 # Executables
 *.exe

programs/Makefile

@@ -97,6 +97,7 @@ zstd-pgo : clean zstd
 	./zstd -b7i2 $(PROFILE_WITH)
 	./zstd -b5 $(PROFILE_WITH)
 	$(RM) zstd
+	$(RM) $(ZSTDDIR)/decompress/zstd_decompress.o
 	$(MAKE) zstd MOREFLAGS=-fprofile-use
 
 zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
@@ -116,9 +117,10 @@ zstd-small: clean
 
 clean:
 	$(MAKE) -C ../lib clean
-	@$(RM) ../lib/decompress/*.o
+	@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
 	@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
-        zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT)
+        zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
+        *.gcda default.profraw
 	@echo Cleaning completed
 
 

programs/zstd.1

@@ -135,7 +135,7 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
 
 
 .SH BUGS
-Report bugs at:- https://github.com/Cyan4973/zstd/issues
+Report bugs at:- https://github.com/facebook/zstd/issues
 
 .SH AUTHOR
 Yann Collet

programs/zstdcli.c

@@ -52,7 +52,7 @@
 #  include <io.h>       /* _isatty */
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
 #else
-#  if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE)
+#  if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__))  /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
 #    include <unistd.h>   /* isatty */
 #    define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
 #  else
@@ -83,7 +83,7 @@
 
 static const char*    g_defaultDictName = "dictionary";
 static const unsigned g_defaultMaxDictSize = 110 KB;
-static const int      g_defaultDictCLevel = 5;
+static const int      g_defaultDictCLevel = 3;
 static const unsigned g_defaultSelectivityLevel = 9;
 
 
@@ -182,7 +182,7 @@ static void waitEnter(void)
 /*! readU32FromChar() :
     @return : unsigned integer value reach from input in `char` format
     Will also modify `*stringPtr`, advancing it to position where it stopped reading.
-    Note : this function can overflow if result > MAX_UINT */
+    Note : this function can overflow if digit string > MAX_UINT */
 static unsigned readU32FromChar(const char** stringPtr)
 {
     unsigned result = 0;
@@ -289,13 +289,13 @@ int main(int argCount, char** argv)
                 argument++;
 
                 while (argument[0]!=0) {
-    #ifndef ZSTD_NOCOMPRESS
+#ifndef ZSTD_NOCOMPRESS
                     /* compression Level */
                     if ((*argument>='0') && (*argument<='9')) {
                         dictCLevel = cLevel = readU32FromChar(&argument);
                         continue;
                     }
-    #endif
+#endif
 
                     switch(argument[0])
                     {
@@ -337,7 +337,7 @@ int main(int argCount, char** argv)
                         /* recursive */
                     case 'r': recursive=1; argument++; break;
 
-    #ifndef ZSTD_NOBENCH
+#ifndef ZSTD_NOBENCH
                         /* Benchmark */
                     case 'b': bench=1; argument++; break;
 
@@ -368,7 +368,7 @@ int main(int argCount, char** argv)
                             BMK_SetBlockSize(bSize);
                         }
                         break;
-    #endif   /* ZSTD_NOBENCH */
+#endif   /* ZSTD_NOBENCH */
 
                         /* Dictionary Selection level */
                     case 's':
@@ -378,11 +378,11 @@ int main(int argCount, char** argv)
 
                         /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
                     case 'p': argument++;
-    #ifndef ZSTD_NOBENCH
+#ifndef ZSTD_NOBENCH
                         if ((*argument>='0') && (*argument<='9')) {
                             BMK_setAdditionalParam(readU32FromChar(&argument));
                         } else
-    #endif
+#endif
                             main_pause=1;
                         break;
                         /* unknown command */
@@ -470,7 +470,7 @@ int main(int argCount, char** argv)
 
     /* Check if input/output defined as console; trigger an error in this case */
     if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
-    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode))
+    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && strcmp(filenameTable[0], stdinmark) && !(forceStdout && decode))
         CLEAN_RETURN(badusage(programName));
 
     /* user-selected output filename, only possible with a single file */

projects/README.md

@@ -14,7 +14,7 @@ The following projects are included with the zstd distribution:
 #### How to compile zstd with Visual Studio
 
 1. Install Visual Studio e.g. VS 2015 Community Edition (it's free).
-2. Download the latest version of zstd from https://github.com/Cyan4973/zstd/releases
+2. Download the latest version of zstd from https://github.com/facebook/zstd/releases
 3. Decompress ZIP archive.
 4. Go to decompressed directory then to `projects` then `VS2010` and open `zstd.sln`
 5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree.

projects/cmake/CMakeLists.txt

@@ -1,34 +1,10 @@
 # ################################################################
-# zstd - Makefile
-# Copyright (C) Yann Collet 2014-2016
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 # All rights reserved.
-# 
-# BSD license
 #
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-# 
-# * Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
-# 
-# * Redistributions in binary form must reproduce the above copyright notice, this
-#   list of conditions and the following disclaimer in the documentation and/or
-#   other materials provided with the distribution.
-# 
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-# 
-# You can contact the author at :
-#  - zstd source repository : https://github.com/Cyan4973/zstd
-#  - Public forum : https://groups.google.com/forum/#!forum/lz4c
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
 # ################################################################
 
 PROJECT(zstd)

tests/Makefile

@@ -160,10 +160,10 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest
 	$(VALGRIND) ./datagen -g50M > $(VOID)
 	$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
 	./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID)
-	./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID)
+	./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
 	./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
-	$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -o $(VOID)
-	./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID)
+	$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID)
+	./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
 	@rm tmp
 	$(VALGRIND) ./fuzzer -T1mn -t1
 	$(VALGRIND) ./fullbench -i1

tests/fuzzer.c

@@ -23,7 +23,6 @@
 **************************************/
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
-#include <sys/timeb.h>    /* timeb */
 #include <string.h>       /* strcmp */
 #include <time.h>         /* clock_t */
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressContinue, ZSTD_compressBlock */

tests/playTests.sh

@@ -41,6 +41,7 @@ $ECHO "\nStarting playTests.sh isWindows=$isWindows"
 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
 
 file $ZSTD
+
 $ECHO "\n**** simple tests **** "
 
 ./datagen > tmp
@@ -51,6 +52,8 @@ $ZSTD -99 -f tmp  # too large compression level, automatic sized down
 $ECHO "test : compress to stdout"
 $ZSTD tmp -c > tmpCompressed
 $ZSTD tmp --stdout > tmpCompressed       # long command format
+$ECHO "test : implied stdout when input is stdin"
+$ECHO bob | $ZSTD | $ZSTD -d
 $ECHO "test : null-length file roundtrip"
 $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
 $ECHO "test : decompress file with wrong suffix (must fail)"

tests/test-zstd-speed.py

@@ -18,7 +18,7 @@ import traceback
 import hashlib
 
 script_version = 'v0.8.0 (2016-08-03)'
-default_repo_url = 'https://github.com/Cyan4973/zstd.git'
+default_repo_url = 'https://github.com/facebook/zstd.git'
 working_dir_name = 'speedTest'
 working_path = os.getcwd() + '/' + working_dir_name     # /path/to/zstd/tests/speedTest
 clone_path = working_path + '/' + 'zstd'                # /path/to/zstd/tests/speedTest/zstd

tests/test-zstd-versions.py

@@ -19,7 +19,7 @@ import sys
 import subprocess
 from subprocess import Popen, PIPE
 
-repo_url = 'https://github.com/Cyan4973/zstd.git'
+repo_url = 'https://github.com/facebook/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
 make_cmd = 'make'
 git_cmd = 'git'

tests/zbufftest.c

@@ -23,7 +23,7 @@
 **************************************/
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
-#include <sys/timeb.h>    /* timeb */
+#include <time.h>         /* clock_t, clock() */
 #include <string.h>       /* strcmp */
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
@@ -58,13 +58,13 @@ static const U32 prime2 = 2246822519U;
 static U32 g_displayLevel = 2;
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
-            { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
+            if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
             if (g_displayLevel>=4) fflush(stdout); } }
-static const U32 g_refreshRate = 150;
-static U32 g_displayTime = 0;
+static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_displayClock = 0;
 
-static U32 g_testTime = 0;
+static clock_t g_clockTime = 0;
 
 
 /*-*******************************************************
@@ -72,23 +72,9 @@ static U32 g_testTime = 0;
 *********************************************************/
 #define MAX(a,b) ((a)>(b)?(a):(b))
 
-static U32 FUZ_GetMilliStart(void)
+static clock_t FUZ_GetClockSpan(clock_t clockStart)
 {
-    struct timeb tb;
-    U32 nCount;
-    ftime( &tb );
-    nCount = (U32) (((tb.time & 0xFFFFF) * 1000) +  tb.millitm);
-    return nCount;
-}
-
-
-static U32 FUZ_GetMilliSpan(U32 nTimeStart)
-{
-    U32 const nCurrent = FUZ_GetMilliStart();
-    U32 nSpan = nCurrent - nTimeStart;
-    if (nTimeStart > nCurrent)
-        nSpan += 0x100000 * 1000;
-    return nSpan;
+    return clock() - clockStart;  /* works even when overflow. Max span ~ 30 mn */
 }
 
 /*! FUZ_rand() :
@@ -291,7 +277,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     U32 coreSeed = seed;
     ZBUFF_CCtx* zc;
     ZBUFF_DCtx* zd;
-    U32 startTime = FUZ_GetMilliStart();
+    clock_t startClock = clock();
 
     /* allocations */
     zc = ZBUFF_createCCtx();
@@ -321,7 +307,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         FUZ_rand(&coreSeed);
 
     /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
         U32 lseed;
         const BYTE* srcBuffer;
         const BYTE* dict;
@@ -543,7 +529,7 @@ int main(int argc, const char** argv)
 
                 case 'i':
                     argument++;
-                    nbTests=0; g_testTime=0;
+                    nbTests=0; g_clockTime=0;
                     while ((*argument>='0') && (*argument<='9')) {
                         nbTests *= 10;
                         nbTests += *argument - '0';
@@ -553,15 +539,15 @@ int main(int argc, const char** argv)
 
                 case 'T':
                     argument++;
-                    nbTests=0; g_testTime=0;
+                    nbTests=0; g_clockTime=0;
                     while ((*argument>='0') && (*argument<='9')) {
-                        g_testTime *= 10;
-                        g_testTime += *argument - '0';
+                        g_clockTime *= 10;
+                        g_clockTime += *argument - '0';
                         argument++;
                     }
-                    if (*argument=='m') g_testTime *=60, argument++;
+                    if (*argument=='m') g_clockTime *=60, argument++;
                     if (*argument=='n') argument++;
-                    g_testTime *= 1000;
+                    g_clockTime *= CLOCKS_PER_SEC;
                     break;
 
                 case 's':
@@ -605,7 +591,11 @@ int main(int argc, const char** argv)
     /* Get Seed */
     DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
 
-    if (!seedset) seed = FUZ_GetMilliStart() % 10000;
+    if (!seedset) {
+        time_t const t = time(NULL);
+        U32 const h = XXH32(&t, sizeof(t), 1);
+        seed = h % 10000;
+    }
     DISPLAY("Seed = %u\n", seed);
     if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
 

tests/zstreamtest.c

@@ -23,13 +23,13 @@
 **************************************/
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
-#include <sys/timeb.h>    /* timeb */
+#include <time.h>         /* clock_t, clock() */
 #include <string.h>       /* strcmp */
 #include "mem.h"
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel, ZSTD_customMem */
 #include "zstd.h"         /* ZSTD_compressBound */
 #include "datagen.h"      /* RDG_genBuffer */
-#define XXH_STATIC_LINKING_ONLY
+#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"       /* XXH64_* */
 
 
@@ -55,13 +55,13 @@ static const U32 prime2 = 2246822519U;
 static U32 g_displayLevel = 2;
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
-            { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
+            if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
             if (g_displayLevel>=4) fflush(stdout); } }
-static const U32 g_refreshRate = 150;
-static U32 g_displayTime = 0;
+static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_displayClock = 0;
 
-static U32 g_testTime = 0;
+static clock_t g_clockTime = 0;
 
 
 /*-*******************************************************
@@ -69,22 +69,9 @@ static U32 g_testTime = 0;
 *********************************************************/
 #define MAX(a,b) ((a)>(b)?(a):(b))
 
-static U32 FUZ_GetMilliStart(void)
+static clock_t FUZ_GetClockSpan(clock_t clockStart)
 {
-    struct timeb tb;
-    U32 nCount;
-    ftime( &tb );
-    nCount = (U32) (((tb.time & 0xFFFFF) * 1000) +  tb.millitm);
-    return nCount;
-}
-
-static U32 FUZ_GetMilliSpan(U32 nTimeStart)
-{
-    U32 const nCurrent = FUZ_GetMilliStart();
-    U32 nSpan = nCurrent - nTimeStart;
-    if (nTimeStart > nCurrent)
-        nSpan += 0x100000 * 1000;
-    return nSpan;
+    return clock() - clockStart;  /* works even when overflow. Max span ~ 30 mn */
 }
 
 /*! FUZ_rand() :
@@ -336,7 +323,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     U32 coreSeed = seed;
     ZSTD_CStream* zc;
     ZSTD_DStream* zd;
-    U32 startTime = FUZ_GetMilliStart();
+    clock_t startClock = clock();
 
     /* allocations */
     zc = ZSTD_createCStream();
@@ -366,7 +353,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         FUZ_rand(&coreSeed);
 
     /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
         U32 lseed;
         const BYTE* srcBuffer;
         const BYTE* dict;
@@ -594,7 +581,7 @@ int main(int argc, const char** argv)
 
                 case 'i':
                     argument++;
-                    nbTests=0; g_testTime=0;
+                    nbTests=0; g_clockTime=0;
                     while ((*argument>='0') && (*argument<='9')) {
                         nbTests *= 10;
                         nbTests += *argument - '0';
@@ -604,15 +591,15 @@ int main(int argc, const char** argv)
 
                 case 'T':
                     argument++;
-                    nbTests=0; g_testTime=0;
+                    nbTests=0; g_clockTime=0;
                     while ((*argument>='0') && (*argument<='9')) {
-                        g_testTime *= 10;
-                        g_testTime += *argument - '0';
+                        g_clockTime *= 10;
+                        g_clockTime += *argument - '0';
                         argument++;
                     }
-                    if (*argument=='m') g_testTime *=60, argument++;
+                    if (*argument=='m') g_clockTime *=60, argument++;
                     if (*argument=='n') argument++;
-                    g_testTime *= 1000;
+                    g_clockTime *= CLOCKS_PER_SEC;
                     break;
 
                 case 's':
@@ -656,7 +643,12 @@ int main(int argc, const char** argv)
     /* Get Seed */
     DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
 
-    if (!seedset) seed = FUZ_GetMilliStart() % 10000;
+    if (!seedset) {
+        time_t const t = time(NULL);
+        U32 const h = XXH32(&t, sizeof(t), 1);
+        seed = h % 10000;
+    }
+
     DISPLAY("Seed = %u\n", seed);
     if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
 

zstd.rb

@@ -1,18 +0,0 @@
-class Zstd < Formula
-  desc "Zstandard - Fast real-time compression algorithm"
-  homepage "http://www.zstd.net/"
-  url "https://github.com/Cyan4973/zstd/archive/v1.0.0.tar.gz"
-  sha256 "197e6ef74da878cbf72844f38461bb18129d144fd5221b3598e973ecda6f5963"
-
-  def install
-    system "make", "install", "PREFIX=#{prefix}"
-  end
-
-  test do
-    (testpath/"input.txt").write("Hello, world." * 10)
-    system "#{bin}/zstd", "input.txt", "-o", "compressed.zst"
-    system "#{bin}/zstd", "--test", "compressed.zst"
-    system "#{bin}/zstd", "-d", "compressed.zst", "-o", "decompressed.txt"
-    system "cmp", "input.txt", "decompressed.txt"
-  end
-end

zstd_compression_format.md

@@ -551,7 +551,7 @@ Let's presume the following Huffman tree must be described :
 The tree depth is 4, since its smallest element uses 4 bits.
 Value `5` will not be listed, nor will values above `5`.
 Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
-Weight formula is : 
+Weight formula is :
 ```
 Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0
 ```
@@ -779,7 +779,7 @@ which specifies `Baseline` and `Number_of_Bits` to add.
 _Codes_ are FSE compressed,
 and interleaved with raw additional bits in the same bitstream.
 
-##### Literals length codes 
+##### Literals length codes
 
 Literals length codes are values ranging from `0` to `35` included.
 They define lengths from 0 to 131071 bytes.
@@ -1126,10 +1126,10 @@ When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
 Dictionary format
 -----------------
 
-`zstd` is compatible with "pure content" dictionaries, free of any format restriction.
+`zstd` is compatible with "raw content" dictionaries, free of any format restriction.
 But dictionaries created by `zstd --train` follow a format, described here.
 
-__Pre-requisites__ : a dictionary has a known length,
+__Pre-requisites__ : a dictionary has a size,
                      defined either by a buffer limit, or a file size.
 
 | `Magic_Number` | `Dictionary_ID` | `Entropy_Tables` | `Content` |
@@ -1151,20 +1151,21 @@ _Reserved ranges :_
               - high range : >= (2^31)
 
 __`Entropy_Tables`__ : following the same format as a [compressed blocks].
-            They are stored in following order :
-            Huffman tables for literals, FSE table for offsets,
-            FSE table for match lengths, and FSE table for literals lengths.
-            It's finally followed by 3 offset values, populating recent offsets,
-            stored in order, 4-bytes little-endian each, for a total of 12 bytes.
+              They are stored in following order :
+              Huffman tables for literals, FSE table for offsets,
+              FSE table for match lengths, and FSE table for literals lengths.
+              It's finally followed by 3 offset values, populating recent offsets,
+              stored in order, 4-bytes little-endian each, for a total of 12 bytes.
 
-__`Content`__ : Where the actual dictionary content is.
-              Content size depends on Dictionary size.
+__`Content`__ : The rest of the dictionary is its content.
+              The content act as a "past" in front of data to compress or decompress.
 
 [compressed blocks]: #the-format-of-compressed_block
 
 
 Version changes
 ---------------
+- 0.2.1 : clarify field names, by Przemyslaw Skibinski
 - 0.2.0 : numerous format adjustments for zstd v0.8
 - 0.1.2 : limit Huffman tree depth to 11 bits
 - 0.1.1 : reserved dictID ranges