Merge pull request #382 from terrelln/dev

[pzstd] CLI + memory usage + 32 bit mode

.travis.yml

@@ -66,12 +66,18 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
+      install:
+        - export CXX="g++-4.8" CC="gcc-4.8"
+      env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd pzstd32 && make -C contrib/pzstd googletest32 && make -C contrib/pzstd test32 && make -C contrib/pzstd clean"
       addons:
         apt:
           packages:
             - libc6-dev-i386
-            - g++-multilib 
+            - g++-multilib
+            - gcc-4.8
+            - gcc-4.8-multilib
+            - g++-4.8
+            - g++-4.8-multilib
     - os: linux
       dist: trusty
       sudo: required

appveyor.yml

@@ -52,6 +52,8 @@ build_script:
       ECHO *** &&
       ECHO make -C contrib\pzstd pzstd &&
       make -C contrib\pzstd pzstd &&
+      make -C contrib\pzstd googletest-mingw64 &&
+      make -C contrib\pzstd test &&
       make -C contrib\pzstd clean
     )
   - if [%COMPILER%]==[gcc] (

contrib/pzstd/Makefile

@@ -30,7 +30,7 @@ else
 EXT =
 endif
 
-.PHONY: default all test clean
+.PHONY: default all test clean test32 googletest googletest32
 
 default: pzstd
 
@@ -41,7 +41,6 @@ libzstd.a: $(ZSTD_FILES)
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	@cp $(ZSTDDIR)/libzstd.a .
 
-
 Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
 	$(CXX) $(FLAGS) -c Pzstd.cpp -o $@
 
@@ -54,23 +53,66 @@ Options.o: Options.h Options.cpp
 main.o: main.cpp *.h utils/*.h
 	$(CXX) $(FLAGS) -c main.cpp -o $@
 
-pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a 
+pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a
 	$(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread
 
+libzstd32.a: $(ZSTD_FILES)
+	$(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32"
+	@cp $(ZSTDDIR)/libzstd.a libzstd32.a
+
+Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
+	$(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@
+
+SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
+	$(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@
+
+Options32.o: Options.h Options.cpp
+	$(CXX) -m32 $(FLAGS) -c Options.cpp -o $@
+
+main32.o: main.cpp *.h utils/*.h
+	$(CXX) -m32 $(FLAGS) -c main.cpp -o $@
+
+pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a
+	$(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread
+
 googletest:
+	@$(RM) -rf googletest
 	@git clone https://github.com/google/googletest
 	@mkdir -p googletest/build
 	@cd googletest/build && cmake .. && make
 
-test: libzstd.a Pzstd.o Options.o SkippableFrame.o
+googletest32:
+	@$(RM) -rf googletest
+	@git clone https://github.com/google/googletest
+	@mkdir -p googletest/build
+	@cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make
+
+googletest-mingw64:
+	$(RM) -rf googletest
+	git clone https://github.com/google/googletest
+	mkdir -p googletest/build
+	cd googletest/build && cmake -G "MSYS Makefiles" .. && $(MAKE)
+
+test:
+	$(MAKE) libzstd.a
+	$(MAKE) pzstd MOREFLAGS="-Wall -Wextra -pedantic -Werror"
 	$(MAKE) -C utils/test clean
-	$(MAKE) -C utils/test test
+	$(MAKE) -C utils/test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
 	$(MAKE) -C test clean
-	$(MAKE) -C test test
+	$(MAKE) -C test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
+
+test32:
+	$(MAKE) libzstd.a MOREFLAGS="-m32"
+	$(MAKE) pzstd MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+	$(MAKE) -C utils/test clean
+	$(MAKE) -C utils/test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+	$(MAKE) -C test clean
+	$(MAKE) -C test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+
 
 clean:
 	$(MAKE) -C $(ZSTDDIR) clean
 	$(MAKE) -C utils/test clean
 	$(MAKE) -C test clean
-	@$(RM) -rf libzstd.a *.o pzstd$(EXT)
+	@$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT)
 	@echo Cleaning completed

contrib/pzstd/Options.cpp

@@ -7,182 +7,419 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 #include "Options.h"
+#include "utils/ScopeGuard.h"
 
+#include <algorithm>
+#include <cassert>
 #include <cstdio>
 #include <cstring>
 #include <thread>
+#include <util.h>
+#include <vector>
+
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) ||     \
+    defined(__CYGWIN__)
+#include <io.h> /* _isatty */
+#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) ||                      \
+    defined(_POSIX_SOURCE) ||                                                  \
+    (defined(__APPLE__) &&                                                     \
+     defined(                                                                  \
+         __MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ \
+                       */
+#include <unistd.h> /* isatty */
+#define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#else
+#define IS_CONSOLE(stdStream) 0
+#endif
+#endif
 
 namespace pzstd {
 
 namespace {
-unsigned parseUnsigned(const char* arg) {
+unsigned defaultNumThreads() {
+#ifdef PZSTD_NUM_THREADS
+  return PZSTD_NUM_THREADS;
+#else
+  return std::thread::hardware_concurrency();
+#endif
+}
+
+unsigned parseUnsigned(const char **arg) {
   unsigned result = 0;
-  while (*arg >= '0' && *arg <= '9') {
+  while (**arg >= '0' && **arg <= '9') {
     result *= 10;
-    result += *arg - '0';
+    result += **arg - '0';
-    ++arg;
+    ++(*arg);
   }
   return result;
 }
 
-const std::string zstdExtension = ".zst";
+const char *getArgument(const char *options, const char **argv, int &i,
-constexpr unsigned defaultCompressionLevel = 3;
+                        int argc) {
-constexpr unsigned maxNonUltraCompressionLevel = 19;
+  if (options[1] != 0) {
+    return options + 1;
+  }
+  ++i;
+  if (i == argc) {
+    std::fprintf(stderr, "Option -%c requires an argument, but none provided\n",
+                 *options);
+    return nullptr;
+  }
+  return argv[i];
+}
+
+const std::string kZstdExtension = ".zst";
+constexpr char kStdIn[] = "-";
+constexpr char kStdOut[] = "-";
+constexpr unsigned kDefaultCompressionLevel = 3;
+constexpr unsigned kMaxNonUltraCompressionLevel = 19;
+
+#ifdef _WIN32
+const char nullOutput[] = "nul";
+#else
+const char nullOutput[] = "/dev/null";
+#endif
+
+void notSupported(const char *option) {
+  std::fprintf(stderr, "Operation not supported: %s\n", option);
+}
 
 void usage() {
   std::fprintf(stderr, "Usage:\n");
-  std::fprintf(stderr, "\tpzstd [args] FILE\n");
+  std::fprintf(stderr, "  pzstd [args] [FILE(s)]\n");
   std::fprintf(stderr, "Parallel ZSTD options:\n");
-  std::fprintf(stderr, "\t-n/--num-threads #: Number of threads to spawn\n");
+  std::fprintf(stderr, "  -p, --processes   #    : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads());
-  std::fprintf(stderr, "\t-p/--pzstd-headers: Write pzstd headers to enable parallel decompression\n");
 
   std::fprintf(stderr, "ZSTD options:\n");
-  std::fprintf(stderr, "\t-u/--ultra        : enable levels beyond %i, up to %i (requires more memory)\n", maxNonUltraCompressionLevel, ZSTD_maxCLevel());
+  std::fprintf(stderr, "  -#                     : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
-  std::fprintf(stderr, "\t-h/--help         : display help and exit\n");
+  std::fprintf(stderr, "  -d, --decompress       : decompression\n");
-  std::fprintf(stderr, "\t-V/--version      : display version number and exit\n");
+  std::fprintf(stderr, "  -o                file : result stored into `file` (only if 1 input file)\n");
-  std::fprintf(stderr, "\t-d/--decompress   : decompression\n");
+  std::fprintf(stderr, "  -f, --force            : overwrite output without prompting\n");
-  std::fprintf(stderr, "\t-f/--force        : overwrite output\n");
+  std::fprintf(stderr, "      --rm               : remove source file(s) after successful (de)compression\n");
-  std::fprintf(stderr, "\t-o/--output file  : result stored into `file`\n");
+  std::fprintf(stderr, "  -k, --keep             : preserve source file(s) (default)\n");
-  std::fprintf(stderr, "\t-c/--stdout       : write output to standard output\n");
+  std::fprintf(stderr, "  -h, --help             : display help and exit\n");
-  std::fprintf(stderr, "\t-#                : # compression level (1-%d, default:%d)\n", maxNonUltraCompressionLevel, defaultCompressionLevel);
+  std::fprintf(stderr, "  -V, --version          : display version number and exit\n");
+  std::fprintf(stderr, "  -v, --verbose          : verbose mode; specify multiple times to increase log level (default:2)\n");
+  std::fprintf(stderr, "  -q, --quiet            : suppress warnings; specify twice to suppress errors too\n");
+  std::fprintf(stderr, "  -c, --stdout           : force wrtie to standard output, even if it is the console\n");
+#ifdef UTIL_HAS_CREATEFILELIST
+  std::fprintf(stderr, "  -r                     : operate recursively on directories\n");
+#endif
+  std::fprintf(stderr, "      --ultra            : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel());
+  std::fprintf(stderr, "  -C, --check            : integrity check (default)\n");
+  std::fprintf(stderr, "      --no-check         : no integrity check\n");
+  std::fprintf(stderr, "  -t, --test             : test compressed file integrity\n");
+  std::fprintf(stderr, "  --                     : all arguments after \"--\" are treated as files\n");
 }
 } // anonymous namespace
 
 Options::Options()
-    : numThreads(0),
+    : numThreads(defaultNumThreads()), maxWindowLog(23),
-      maxWindowLog(23),
+      compressionLevel(kDefaultCompressionLevel), decompress(false),
-      compressionLevel(defaultCompressionLevel),
+      overwrite(false), keepSource(true), writeMode(WriteMode::Auto),
-      decompress(false),
+      checksum(true), verbosity(2) {}
-      overwrite(false),
-      pzstdHeaders(false) {}
 
-bool Options::parse(int argc, const char** argv) {
+Options::Status Options::parse(int argc, const char **argv) {
+  bool test = false;
+  bool recursive = false;
   bool ultra = false;
+  bool forceStdout = false;
+  // Local copy of input files, which are pointers into argv.
+  std::vector<const char *> localInputFiles;
   for (int i = 1; i < argc; ++i) {
-    const char* arg = argv[i];
+    const char *arg = argv[i];
-    // Arguments with a short option
+    // Protect against empty arguments
-    char option = 0;
+    if (arg[0] == 0) {
-    if (!std::strcmp(arg, "--num-threads")) {
-      option = 'n';
-    } else if (!std::strcmp(arg, "--pzstd-headers")) {
-      option = 'p';
-    } else if (!std::strcmp(arg, "--ultra")) {
-      option = 'u';
-    } else if (!std::strcmp(arg, "--version")) {
-      option = 'V';
-    } else if (!std::strcmp(arg, "--help")) {
-      option = 'h';
-    } else if (!std::strcmp(arg, "--decompress")) {
-      option = 'd';
-    } else if (!std::strcmp(arg, "--force")) {
-      option = 'f';
-    } else if (!std::strcmp(arg, "--output")) {
-      option = 'o';
-    }  else if (!std::strcmp(arg, "--stdout")) {
-      option = 'c';
-    }else if (arg[0] == '-' && arg[1] != 0) {
-      // Parse the compression level or short option
-      if (arg[1] >= '0' && arg[1] <= '9') {
-        compressionLevel = parseUnsigned(arg + 1);
-        continue;
-      }
-      option = arg[1];
-    } else if (inputFile.empty()) {
-      inputFile = arg;
       continue;
-    } else {
-      std::fprintf(stderr, "Invalid argument: %s.\n", arg);
-      return false;
     }
-
+    // Everything after "--" is an input file
-    switch (option) {
+    if (!std::strcmp(arg, "--")) {
-      case 'n':
+      ++i;
-        if (++i == argc) {
+      std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles));
-          std::fprintf(stderr, "Invalid argument: -n requires an argument.\n");
+      break;
-          return false;
+    }
-        }
+    // Long arguments that don't have a short option
-        numThreads = parseUnsigned(argv[i]);
+    {
-        if (numThreads == 0) {
+      bool isLongOption = true;
-          std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n");
+      if (!std::strcmp(arg, "--rm")) {
-          return false;
+        keepSource = false;
-        }
+      } else if (!std::strcmp(arg, "--ultra")) {
-        break;
-      case 'p':
-        pzstdHeaders = true;
-        break;
-      case 'u':
         ultra = true;
         maxWindowLog = 0;
-        break;
+      } else if (!std::strcmp(arg, "--no-check")) {
-      case 'V':
+        checksum = false;
-        std::fprintf(stderr, "ZSTD version: %s.\n", ZSTD_VERSION_STRING);
+      } else if (!std::strcmp(arg, "--sparse")) {
-        return false;
+        writeMode = WriteMode::Sparse;
+        notSupported("Sparse mode");
+        return Status::Failure;
+      } else if (!std::strcmp(arg, "--no-sparse")) {
+        writeMode = WriteMode::Regular;
+        notSupported("Sparse mode");
+        return Status::Failure;
+      } else if (!std::strcmp(arg, "--dictID")) {
+        notSupported(arg);
+        return Status::Failure;
+      } else if (!std::strcmp(arg, "--no-dictID")) {
+        notSupported(arg);
+        return Status::Failure;
+      } else {
+        isLongOption = false;
+      }
+      if (isLongOption) {
+        continue;
+      }
+    }
+    // Arguments with a short option simply set their short option.
+    const char *options = nullptr;
+    if (!std::strcmp(arg, "--processes")) {
+      options = "p";
+    } else if (!std::strcmp(arg, "--version")) {
+      options = "V";
+    } else if (!std::strcmp(arg, "--help")) {
+      options = "h";
+    } else if (!std::strcmp(arg, "--decompress")) {
+      options = "d";
+    } else if (!std::strcmp(arg, "--force")) {
+      options = "f";
+    } else if (!std::strcmp(arg, "--stdout")) {
+      options = "c";
+    } else if (!std::strcmp(arg, "--keep")) {
+      options = "k";
+    } else if (!std::strcmp(arg, "--verbose")) {
+      options = "v";
+    } else if (!std::strcmp(arg, "--quiet")) {
+      options = "q";
+    } else if (!std::strcmp(arg, "--check")) {
+      options = "C";
+    } else if (!std::strcmp(arg, "--test")) {
+      options = "t";
+    } else if (arg[0] == '-' && arg[1] != 0) {
+      options = arg + 1;
+    } else {
+      localInputFiles.emplace_back(arg);
+      continue;
+    }
+    assert(options != nullptr);
+
+    bool finished = false;
+    while (!finished && *options != 0) {
+      // Parse the compression level
+      if (*options >= '0' && *options <= '9') {
+        compressionLevel = parseUnsigned(&options);
+        continue;
+      }
+
+      switch (*options) {
       case 'h':
+      case 'H':
         usage();
-        return false;
+        return Status::Message;
+      case 'V':
+        std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING);
+        return Status::Message;
+      case 'p': {
+        finished = true;
+        const char *optionArgument = getArgument(options, argv, i, argc);
+        if (optionArgument == nullptr) {
+          return Status::Failure;
+        }
+        if (*optionArgument < '0' || *optionArgument > '9') {
+          std::fprintf(stderr, "Option -p expects a number, but %s provided\n",
+                       optionArgument);
+          return Status::Failure;
+        }
+        numThreads = parseUnsigned(&optionArgument);
+        if (*optionArgument != 0) {
+          std::fprintf(stderr,
+                       "Option -p expects a number, but %u%s provided\n",
+                       numThreads, optionArgument);
+          return Status::Failure;
+        }
+        break;
+      }
+      case 'o': {
+        finished = true;
+        const char *optionArgument = getArgument(options, argv, i, argc);
+        if (optionArgument == nullptr) {
+          return Status::Failure;
+        }
+        outputFile = optionArgument;
+        break;
+      }
+      case 'C':
+        checksum = true;
+        break;
+      case 'k':
+        keepSource = true;
+        break;
       case 'd':
         decompress = true;
         break;
       case 'f':
         overwrite = true;
+        forceStdout = true;
         break;
-      case 'o':
+      case 't':
-        if (++i == argc) {
+        test = true;
-          std::fprintf(stderr, "Invalid argument: -o requires an argument.\n");
+        decompress = true;
-          return false;
-        }
-        outputFile = argv[i];
         break;
+#ifdef UTIL_HAS_CREATEFILELIST
+      case 'r':
+        recursive = true;
+        break;
+#endif
       case 'c':
-        outputFile = '-';
+        outputFile = kStdOut;
+        forceStdout = true;
         break;
+      case 'v':
+        ++verbosity;
+        break;
+      case 'q':
+        --verbosity;
+        // Ignore them for now
+        break;
+      // Unsupported options from Zstd
+      case 'D':
+      case 's':
+        notSupported("Zstd dictionaries.");
+        return Status::Failure;
+      case 'b':
+      case 'e':
+      case 'i':
+      case 'B':
+        notSupported("Zstd benchmarking options.");
+        return Status::Failure;
       default:
-        std::fprintf(stderr, "Invalid argument: %s.\n", arg);
+        std::fprintf(stderr, "Invalid argument: %s\n", arg);
-        return false;
+        return Status::Failure;
-    }
-  }
-  // Determine input file if not specified
-  if (inputFile.empty()) {
-    inputFile = "-";
-  }
-  // Determine output file if not specified
-  if (outputFile.empty()) {
-    if (inputFile == "-") {
-      outputFile = "-";
-    } else {
-      // Attempt to add/remove zstd extension from the input file
-      if (decompress) {
-        int stemSize = inputFile.size() - zstdExtension.size();
-        if (stemSize > 0 && inputFile.substr(stemSize) == zstdExtension) {
-          outputFile = inputFile.substr(0, stemSize);
-        } else {
-          std::fprintf(
-              stderr, "Invalid argument: Unable to determine output file.\n");
-          return false;
-        }
-      } else {
-        outputFile = inputFile + zstdExtension;
       }
+      if (!finished) {
+        ++options;
+      }
+    } // while (*options != 0);
+  }   // for (int i = 1; i < argc; ++i);
+
+  // Input file defaults to standard input if not provided.
+  if (localInputFiles.empty()) {
+    localInputFiles.emplace_back(kStdIn);
+  }
+
+  // Check validity of input files
+  if (localInputFiles.size() > 1) {
+    const auto it = std::find(localInputFiles.begin(), localInputFiles.end(),
+                              std::string{kStdIn});
+    if (it != localInputFiles.end()) {
+      std::fprintf(
+          stderr,
+          "Cannot specify standard input when handling multiple files\n");
+      return Status::Failure;
     }
   }
+  if (localInputFiles.size() > 1 || recursive) {
+    if (!outputFile.empty() && outputFile != nullOutput) {
+      std::fprintf(
+          stderr,
+          "Cannot specify an output file when handling multiple inputs\n");
+      return Status::Failure;
+    }
+  }
+
+  // Translate input files/directories into files to (de)compress
+  if (recursive) {
+    char *scratchBuffer = nullptr;
+    unsigned numFiles = 0;
+    const char **files =
+        UTIL_createFileList(localInputFiles.data(), localInputFiles.size(),
+                            &scratchBuffer, &numFiles);
+    if (files == nullptr) {
+      std::fprintf(stderr, "Error traversing directories\n");
+      return Status::Failure;
+    }
+    auto guard =
+        makeScopeGuard([&] { UTIL_freeFileList(files, scratchBuffer); });
+    if (numFiles == 0) {
+      std::fprintf(stderr, "No files found\n");
+      return Status::Failure;
+    }
+    inputFiles.resize(numFiles);
+    std::copy(files, files + numFiles, inputFiles.begin());
+  } else {
+    inputFiles.resize(localInputFiles.size());
+    std::copy(localInputFiles.begin(), localInputFiles.end(),
+              inputFiles.begin());
+  }
+  localInputFiles.clear();
+  assert(!inputFiles.empty());
+
+  // If reading from standard input, default to standard output
+  if (inputFiles[0] == kStdIn && outputFile.empty()) {
+    assert(inputFiles.size() == 1);
+    outputFile = "-";
+  }
+
+  if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) {
+    assert(inputFiles.size() == 1);
+    std::fprintf(stderr, "Cannot read input from interactive console\n");
+    return Status::Failure;
+  }
+  if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) {
+    std::fprintf(stderr, "Will not write to console stdout unless -c or -f is "
+                         "specified and decompressing\n");
+    return Status::Failure;
+  }
+
   // Check compression level
   {
-    unsigned maxCLevel = ultra ? ZSTD_maxCLevel() : maxNonUltraCompressionLevel;
+    unsigned maxCLevel =
-    if (compressionLevel > maxCLevel) {
+        ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel;
-      std::fprintf(
+    if (compressionLevel > maxCLevel || compressionLevel == 0) {
-          stderr, "Invalid compression level %u.\n", compressionLevel);
+      std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel);
-      return false;
+      return Status::Failure;
     }
   }
+
   // Check that numThreads is set
   if (numThreads == 0) {
-    numThreads = std::thread::hardware_concurrency();
+    std::fprintf(stderr, "Invalid arguments: # of threads not specified "
-    if (numThreads == 0) {
+                         "and unable to determine hardware concurrency.\n");
-      std::fprintf(stderr, "Invalid arguments: # of threads not specified "
+    return Status::Failure;
-                           "and unable to determine hardware concurrency.\n");
+  }
-      return false;
+
+  // Modify verbosity
+  // If we are piping input and output, turn off interaction
+  if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) {
+    verbosity = 1;
+  }
+  // If we are in multi-file mode, turn off interaction
+  if (inputFiles.size() > 1 && verbosity == 2) {
+    verbosity = 1;
+  }
+
+  // Set options for test mode
+  if (test) {
+    outputFile = nullOutput;
+    keepSource = true;
+  }
+  return Status::Success;
+}
+
+std::string Options::getOutputFile(const std::string &inputFile) const {
+  if (!outputFile.empty()) {
+    return outputFile;
+  }
+  // Attempt to add/remove zstd extension from the input file
+  if (decompress) {
+    int stemSize = inputFile.size() - kZstdExtension.size();
+    if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) {
+      return inputFile.substr(0, stemSize);
+    } else {
+      return "";
     }
+  } else {
+    return inputFile + kZstdExtension;
   }
-  return true;
 }
 }

contrib/pzstd/Options.h

@@ -14,47 +14,55 @@
 
 #include <cstdint>
 #include <string>
+#include <vector>
 
 namespace pzstd {
 
 struct Options {
+  enum class WriteMode { Regular, Auto, Sparse };
+
   unsigned numThreads;
   unsigned maxWindowLog;
   unsigned compressionLevel;
   bool decompress;
-  std::string inputFile;
+  std::vector<std::string> inputFiles;
   std::string outputFile;
   bool overwrite;
-  bool pzstdHeaders;
+  bool keepSource;
+  WriteMode writeMode;
+  bool checksum;
+  int verbosity;
+
+  enum class Status {
+    Success, // Successfully parsed options
+    Failure, // Failure to parse options
+    Message  // Options specified to print a message (e.g. "-h")
+  };
 
   Options();
-  Options(
+  Options(unsigned numThreads, unsigned maxWindowLog, unsigned compressionLevel,
-      unsigned numThreads,
+          bool decompress, std::vector<std::string> inputFiles,
-      unsigned maxWindowLog,
+          std::string outputFile, bool overwrite, bool keepSource,
-      unsigned compressionLevel,
+          WriteMode writeMode, bool checksum, int verbosity)
-      bool decompress,
+      : numThreads(numThreads), maxWindowLog(maxWindowLog),
-      const std::string& inputFile,
+        compressionLevel(compressionLevel), decompress(decompress),
-      const std::string& outputFile,
+        inputFiles(std::move(inputFiles)), outputFile(std::move(outputFile)),
-      bool overwrite,
+        overwrite(overwrite), keepSource(keepSource), writeMode(writeMode),
-      bool pzstdHeaders)
+        checksum(checksum), verbosity(verbosity) {}
-      : numThreads(numThreads),
-        maxWindowLog(maxWindowLog),
-        compressionLevel(compressionLevel),
-        decompress(decompress),
-        inputFile(inputFile),
-        outputFile(outputFile),
-        overwrite(overwrite),
-        pzstdHeaders(pzstdHeaders) {}
 
-  bool parse(int argc, const char** argv);
+  Status parse(int argc, const char **argv);
 
   ZSTD_parameters determineParameters() const {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
+    params.fParams.contentSizeFlag = 1;
+    params.fParams.checksumFlag = checksum;
     if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) {
       params.cParams.windowLog = maxWindowLog;
       params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0);
     }
     return params;
   }
+
+  std::string getOutputFile(const std::string &inputFile) const;
 };
 }

contrib/pzstd/Pzstd.cpp

@@ -19,6 +19,15 @@
 #include <memory>
 #include <string>
 
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>    /* _O_BINARY */
+#  include <io.h>       /* _setmode, _isatty */
+#  define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); }
+#else
+#  include <unistd.h>   /* isatty */
+#  define SET_BINARY_MODE(file)
+#endif
+
 namespace pzstd {
 
 namespace {
@@ -31,43 +40,27 @@ const std::string nullOutput = "/dev/null";
 
 using std::size_t;
 
-size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) {
+static std::uintmax_t fileSizeOrZero(const std::string &file) {
-  // Open the input file and attempt to determine its size
+  if (file == "-") {
-  FILE* inputFd = stdin;
+    return 0;
-  std::uintmax_t inputSize = 0;
-  if (options.inputFile != "-") {
-    inputFd = std::fopen(options.inputFile.c_str(), "rb");
-    if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
-      return 0;
-    }
-    std::error_code ec;
-    inputSize = file_size(options.inputFile, ec);
-    if (ec) {
-      inputSize = 0;
-    }
   }
-  auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); });
+  std::error_code ec;
-
+  auto size = file_size(file, ec);
-  // Check if the output file exists and then open it
+  if (ec) {
-  FILE* outputFd = stdout;
+    size = 0;
-  if (options.outputFile != "-") {
-    if (!options.overwrite && options.outputFile != nullOutput) {
-      outputFd = std::fopen(options.outputFile.c_str(), "rb");
-      if (!errorHolder.check(outputFd == nullptr, "Output file exists")) {
-        return 0;
-      }
-    }
-    outputFd = std::fopen(options.outputFile.c_str(), "wb");
-    if (!errorHolder.check(
-            outputFd != nullptr, "Failed to open output file")) {
-      return 0;
-    }
   }
-  auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); });
+  return size;
+}
 
+static size_t handleOneInput(const Options &options,
+                             const std::string &inputFile,
+                             FILE* inputFd,
+                             FILE* outputFd,
+                             ErrorHolder &errorHolder) {
+  auto inputSize = fileSizeOrZero(inputFile);
   // WorkQueue outlives ThreadPool so in the case of error we are certain
   // we don't accidently try to call push() on it after it is destroyed.
-  WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{2 * options.numThreads};
+  WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{options.numThreads + 1};
   size_t bytesWritten;
   {
     // Initialize the thread pool with numThreads + 1
@@ -89,21 +82,136 @@ size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) {
                 options.determineParameters());
           });
       // Start writing
-      bytesWritten =
+      bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress);
-          writeFile(errorHolder, outs, outputFd, options.pzstdHeaders);
     } else {
       // Add a job that reads the input and starts all the decompression jobs
       executor.add([&errorHolder, &outs, &executor, inputFd] {
         asyncDecompressFrames(errorHolder, outs, executor, inputFd);
       });
       // Start writing
-      bytesWritten = writeFile(
+      bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress);
-          errorHolder, outs, outputFd, /* writeSkippableFrames */ false);
     }
   }
   return bytesWritten;
 }
 
+static FILE *openInputFile(const std::string &inputFile,
+                           ErrorHolder &errorHolder) {
+  if (inputFile == "-") {
+    SET_BINARY_MODE(stdin);
+    return stdin;
+  }
+  // Check if input file is a directory
+  {
+    std::error_code ec;
+    if (is_directory(inputFile, ec)) {
+      errorHolder.setError("Output file is a directory -- ignored");
+      return nullptr;
+    }
+  }
+  auto inputFd = std::fopen(inputFile.c_str(), "rb");
+  if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
+    return nullptr;
+  }
+  return inputFd;
+}
+
+static FILE *openOutputFile(const Options &options,
+                            const std::string &outputFile,
+                            ErrorHolder &errorHolder) {
+  if (outputFile == "-") {
+    SET_BINARY_MODE(stdout);
+    return stdout;
+  }
+  // Check if the output file exists and then open it
+  if (!options.overwrite && outputFile != nullOutput) {
+    auto outputFd = std::fopen(outputFile.c_str(), "rb");
+    if (outputFd != nullptr) {
+      std::fclose(outputFd);
+      if (options.verbosity <= 1) {
+        errorHolder.setError("Output file exists");
+        return nullptr;
+      }
+      std::fprintf(
+          stderr,
+          "pzstd: %s already exists; do you wish to overwrite (y/n) ? ",
+          outputFile.c_str());
+      int c = getchar();
+      if (c != 'y' && c != 'Y') {
+        errorHolder.setError("Not overwritten");
+        return nullptr;
+      }
+    }
+  }
+  auto outputFd = std::fopen(outputFile.c_str(), "wb");
+  if (!errorHolder.check(
+          outputFd != nullptr, "Failed to open output file")) {
+    return 0;
+  }
+  return outputFd;
+}
+
+int pzstdMain(const Options &options) {
+  int returnCode = 0;
+  for (const auto& input : options.inputFiles) {
+    // Setup the error holder
+    ErrorHolder errorHolder;
+    auto printErrorGuard = makeScopeGuard([&] {
+      if (errorHolder.hasError()) {
+        returnCode = 1;
+        if (options.verbosity > 0) {
+          std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(),
+                       errorHolder.getError().c_str());
+        }
+      } else {
+
+      }
+    });
+    // Open the input file
+    auto inputFd = openInputFile(input, errorHolder);
+    if (inputFd == nullptr) {
+      continue;
+    }
+    auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); });
+    // Open the output file
+    auto outputFile = options.getOutputFile(input);
+    if (!errorHolder.check(outputFile != "",
+                           "Input file does not have extension .zst")) {
+      continue;
+    }
+    auto outputFd = openOutputFile(options, outputFile, errorHolder);
+    if (outputFd == nullptr) {
+      continue;
+    }
+    auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); });
+    // (de)compress the file
+    handleOneInput(options, input, inputFd, outputFd, errorHolder);
+    if (errorHolder.hasError()) {
+      continue;
+    }
+    // Delete the input file if necessary
+    if (!options.keepSource) {
+      // Be sure that we are done and have written everything before we delete
+      if (!errorHolder.check(std::fclose(inputFd) == 0,
+                             "Failed to close input file")) {
+        continue;
+      }
+      closeInputGuard.dismiss();
+      if (!errorHolder.check(std::fclose(outputFd) == 0,
+                             "Failed to close output file")) {
+        continue;
+      }
+      closeOutputGuard.dismiss();
+      if (std::remove(input.c_str()) != 0) {
+        errorHolder.setError("Failed to remove input file");
+        continue;
+      }
+    }
+  }
+  // Returns 1 if any of the files failed to (de)compress.
+  return returnCode;
+}
+
 /// Construct a `ZSTD_inBuffer` that points to the data in `buffer`.
 static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) {
   return ZSTD_inBuffer{buffer.data(), buffer.size(), 0};
@@ -224,10 +332,9 @@ static size_t calculateStep(
   size_t step = size_t{1} << (params.cParams.windowLog + 2);
   // If file size is known, see if a smaller step will spread work more evenly
   if (size != 0) {
-    const std::uintmax_t newStep = size / std::uintmax_t{numThreads};
+    const std::uintmax_t newStep = size / numThreads;
-    if (newStep != 0 &&
+    if (newStep != 0 && newStep <= std::numeric_limits<size_t>::max()) {
-        newStep <= std::uintmax_t{std::numeric_limits<size_t>::max()}) {
+      step = std::min(step, static_cast<size_t>(newStep));
-      step = std::min(step, size_t{newStep});
     }
   }
   return step;
@@ -451,12 +558,12 @@ size_t writeFile(
     ErrorHolder& errorHolder,
     WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
     FILE* outputFd,
-    bool writeSkippableFrames) {
+    bool decompress) {
   size_t bytesWritten = 0;
   std::shared_ptr<BufferWorkQueue> out;
   // Grab the output queue for each decompression job (in order).
   while (outs.pop(out) && !errorHolder.hasError()) {
-    if (writeSkippableFrames) {
+    if (!decompress) {
       // If we are compressing and want to write skippable frames we can't
       // start writing before compression is done because we need to know the
       // compressed size.

contrib/pzstd/Pzstd.h

@@ -28,11 +28,9 @@ namespace pzstd {
  * An error occurred if `errorHandler.hasError()`.
  *
  * @param options      The pzstd options to use for (de)compression
- * @param errorHolder  Used to report errors and coordinate early shutdown
+ * @returns            0 upon success and non-zero on failure.
- *                      if an error occured
- * @returns            The number of bytes written.
  */
-std::size_t pzstdMain(const Options& options, ErrorHolder& errorHolder);
+int pzstdMain(const Options& options);
 
 /**
  * Streams input from `fd`, breaks input up into chunks, and compresses each
@@ -79,16 +77,16 @@ void asyncDecompressFrames(
  * Streams input in from each queue in `outs` in order, and writes the data to
  * `outputFd`.
  *
- * @param errorHolder          Used to report errors and coordinate early exit
+ * @param errorHolder  Used to report errors and coordinate early exit
- * @param outs                 A queue of output queues, one for each
+ * @param outs         A queue of output queues, one for each
- *                              (de)compression job.
+ *                      (de)compression job.
- * @param outputFd             The file descriptor to write to
+ * @param outputFd     The file descriptor to write to
- * @param writeSkippableFrames Should we write pzstd headers?
+ * @param decompress   Are we decompressing?
- * @returns                    The number of bytes written
+ * @returns            The number of bytes written
  */
 std::size_t writeFile(
     ErrorHolder& errorHolder,
     WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
     FILE* outputFd,
-    bool writeSkippableFrames);
+    bool decompress);
 }

contrib/pzstd/main.cpp

@@ -19,16 +19,14 @@ using namespace pzstd;
 
 int main(int argc, const char** argv) {
   Options options;
-  if (!options.parse(argc, argv)) {
+  switch (options.parse(argc, argv)) {
+  case Options::Status::Failure:
     return 1;
+  case Options::Status::Message:
+    return 0;
+  default:
+    break;
   }
 
-  ErrorHolder errorHolder;
+  return pzstdMain(options);
-  pzstdMain(options, errorHolder);
-
-  if (errorHolder.hasError()) {
-    std::fprintf(stderr, "Error: %s.\n", errorHolder.getError().c_str());
-    return 1;
-  }
-  return 0;
 }

contrib/pzstd/test/Makefile

@@ -21,19 +21,19 @@ ZSTDDIR = ../../../lib
 # Set GTEST_INC and GTEST_LIB to work with your install of gtest
 GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
 GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
-
+GTEST_FLAGS = $(GTEST_INC) $(GTEST_LIB)
-CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+CPPFLAGS = -I$(PZSTDDIR) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
 
 CXXFLAGS  ?= -O3
-CXXFLAGS  += -std=c++11
+CXXFLAGS  += -std=c++11 -Wno-deprecated-declarations
 CXXFLAGS  += $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
 
 datagen.o: $(PROGDIR)/datagen.*
-	$(CXX) $(FLAGS) $(PROGDIR)/datagen.c -c -o $@
+	$(CC) $(CPPFLAGS) -O3 $(MOREFLAGS) $(LDFLAGS) -Wno-long-long -Wno-variadic-macros $(PROGDIR)/datagen.c -c -o $@
 
 %: %.cpp *.h datagen.o
-	$(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) -lgtest -lgtest_main -lpthread
+	$(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) $(GTEST_FLAGS) -lgtest -lgtest_main -lpthread
 
 .PHONY: test clean
 

contrib/pzstd/test/OptionsTest.cpp

@@ -8,172 +8,535 @@
  */
 #include "Options.h"
 
-#include <gtest/gtest.h>
 #include <array>
+#include <gtest/gtest.h>
 
 using namespace pzstd;
 
 namespace pzstd {
-bool operator==(const Options& lhs, const Options& rhs) {
+bool operator==(const Options &lhs, const Options &rhs) {
   return lhs.numThreads == rhs.numThreads &&
-      lhs.maxWindowLog == rhs.maxWindowLog &&
+         lhs.maxWindowLog == rhs.maxWindowLog &&
-      lhs.compressionLevel == rhs.compressionLevel &&
+         lhs.compressionLevel == rhs.compressionLevel &&
-      lhs.decompress == rhs.decompress && lhs.inputFile == rhs.inputFile &&
+         lhs.decompress == rhs.decompress && lhs.inputFiles == rhs.inputFiles &&
-      lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite &&
+         lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite &&
-      lhs.pzstdHeaders == rhs.pzstdHeaders;
+         lhs.keepSource == rhs.keepSource && lhs.writeMode == rhs.writeMode &&
+         lhs.checksum == rhs.checksum && lhs.verbosity == rhs.verbosity;
 }
+
+std::ostream &operator<<(std::ostream &out, const Options &opt) {
+  out << "{";
+  {
+    out << "\n\t"
+        << "numThreads: " << opt.numThreads;
+    out << ",\n\t"
+        << "maxWindowLog: " << opt.maxWindowLog;
+    out << ",\n\t"
+        << "compressionLevel: " << opt.compressionLevel;
+    out << ",\n\t"
+        << "decompress: " << opt.decompress;
+    out << ",\n\t"
+        << "inputFiles: {";
+    {
+      bool first = true;
+      for (const auto &file : opt.inputFiles) {
+        if (!first) {
+          out << ",";
+        }
+        first = false;
+        out << "\n\t\t" << file;
+      }
+    }
+    out << "\n\t}";
+    out << ",\n\t"
+        << "outputFile: " << opt.outputFile;
+    out << ",\n\t"
+        << "overwrite: " << opt.overwrite;
+    out << ",\n\t"
+        << "keepSource: " << opt.keepSource;
+    out << ",\n\t"
+        << "writeMode: " << static_cast<int>(opt.writeMode);
+    out << ",\n\t"
+        << "checksum: " << opt.checksum;
+    out << ",\n\t"
+        << "verbosity: " << opt.verbosity;
+  }
+  out << "\n}";
+  return out;
+}
+}
+
+namespace {
+#ifdef _WIN32
+const char nullOutput[] = "nul";
+#else
+const char nullOutput[] = "/dev/null";
+#endif
+
+constexpr auto autoMode = Options::WriteMode::Auto;
+} // anonymous namespace
+
+#define EXPECT_SUCCESS(...) EXPECT_EQ(Options::Status::Success, __VA_ARGS__)
+#define EXPECT_FAILURE(...) EXPECT_EQ(Options::Status::Failure, __VA_ARGS__)
+#define EXPECT_MESSAGE(...) EXPECT_EQ(Options::Status::Message, __VA_ARGS__)
+
+template <typename... Args>
+std::array<const char *, sizeof...(Args) + 1> makeArray(Args... args) {
+  return {{nullptr, args...}};
 }
 
 TEST(Options, ValidInputs) {
   {
     Options options;
-    std::array<const char*, 6> args = {
+    auto args = makeArray("--processes", "5", "-o", "x", "y", "-f");
-        {nullptr, "--num-threads", "5", "-o", "-", "-f"}};
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {5,    23,   3,        false, {"y"}, "x",
-    Options expected = {5, 23, 3, false, "-", "-", true, false};
+                        true, true, autoMode, true,  2};
     EXPECT_EQ(expected, options);
   }
   {
     Options options;
-    std::array<const char*, 6> args = {
+    auto args = makeArray("-p", "1", "input", "-19");
-        {nullptr, "-n", "1", "input", "-19", "-p"}};
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {1,     23,   19,       false, {"input"}, "",
-    Options expected = {1, 23, 19, false, "input", "input.zst", false, true};
+                        false, true, autoMode, true,  2};
     EXPECT_EQ(expected, options);
   }
   {
     Options options;
-    std::array<const char*, 10> args = {{nullptr,
+    auto args =
-                                         "--ultra",
+        makeArray("--ultra", "-22", "-p", "1", "-o", "x", "-d", "x.zst", "-f");
-                                         "-22",
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-                                         "-n",
+    Options expected = {1,    0,    22,       true, {"x.zst"}, "x",
-                                         "1",
+                        true, true, autoMode, true, 2};
-                                         "--output",
-                                         "x",
-                                         "-d",
-                                         "x.zst",
-                                         "-f"}};
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
-    Options expected = {1, 0, 22, true, "x.zst", "x", true, false};
     EXPECT_EQ(expected, options);
   }
   {
     Options options;
-    std::array<const char*, 6> args = {{nullptr,
+    auto args = makeArray("--processes", "100", "hello.zst", "--decompress",
-                                        "--num-threads",
+                          "--force");
-                                        "100",
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-                                        "hello.zst",
+    Options expected = {100,  23,       3,    true, {"hello.zst"}, "", true,
-                                        "--decompress",
+                        true, autoMode, true, 2};
-                                        "--force"}};
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
-    Options expected = {100, 23, 3, true, "hello.zst", "hello", true, false};
     EXPECT_EQ(expected, options);
   }
   {
     Options options;
-    std::array<const char*, 5> args = {{nullptr, "-", "-n", "1", "-c"}};
+    auto args = makeArray("x", "-dp", "1", "-c");
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-    Options expected = {1, 23, 3, false, "-", "-", false, false};
+    Options expected = {1,     23,   3,        true, {"x"}, "-",
+                        false, true, autoMode, true, 2};
     EXPECT_EQ(expected, options);
   }
   {
     Options options;
-    std::array<const char*, 5> args = {{nullptr, "-", "-n", "1", "--stdout"}};
+    auto args = makeArray("x", "-dp", "1", "--stdout");
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-    Options expected = {1, 23, 3, false, "-", "-", false, false};
+    Options expected = {1,     23,   3,        true, {"x"}, "-",
+                        false, true, autoMode, true, 2};
     EXPECT_EQ(expected, options);
   }
   {
     Options options;
-    std::array<const char*, 10> args = {{nullptr,
+    auto args = makeArray("-p", "1", "x", "-5", "-fo", "-", "--ultra", "-d");
-                                         "-n",
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-                                         "1",
+    Options expected = {1,    0,    5,        true, {"x"}, "-",
-                                         "-",
+                        true, true, autoMode, true, 2};
-                                         "-5",
+    EXPECT_EQ(expected, options);
-                                         "-o",
-                                         "-",
-                                         "-u",
-                                         "-d",
-                                         "--pzstd-headers"}};
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
-    Options expected = {1, 0, 5, true, "-", "-", false, true};
   }
   {
     Options options;
-    std::array<const char*, 6> args = {
+    auto args = makeArray("silesia.tar", "-o", "silesia.tar.pzstd", "-p", "2");
-        {nullptr, "silesia.tar", "-o", "silesia.tar.pzstd", "-n", "2"}};
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    Options expected = {2,
-    Options expected = {
+                        23,
-        2, 23, 3, false, "silesia.tar", "silesia.tar.pzstd", false, false};
+                        3,
+                        false,
+                        {"silesia.tar"},
+                        "silesia.tar.pzstd",
+                        false,
+                        true,
+                        autoMode,
+                        true,
+                        2};
+    EXPECT_EQ(expected, options);
   }
   {
     Options options;
-    std::array<const char*, 3> args = {{nullptr, "-n", "1"}};
+    auto args = makeArray("x", "-p", "1");
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
   }
   {
     Options options;
-    std::array<const char*, 4> args = {{nullptr, "-", "-n", "1"}};
+    auto args = makeArray("x", "-p", "1");
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, GetOutputFile) {
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("-o-");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+    EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "-o", nullOutput);
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("x.zst", "-do", nullOutput);
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("x.zst", "-d");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("x", options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("xzst", "-d");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("", options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("xzst", "-doxx");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("xx", options.getOutputFile(options.inputFiles[0]));
+  }
+}
+
+TEST(Options, MultipleFiles) {
+  {
+    Options options;
+    auto args = makeArray("x", "y", "z");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected;
+    expected.inputFiles = {"x", "y", "z"};
+    expected.verbosity = 1;
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "z", "-o", nullOutput);
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected;
+    expected.inputFiles = {"x", "y", "z"};
+    expected.outputFile = nullOutput;
+    expected.verbosity = 1;
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "-o-");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "-o", "file");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-qqvd12qp4", "-f", "x", "--", "--rm", "-c");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {4,  23,   12,   true,     {"x", "--rm", "-c"},
+                        "", true, true, autoMode, true,
+                        0};
+    EXPECT_EQ(expected, options);
   }
 }
 
 TEST(Options, NumThreads) {
   {
     Options options;
-    std::array<const char*, 3> args = {{nullptr, "-o", "-"}};
+    auto args = makeArray("x", "-dfo", "-");
-    EXPECT_TRUE(options.parse(args.size(), args.data()));
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
   }
   {
     Options options;
-    std::array<const char*, 5> args = {{nullptr, "-n", "0", "-o", "-"}};
+    auto args = makeArray("x", "-p", "0", "-fo", "-");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
   }
   {
     Options options;
-    std::array<const char*, 4> args = {{nullptr, "-n", "-o", "-"}};
+    auto args = makeArray("-f", "-p", "-o", "-");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
   }
 }
 
 TEST(Options, BadCompressionLevel) {
   {
     Options options;
-    std::array<const char*, 3> args = {{nullptr, "x", "-20"}};
+    auto args = makeArray("x", "-20");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
   }
   {
     Options options;
-    std::array<const char*, 4> args = {{nullptr, "x", "-u", "-23"}};
+    auto args = makeArray("x", "--ultra", "-23");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--1"); // negative 1?
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
   }
 }
 
 TEST(Options, InvalidOption) {
   {
     Options options;
-    std::array<const char*, 3> args = {{nullptr, "x", "-x"}};
+    auto args = makeArray("x", "-x");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
   }
 }
 
 TEST(Options, BadOutputFile) {
   {
     Options options;
-    std::array<const char*, 5> args = {{nullptr, "notzst", "-d", "-n", "1"}};
+    auto args = makeArray("notzst", "-d", "-p", "1");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("", options.getOutputFile(options.inputFiles.front()));
+  }
+}
+
+TEST(Options, BadOptionsWithArguments) {
+  {
+    Options options;
+    auto args = makeArray("x", "-pf");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-p", "10f");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-p");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-o");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-o");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, KeepSource) {
+  {
+    Options options;
+    auto args = makeArray("x", "--rm", "-k");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--rm", "--keep");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+  }
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--rm");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(false, options.keepSource);
+  }
+}
+
+TEST(Options, Verbosity) {
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(2, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("--quiet", "-qq", "x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(-1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("--", "x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("-qv", "x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("-v", "x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(3, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("-v", "x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(3, options.verbosity);
+  }
+}
+
+TEST(Options, TestMode) {
+  {
+    Options options;
+    auto args = makeArray("x", "-t");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+    EXPECT_EQ(true, options.decompress);
+    EXPECT_EQ(nullOutput, options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--test", "--rm", "-ohello");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+    EXPECT_EQ(true, options.decompress);
+    EXPECT_EQ(nullOutput, options.outputFile);
+  }
+}
+
+TEST(Options, Checksum) {
+  {
+    Options options;
+    auto args = makeArray("x.zst", "--no-check", "-Cd");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.checksum);
+  }
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.checksum);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--no-check", "--check");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.checksum);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--no-check");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(false, options.checksum);
+  }
+}
+
+TEST(Options, InputFiles) {
+  {
+    Options options;
+    auto args = makeArray("-cd");
+    options.parse(args.size(), args.data());
+    EXPECT_EQ(1, options.inputFiles.size());
+    EXPECT_EQ("-", options.inputFiles[0]);
+    EXPECT_EQ("-", options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray();
+    options.parse(args.size(), args.data());
+    EXPECT_EQ(1, options.inputFiles.size());
+    EXPECT_EQ("-", options.inputFiles[0]);
+    EXPECT_EQ("-", options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray("-d");
+    options.parse(args.size(), args.data());
+    EXPECT_EQ(1, options.inputFiles.size());
+    EXPECT_EQ("-", options.inputFiles[0]);
+    EXPECT_EQ("-", options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, InvalidOptions) {
+  {
+    Options options;
+    auto args = makeArray("-ibasdf");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("- ");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-n15");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-0", "x");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
   }
 }
 
 TEST(Options, Extras) {
   {
     Options options;
-    std::array<const char*, 2> args = {{nullptr, "-h"}};
+    auto args = makeArray("-h");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
   }
   {
     Options options;
-    std::array<const char*, 2> args = {{nullptr, "-V"}};
+    auto args = makeArray("-H");
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-V");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("--help");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("--version");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
   }
 }

contrib/pzstd/test/PzstdTest.cpp

@@ -6,14 +6,16 @@
  * LICENSE file in the root directory of this source tree. An additional grant
  * of patent rights can be found in the PATENTS file in the same directory.
  */
-#include "datagen.h"
 #include "Pzstd.h"
+extern "C" {
+#include "datagen.h"
+}
 #include "test/RoundTrip.h"
 #include "utils/ScopeGuard.h"
 
-#include <gtest/gtest.h>
 #include <cstddef>
 #include <cstdio>
+#include <gtest/gtest.h>
 #include <memory>
 #include <random>
 
@@ -25,11 +27,14 @@ TEST(Pzstd, SmallSizes) {
   std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed);
   std::mt19937 gen(seed);
 
-  for (unsigned len = 1; len < 1028; ++len) {
+  for (unsigned len = 1; len < 256; ++len) {
+    if (len % 16 == 0) {
+      std::fprintf(stderr, "%u / 16\n", len / 16);
+    }
     std::string inputFile = std::tmpnam(nullptr);
     auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
     {
-      static uint8_t buf[1028];
+      static uint8_t buf[256];
       RDG_genBuffer(buf, len, 0.5, 0.0, gen());
       auto fd = std::fopen(inputFile.c_str(), "wb");
       auto written = std::fwrite(buf, 1, len, fd);
@@ -37,19 +42,16 @@ TEST(Pzstd, SmallSizes) {
       ASSERT_EQ(written, len);
     }
     for (unsigned headers = 0; headers <= 1; ++headers) {
-      for (unsigned numThreads = 1; numThreads <= 4; numThreads *= 2) {
+      for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) {
-        for (unsigned level = 1; level <= 8; level *= 8) {
+        for (unsigned level = 1; level <= 4; level *= 4) {
           auto errorGuard = makeScopeGuard([&] {
-            guard.dismiss();
-            std::fprintf(stderr, "file: %s\n", inputFile.c_str());
             std::fprintf(stderr, "pzstd headers: %u\n", headers);
             std::fprintf(stderr, "# threads: %u\n", numThreads);
             std::fprintf(stderr, "compression level: %u\n", level);
           });
           Options options;
-          options.pzstdHeaders = headers;
           options.overwrite = true;
-          options.inputFile = inputFile;
+          options.inputFiles = {inputFile};
           options.numThreads = numThreads;
           options.compressionLevel = level;
           ASSERT_TRUE(roundTrip(options));
@@ -80,17 +82,14 @@ TEST(Pzstd, LargeSizes) {
       for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
         for (unsigned level = 1; level <= 4; level *= 2) {
           auto errorGuard = makeScopeGuard([&] {
-            guard.dismiss();
-            std::fprintf(stderr, "file: %s\n", inputFile.c_str());
             std::fprintf(stderr, "pzstd headers: %u\n", headers);
             std::fprintf(stderr, "# threads: %u\n", numThreads);
             std::fprintf(stderr, "compression level: %u\n", level);
           });
           Options options;
-          options.pzstdHeaders = headers;
           options.overwrite = true;
-          options.inputFile = inputFile;
+          options.inputFiles = {inputFile};
-          options.numThreads = numThreads;
+          options.numThreads = std::min(numThreads, options.numThreads);
           options.compressionLevel = level;
           ASSERT_TRUE(roundTrip(options));
           errorGuard.dismiss();
@@ -100,6 +99,40 @@ TEST(Pzstd, LargeSizes) {
   }
 }
 
+TEST(Pzstd, ExtremelyLargeSize) {
+  unsigned seed = std::random_device{}();
+  std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed);
+  std::mt19937 gen(seed);
+
+  std::string inputFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+
+  {
+    // Write 4GB + 64 MB
+    constexpr size_t kLength = 1 << 26;
+    std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]);
+    auto fd = std::fopen(inputFile.c_str(), "wb");
+    auto closeGuard = makeScopeGuard([&] { std::fclose(fd); });
+    for (size_t i = 0; i < (1 << 6) + 1; ++i) {
+      RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen());
+      auto written = std::fwrite(buf.get(), 1, kLength, fd);
+      if (written != kLength) {
+        std::fprintf(stderr, "Failed to write file, skipping test\n");
+        return;
+      }
+    }
+  }
+
+  Options options;
+  options.overwrite = true;
+  options.inputFiles = {inputFile};
+  options.compressionLevel = 1;
+  if (options.numThreads == 0) {
+    options.numThreads = 1;
+  }
+  ASSERT_TRUE(roundTrip(options));
+}
+
 TEST(Pzstd, ExtremelyCompressible) {
   std::string inputFile = std::tmpnam(nullptr);
   auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
@@ -112,9 +145,8 @@ TEST(Pzstd, ExtremelyCompressible) {
     ASSERT_EQ(written, 10000);
   }
   Options options;
-  options.pzstdHeaders = false;
   options.overwrite = true;
-  options.inputFile = inputFile;
+  options.inputFiles = {inputFile};
   options.numThreads = 1;
   options.compressionLevel = 1;
   ASSERT_TRUE(roundTrip(options));

contrib/pzstd/test/RoundTrip.h

@@ -55,7 +55,10 @@ inline bool check(std::string source, std::string decompressed) {
 }
 
 inline bool roundTrip(Options& options) {
-  std::string source = options.inputFile;
+  if (options.inputFiles.size() != 1) {
+    return false;
+  }
+  std::string source = options.inputFiles.front();
   std::string compressedFile = std::tmpnam(nullptr);
   std::string decompressedFile = std::tmpnam(nullptr);
   auto guard = makeScopeGuard([&] {
@@ -66,21 +69,15 @@ inline bool roundTrip(Options& options) {
   {
     options.outputFile = compressedFile;
     options.decompress = false;
-    ErrorHolder errorHolder;
+    if (pzstdMain(options) != 0) {
-    pzstdMain(options, errorHolder);
-    if (errorHolder.hasError()) {
-      errorHolder.getError();
       return false;
     }
   }
   {
     options.decompress = true;
-    options.inputFile = compressedFile;
+    options.inputFiles.front() = compressedFile;
     options.outputFile = decompressedFile;
-    ErrorHolder errorHolder;
+    if (pzstdMain(options) != 0) {
-    pzstdMain(options, errorHolder);
-    if (errorHolder.hasError()) {
-      errorHolder.getError();
       return false;
     }
   }

contrib/pzstd/test/RoundTripTest.cpp

@@ -6,7 +6,9 @@
  * LICENSE file in the root directory of this source tree. An additional grant
  * of patent rights can be found in the PATENTS file in the same directory.
  */
+extern "C" {
 #include "datagen.h"
+}
 #include "Options.h"
 #include "test/RoundTrip.h"
 #include "utils/ScopeGuard.h"
@@ -46,14 +48,12 @@ string generateInputFile(Generator& gen) {
 template <typename Generator>
 Options generateOptions(Generator& gen, const string& inputFile) {
   Options options;
-  options.inputFile = inputFile;
+  options.inputFiles = {inputFile};
   options.overwrite = true;
 
-  std::bernoulli_distribution pzstdHeaders{0.75};
   std::uniform_int_distribution<unsigned> numThreads{1, 32};
   std::uniform_int_distribution<unsigned> compressionLevel{1, 10};
 
-  options.pzstdHeaders = pzstdHeaders(gen);
   options.numThreads = numThreads(gen);
   options.compressionLevel = compressionLevel(gen);
 
@@ -61,7 +61,7 @@ Options generateOptions(Generator& gen, const string& inputFile) {
 }
 }
 
-int main(int argc, char** argv) {
+int main() {
   std::mt19937 gen(std::random_device{}());
 
   auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); });
@@ -77,8 +77,6 @@ int main(int argc, char** argv) {
         std::fprintf(stderr, "numThreads: %u\n", options.numThreads);
         std::fprintf(stderr, "level: %u\n", options.compressionLevel);
         std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress);
-        std::fprintf(
-            stderr, "pzstd headers? %u\n", (unsigned)options.pzstdHeaders);
         std::fprintf(stderr, "file: %s\n", inputFile.c_str());
         return 1;
       }

contrib/pzstd/utils/FileSystem.h

@@ -21,10 +21,11 @@
 
 namespace pzstd {
 
+// using file_status = ... causes gcc to emit a false positive warning
 #if defined(_MSC_VER)
-using file_status = struct ::_stat64;
+typedef struct ::_stat64 file_status;
 #else
-using file_status = struct ::stat;
+typedef struct ::stat file_status;
 #endif
 
 /// http://en.cppreference.com/w/cpp/filesystem/status
@@ -59,6 +60,22 @@ inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept {
   return is_regular_file(status(path, ec));
 }
 
+/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+inline bool is_directory(file_status status) noexcept {
+#if defined(S_ISDIR)
+  return S_ISDIR(status.st_mode);
+#elif !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+  return (status.st_mode & S_IFMT) == S_IFDIR;
+#else
+  static_assert(false, "NO POSIX stat() support.");
+#endif
+}
+
+/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+inline bool is_directory(StringPiece path, std::error_code& ec) noexcept {
+  return is_directory(status(path, ec));
+}
+
 /// http://en.cppreference.com/w/cpp/filesystem/file_size
 inline std::uintmax_t file_size(
     StringPiece path,

contrib/pzstd/utils/test/ThreadPoolTest.cpp

@@ -20,12 +20,12 @@ TEST(ThreadPool, Ordering) {
 
   {
     ThreadPool executor(1);
-    for (int i = 0; i < 100; ++i) {
+    for (int i = 0; i < 10; ++i) {
       executor.add([ &results, i ] { results.push_back(i); });
     }
   }
 
-  for (int i = 0; i < 100; ++i) {
+  for (int i = 0; i < 10; ++i) {
     EXPECT_EQ(i, results[i]);
   }
 }
@@ -35,7 +35,7 @@ TEST(ThreadPool, AllJobsFinished) {
   std::atomic<bool> start{false};
   {
     ThreadPool executor(5);
-    for (int i = 0; i < 1000; ++i) {
+    for (int i = 0; i < 10; ++i) {
       executor.add([ &numFinished, &start ] {
         while (!start.load()) {
           // spin
@@ -45,7 +45,7 @@ TEST(ThreadPool, AllJobsFinished) {
     }
     start.store(true);
   }
-  EXPECT_EQ(1000, numFinished.load());
+  EXPECT_EQ(10, numFinished.load());
 }
 
 TEST(ThreadPool, AddJobWhileJoining) {

contrib/pzstd/utils/test/WorkQueueTest.cpp

@@ -89,14 +89,14 @@ TEST(WorkQueue, SPSC) {
 
 TEST(WorkQueue, SPMC) {
   WorkQueue<int> queue;
-  std::vector<int> results(10000, -1);
+  std::vector<int> results(50, -1);
   std::mutex mutex;
   std::vector<std::thread> threads;
-  for (int i = 0; i < 100; ++i) {
+  for (int i = 0; i < 5; ++i) {
     threads.emplace_back(Popper{&queue, results.data(), &mutex});
   }
 
-  for (int i = 0; i < 10000; ++i) {
+  for (int i = 0; i < 50; ++i) {
     queue.push(i);
   }
   queue.finish();
@@ -105,24 +105,24 @@ TEST(WorkQueue, SPMC) {
     thread.join();
   }
 
-  for (int i = 0; i < 10000; ++i) {
+  for (int i = 0; i < 50; ++i) {
     EXPECT_EQ(i, results[i]);
   }
 }
 
 TEST(WorkQueue, MPMC) {
   WorkQueue<int> queue;
-  std::vector<int> results(10000, -1);
+  std::vector<int> results(100, -1);
   std::mutex mutex;
   std::vector<std::thread> popperThreads;
-  for (int i = 0; i < 100; ++i) {
+  for (int i = 0; i < 4; ++i) {
     popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
   }
 
   std::vector<std::thread> pusherThreads;
-  for (int i = 0; i < 10; ++i) {
+  for (int i = 0; i < 2; ++i) {
-    auto min = i * 1000;
+    auto min = i * 50;
-    auto max = (i + 1) * 1000;
+    auto max = (i + 1) * 50;
     pusherThreads.emplace_back(
         [ &queue, min, max ] {
           for (int i = min; i < max; ++i) {
@@ -140,7 +140,7 @@ TEST(WorkQueue, MPMC) {
     thread.join();
   }
 
-  for (int i = 0; i < 10000; ++i) {
+  for (int i = 0; i < 100; ++i) {
     EXPECT_EQ(i, results[i]);
   }
 }
@@ -197,16 +197,16 @@ TEST(WorkQueue, SetMaxSize) {
 }
 
 TEST(WorkQueue, BoundedSizeMPMC) {
-  WorkQueue<int> queue(100);
+  WorkQueue<int> queue(10);
-  std::vector<int> results(10000, -1);
+  std::vector<int> results(200, -1);
   std::mutex mutex;
   std::vector<std::thread> popperThreads;
-  for (int i = 0; i < 10; ++i) {
+  for (int i = 0; i < 4; ++i) {
     popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
   }
 
   std::vector<std::thread> pusherThreads;
-  for (int i = 0; i < 100; ++i) {
+  for (int i = 0; i < 2; ++i) {
     auto min = i * 100;
     auto max = (i + 1) * 100;
     pusherThreads.emplace_back(
@@ -226,7 +226,7 @@ TEST(WorkQueue, BoundedSizeMPMC) {
     thread.join();
   }
 
-  for (int i = 0; i < 10000; ++i) {
+  for (int i = 0; i < 200; ++i) {
     EXPECT_EQ(i, results[i]);
   }
 }

programs/datagen.h

@@ -6,7 +6,8 @@
  * LICENSE file in the root directory of this source tree. An additional grant
  * of patent rights can be found in the PATENTS file in the same directory.
  */
-
+#ifndef DATAGEN_H
+#define DATAGEN_H
 
 #include <stddef.h>   /* size_t */
 
@@ -22,3 +23,5 @@ void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba
    RDG_genStdout
    Same as RDG_genBuffer, but generates data into stdout
 */
+
+#endif

programs/util.h

@@ -31,12 +31,12 @@ extern "C" {
 
 
 /* Unix Large Files support (>4GB) */
-#if !defined(__LP64__)              /* No point defining Large file for 64 bit */
+#if !defined(__LP64__)                                  /* No point defining Large file for 64 bit */
-#   define _FILE_OFFSET_BITS 64     /* turn off_t into a 64-bit type for ftello, fseeko */
+#   define _FILE_OFFSET_BITS 64                         /* turn off_t into a 64-bit type for ftello, fseeko */
-#   if defined(__sun__)             /* Sun Solaris 32-bits requires specific definitions */
+#   if defined(__sun__) && !defined(_LARGEFILE_SOURCE)  /* Sun Solaris 32-bits requires specific definitions */
-#      define _LARGEFILE_SOURCE     /* fseeko, ftello */
+#      define _LARGEFILE_SOURCE                         /* fseeko, ftello */
-#   else
+#   elif !defined(_LARGEFILE64_SOURCE)
-#      define _LARGEFILE64_SOURCE   /* off64_t, fseeko64, ftello64 */
+#      define _LARGEFILE64_SOURCE                       /* off64_t, fseeko64, ftello64 */
 #   endif
 #endif