diff --git a/.gitignore b/.gitignore index 5f2ca97de..751b674e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Object files *.o *.ko +*.dSYM # Libraries *.lib @@ -30,3 +31,4 @@ _zstdbench/ *.idea *.swp .DS_Store +googletest/ diff --git a/.travis.yml b/.travis.yml index 350fc1e6b..d5479a974 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,17 @@ matrix: env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest" + language: cpp + install: + - export CXX="g++-4.8" CC="gcc-4.8" + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - g++-4.8 + env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean" - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" diff --git a/NEWS b/NEWS index 2e300726c..fce31cb2e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +v1.0.1 +New : contrib/pzstd, parallel version of zstd, by Nick Terrell +Fixed : CLI -d output to stdout by default when input is stdin (#322) +Fixed : CLI correctly detects console on Mac OS-X +Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : zstd-pgo, reported by octoploid (#329) + v1.0.0 Change Licensing, all project is now BSD, Copyright Facebook Small decompression speed improvement @@ -29,6 +36,9 @@ Modified : minor compression level adaptations Updated : compression format specification to v0.2.0 changed : zstd.h moved to /lib directory +v0.7.5 +Transition version, supporting decoding of v0.8.x + v0.7.4 Added : homebrew for Mac, by Daniel Cade Added : more examples @@ -179,6 +189,6 @@ frame concatenation support v0.1.1 fix compression bug detects write-flush errors -git@github.com:Cyan4973/zstd.git + v0.1.0 first release diff --git a/contrib/pzstd/.gitignore b/contrib/pzstd/.gitignore new file mode 100644 index 000000000..84e68fb07 --- /dev/null +++ b/contrib/pzstd/.gitignore @@ -0,0 +1,2 @@ +# compilation result +pzstd diff --git a/contrib/pzstd/ErrorHolder.h b/contrib/pzstd/ErrorHolder.h new file mode 100644 index 000000000..188badcad --- /dev/null +++ b/contrib/pzstd/ErrorHolder.h @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include +#include +#include +#include + +namespace pzstd { + +// Coordinates graceful shutdown of the pzstd pipeline +class ErrorHolder { + std::atomic error_; + std::string message_; + + public: + ErrorHolder() : error_(false) {} + + bool hasError() noexcept { + return error_.load(); + } + + void setError(std::string message) noexcept { + // Given multiple possibly concurrent calls, exactly one will ever succeed. + bool expected = false; + if (error_.compare_exchange_strong(expected, true)) { + message_ = std::move(message); + } + } + + bool check(bool predicate, std::string message) noexcept { + if (!predicate) { + setError(std::move(message)); + } + return !hasError(); + } + + std::string getError() noexcept { + error_.store(false); + return std::move(message_); + } + + ~ErrorHolder() { + assert(!hasError()); + } +}; +} diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile new file mode 100644 index 000000000..d71cf5b34 --- /dev/null +++ b/contrib/pzstd/Makefile @@ -0,0 +1,76 @@ +# ########################################################################## +# Copyright (c) 2016-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ########################################################################## + +ZSTDDIR = ../../lib +PROGDIR = ../../programs + +CPPFLAGS = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. +CXXFLAGS ?= -O3 +CXXFLAGS += -std=c++11 +CXXFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) + + +ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c +ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c +ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) + + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + +.PHONY: default all test clean + +default: pzstd + +all: pzstd + + +libzstd.a: $(ZSTD_FILES) + $(MAKE) -C $(ZSTDDIR) libzstd + @cp $(ZSTDDIR)/libzstd.a . + + +Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h + $(CXX) $(FLAGS) -c Pzstd.cpp -o $@ + +SkippableFrame.o: SkippableFrame.h SkippableFrame.cpp utils/*.h + $(CXX) $(FLAGS) -c SkippableFrame.cpp -o $@ + +Options.o: Options.h Options.cpp + $(CXX) $(FLAGS) -c Options.cpp -o $@ + +main.o: main.cpp *.h utils/*.h + $(CXX) $(FLAGS) -c main.cpp -o $@ + +pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a + $(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread + +googletest: + @git clone https://github.com/google/googletest + @mkdir -p googletest/build + @cd googletest/build && cmake .. && make + +test: libzstd.a Pzstd.o Options.o SkippableFrame.o + $(MAKE) -C utils/test clean + $(MAKE) -C utils/test test + $(MAKE) -C test clean + $(MAKE) -C test test + +clean: + $(MAKE) -C $(ZSTDDIR) clean + $(MAKE) -C utils/test clean + $(MAKE) -C test clean + @$(RM) -rf libzstd.a *.o pzstd$(EXT) + @echo Cleaning completed diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp new file mode 100644 index 000000000..616130996 --- /dev/null +++ b/contrib/pzstd/Options.cpp @@ -0,0 +1,181 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "Options.h" + +#include +#include + +namespace pzstd { + +namespace { +unsigned parseUnsigned(const char* arg) { + unsigned result = 0; + while (*arg >= '0' && *arg <= '9') { + result *= 10; + result += *arg - '0'; + ++arg; + } + return result; +} + +const std::string zstdExtension = ".zst"; +constexpr unsigned defaultCompressionLevel = 3; +constexpr unsigned maxNonUltraCompressionLevel = 19; + +void usage() { + std::fprintf(stderr, "Usage:\n"); + std::fprintf(stderr, "\tpzstd [args] FILE\n"); + std::fprintf(stderr, "Parallel ZSTD options:\n"); + std::fprintf(stderr, "\t-n/--num-threads #: Number of threads to spawn\n"); + std::fprintf(stderr, "\t-p/--pzstd-headers: Write pzstd headers to enable parallel decompression\n"); + + std::fprintf(stderr, "ZSTD options:\n"); + std::fprintf(stderr, "\t-u/--ultra : enable levels beyond %i, up to %i (requires more memory)\n", maxNonUltraCompressionLevel, ZSTD_maxCLevel()); + std::fprintf(stderr, "\t-h/--help : display help and exit\n"); + std::fprintf(stderr, "\t-V/--version : display version number and exit\n"); + std::fprintf(stderr, "\t-d/--decompress : decompression\n"); + std::fprintf(stderr, "\t-f/--force : overwrite output\n"); + std::fprintf(stderr, "\t-o/--output file : result stored into `file`\n"); + std::fprintf(stderr, "\t-c/--stdout : write output to standard output\n"); + std::fprintf(stderr, "\t-# : # compression level (1-%d, default:%d)\n", maxNonUltraCompressionLevel, defaultCompressionLevel); +} +} // anonymous namespace + +Options::Options() + : numThreads(0), + maxWindowLog(23), + compressionLevel(defaultCompressionLevel), + decompress(false), + overwrite(false), + pzstdHeaders(false) {} + +bool Options::parse(int argc, const char** argv) { + bool ultra = false; + for (int i = 1; i < argc; ++i) { + const char* arg = argv[i]; + // Arguments with a short option + char option = 0; + if (!std::strcmp(arg, "--num-threads")) { + option = 'n'; + } else if (!std::strcmp(arg, "--pzstd-headers")) { + option = 'p'; + } else if (!std::strcmp(arg, "--ultra")) { + option = 'u'; + } else if (!std::strcmp(arg, "--version")) { + option = 'V'; + } else if (!std::strcmp(arg, "--help")) { + option = 'h'; + } else if (!std::strcmp(arg, "--decompress")) { + option = 'd'; + } else if (!std::strcmp(arg, "--force")) { + option = 'f'; + } else if (!std::strcmp(arg, "--output")) { + option = 'o'; + } else if (!std::strcmp(arg, "--stdout")) { + option = 'c'; + }else if (arg[0] == '-' && arg[1] != 0) { + // Parse the compression level or short option + if (arg[1] >= '0' && arg[1] <= '9') { + compressionLevel = parseUnsigned(arg + 1); + continue; + } + option = arg[1]; + } else if (inputFile.empty()) { + inputFile = arg; + continue; + } else { + std::fprintf(stderr, "Invalid argument: %s.\n", arg); + return false; + } + + switch (option) { + case 'n': + if (++i == argc) { + std::fprintf(stderr, "Invalid argument: -n requires an argument.\n"); + return false; + } + numThreads = parseUnsigned(argv[i]); + if (numThreads == 0) { + std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n"); + } + break; + case 'p': + pzstdHeaders = true; + break; + case 'u': + ultra = true; + maxWindowLog = 0; + break; + case 'V': + std::fprintf(stderr, "ZSTD version: %s.\n", ZSTD_VERSION_STRING); + return false; + case 'h': + usage(); + return false; + case 'd': + decompress = true; + break; + case 'f': + overwrite = true; + break; + case 'o': + if (++i == argc) { + std::fprintf(stderr, "Invalid argument: -o requires an argument.\n"); + return false; + } + outputFile = argv[i]; + break; + case 'c': + outputFile = '-'; + break; + default: + std::fprintf(stderr, "Invalid argument: %s.\n", arg); + return false; + } + } + // Determine input file if not specified + if (inputFile.empty()) { + inputFile = "-"; + } + // Determine output file if not specified + if (outputFile.empty()) { + if (inputFile == "-") { + outputFile = "-"; + } else { + // Attempt to add/remove zstd extension from the input file + if (decompress) { + int stemSize = inputFile.size() - zstdExtension.size(); + if (stemSize > 0 && inputFile.substr(stemSize) == zstdExtension) { + outputFile = inputFile.substr(0, stemSize); + } else { + std::fprintf( + stderr, "Invalid argument: Unable to determine output file.\n"); + return false; + } + } else { + outputFile = inputFile + zstdExtension; + } + } + } + // Check compression level + { + unsigned maxCLevel = ultra ? ZSTD_maxCLevel() : maxNonUltraCompressionLevel; + if (compressionLevel > maxCLevel) { + std::fprintf( + stderr, "Invalid compression level %u.\n", compressionLevel); + } + } + // Check that numThreads is set + if (numThreads == 0) { + std::fprintf(stderr, "Invalid arguments: # of threads not specified.\n"); + return false; + } + return true; +} +} diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h new file mode 100644 index 000000000..47c5f78a6 --- /dev/null +++ b/contrib/pzstd/Options.h @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#undef ZSTD_STATIC_LINKING_ONLY + +#include +#include + +namespace pzstd { + +struct Options { + unsigned numThreads; + unsigned maxWindowLog; + unsigned compressionLevel; + bool decompress; + std::string inputFile; + std::string outputFile; + bool overwrite; + bool pzstdHeaders; + + Options(); + Options( + unsigned numThreads, + unsigned maxWindowLog, + unsigned compressionLevel, + bool decompress, + const std::string& inputFile, + const std::string& outputFile, + bool overwrite, + bool pzstdHeaders) + : numThreads(numThreads), + maxWindowLog(maxWindowLog), + compressionLevel(compressionLevel), + decompress(decompress), + inputFile(inputFile), + outputFile(outputFile), + overwrite(overwrite), + pzstdHeaders(pzstdHeaders) {} + + bool parse(int argc, const char** argv); + + ZSTD_parameters determineParameters() const { + ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0); + if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) { + params.cParams.windowLog = maxWindowLog; + params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0); + } + return params; + } +}; +} diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp new file mode 100644 index 000000000..ddfa59556 --- /dev/null +++ b/contrib/pzstd/Pzstd.cpp @@ -0,0 +1,480 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "Pzstd.h" +#include "SkippableFrame.h" +#include "utils/FileSystem.h" +#include "utils/Range.h" +#include "utils/ScopeGuard.h" +#include "utils/ThreadPool.h" +#include "utils/WorkQueue.h" + +#include +#include +#include +#include + +namespace pzstd { + +namespace { +#ifdef _WIN32 +const std::string nullOutput = "nul"; +#else +const std::string nullOutput = "/dev/null"; +#endif +} + +using std::size_t; + +size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) { + // Open the input file and attempt to determine its size + FILE* inputFd = stdin; + size_t inputSize = 0; + if (options.inputFile != "-") { + inputFd = std::fopen(options.inputFile.c_str(), "rb"); + if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) { + return 0; + } + std::error_code ec; + inputSize = file_size(options.inputFile, ec); + if (ec) { + inputSize = 0; + } + } + auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); }); + + // Check if the output file exists and then open it + FILE* outputFd = stdout; + if (options.outputFile != "-") { + if (!options.overwrite && options.outputFile != nullOutput) { + outputFd = std::fopen(options.outputFile.c_str(), "rb"); + if (!errorHolder.check(outputFd == nullptr, "Output file exists")) { + return 0; + } + } + outputFd = std::fopen(options.outputFile.c_str(), "wb"); + if (!errorHolder.check( + outputFd != nullptr, "Failed to open output file")) { + return 0; + } + } + auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); }); + + // WorkQueue outlives ThreadPool so in the case of error we are certain + // we don't accidently try to call push() on it after it is destroyed. + WorkQueue> outs{2 * options.numThreads}; + size_t bytesWritten; + { + // Initialize the thread pool with numThreads + 1 + // We add one because the read thread spends most of its time waiting. + // This also sets the minimum number of threads to 2, so the algorithm + // doesn't deadlock. + ThreadPool executor(options.numThreads + 1); + if (!options.decompress) { + // Add a job that reads the input and starts all the compression jobs + executor.add( + [&errorHolder, &outs, &executor, inputFd, inputSize, &options] { + asyncCompressChunks( + errorHolder, + outs, + executor, + inputFd, + inputSize, + options.numThreads, + options.determineParameters()); + }); + // Start writing + bytesWritten = + writeFile(errorHolder, outs, outputFd, options.pzstdHeaders); + } else { + // Add a job that reads the input and starts all the decompression jobs + executor.add([&errorHolder, &outs, &executor, inputFd] { + asyncDecompressFrames(errorHolder, outs, executor, inputFd); + }); + // Start writing + bytesWritten = writeFile( + errorHolder, outs, outputFd, /* writeSkippableFrames */ false); + } + } + return bytesWritten; +} + +/// Construct a `ZSTD_inBuffer` that points to the data in `buffer`. +static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) { + return ZSTD_inBuffer{buffer.data(), buffer.size(), 0}; +} + +/** + * Advance `buffer` and `inBuffer` by the amount of data read, as indicated by + * `inBuffer.pos`. + */ +void advance(Buffer& buffer, ZSTD_inBuffer& inBuffer) { + auto pos = inBuffer.pos; + inBuffer.src = static_cast(inBuffer.src) + pos; + inBuffer.size -= pos; + inBuffer.pos = 0; + return buffer.advance(pos); +} + +/// Construct a `ZSTD_outBuffer` that points to the data in `buffer`. +static ZSTD_outBuffer makeZstdOutBuffer(Buffer& buffer) { + return ZSTD_outBuffer{buffer.data(), buffer.size(), 0}; +} + +/** + * Split `buffer` and advance `outBuffer` by the amount of data written, as + * indicated by `outBuffer.pos`. + */ +Buffer split(Buffer& buffer, ZSTD_outBuffer& outBuffer) { + auto pos = outBuffer.pos; + outBuffer.dst = static_cast(outBuffer.dst) + pos; + outBuffer.size -= pos; + outBuffer.pos = 0; + return buffer.splitAt(pos); +} + +/** + * Stream chunks of input from `in`, compress it, and stream it out to `out`. + * + * @param errorHolder Used to report errors and check if an error occured + * @param in Queue that we `pop()` input buffers from + * @param out Queue that we `push()` compressed output buffers to + * @param maxInputSize An upper bound on the size of the input + * @param parameters The zstd parameters to use for compression + */ +static void compress( + ErrorHolder& errorHolder, + std::shared_ptr in, + std::shared_ptr out, + size_t maxInputSize, + ZSTD_parameters parameters) { + auto guard = makeScopeGuard([&] { out->finish(); }); + // Initialize the CCtx + std::unique_ptr ctx( + ZSTD_createCStream(), ZSTD_freeCStream); + if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) { + return; + } + { + auto err = ZSTD_initCStream_advanced(ctx.get(), nullptr, 0, parameters, 0); + if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { + return; + } + } + + // Allocate space for the result + auto outBuffer = Buffer(ZSTD_compressBound(maxInputSize)); + auto zstdOutBuffer = makeZstdOutBuffer(outBuffer); + { + Buffer inBuffer; + // Read a buffer in from the input queue + while (in->pop(inBuffer) && !errorHolder.hasError()) { + auto zstdInBuffer = makeZstdInBuffer(inBuffer); + // Compress the whole buffer and send it to the output queue + while (!inBuffer.empty() && !errorHolder.hasError()) { + if (!errorHolder.check( + !outBuffer.empty(), "ZSTD_compressBound() was too small")) { + return; + } + // Compress + auto err = + ZSTD_compressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer); + if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { + return; + } + // Split the compressed data off outBuffer and pass to the output queue + out->push(split(outBuffer, zstdOutBuffer)); + // Forget about the data we already compressed + advance(inBuffer, zstdInBuffer); + } + } + } + // Write the epilog + size_t bytesLeft; + do { + if (!errorHolder.check( + !outBuffer.empty(), "ZSTD_compressBound() was too small")) { + return; + } + bytesLeft = ZSTD_endStream(ctx.get(), &zstdOutBuffer); + if (!errorHolder.check( + !ZSTD_isError(bytesLeft), ZSTD_getErrorName(bytesLeft))) { + return; + } + out->push(split(outBuffer, zstdOutBuffer)); + } while (bytesLeft != 0 && !errorHolder.hasError()); +} + +/** + * Calculates how large each independently compressed frame should be. + * + * @param size The size of the source if known, 0 otherwise + * @param numThreads The number of threads available to run compression jobs on + * @param params The zstd parameters to be used for compression + */ +static size_t +calculateStep(size_t size, size_t numThreads, const ZSTD_parameters& params) { + size_t step = 1ul << (params.cParams.windowLog + 2); + // If file size is known, see if a smaller step will spread work more evenly + if (size != 0) { + size_t newStep = size / numThreads; + if (newStep != 0) { + step = std::min(step, newStep); + } + } + return step; +} + +namespace { +enum class FileStatus { Continue, Done, Error }; +/// Determines the status of the file descriptor `fd`. +FileStatus fileStatus(FILE* fd) { + if (std::feof(fd)) { + return FileStatus::Done; + } else if (std::ferror(fd)) { + return FileStatus::Error; + } + return FileStatus::Continue; +} +} // anonymous namespace + +/** + * Reads `size` data in chunks of `chunkSize` and puts it into `queue`. + * Will read less if an error or EOF occurs. + * Returns the status of the file after all of the reads have occurred. + */ +static FileStatus +readData(BufferWorkQueue& queue, size_t chunkSize, size_t size, FILE* fd) { + Buffer buffer(size); + while (!buffer.empty()) { + auto bytesRead = + std::fread(buffer.data(), 1, std::min(chunkSize, buffer.size()), fd); + queue.push(buffer.splitAt(bytesRead)); + auto status = fileStatus(fd); + if (status != FileStatus::Continue) { + return status; + } + } + return FileStatus::Continue; +} + +void asyncCompressChunks( + ErrorHolder& errorHolder, + WorkQueue>& chunks, + ThreadPool& executor, + FILE* fd, + size_t size, + size_t numThreads, + ZSTD_parameters params) { + auto chunksGuard = makeScopeGuard([&] { chunks.finish(); }); + + // Break the input up into chunks of size `step` and compress each chunk + // independently. + size_t step = calculateStep(size, numThreads, params); + auto status = FileStatus::Continue; + while (status == FileStatus::Continue && !errorHolder.hasError()) { + // Make a new input queue that we will put the chunk's input data into. + auto in = std::make_shared(); + auto inGuard = makeScopeGuard([&] { in->finish(); }); + // Make a new output queue that compress will put the compressed data into. + auto out = std::make_shared(); + // Start compression in the thread pool + executor.add([&errorHolder, in, out, step, params] { + return compress( + errorHolder, std::move(in), std::move(out), step, params); + }); + // Pass the output queue to the writer thread. + chunks.push(std::move(out)); + // Fill the input queue for the compression job we just started + status = readData(*in, ZSTD_CStreamInSize(), step, fd); + } + errorHolder.check(status != FileStatus::Error, "Error reading input"); +} + +/** + * Decompress a frame, whose data is streamed into `in`, and stream the output + * to `out`. + * + * @param errorHolder Used to report errors and check if an error occured + * @param in Queue that we `pop()` input buffers from. It contains + * exactly one compressed frame. + * @param out Queue that we `push()` decompressed output buffers to + */ +static void decompress( + ErrorHolder& errorHolder, + std::shared_ptr in, + std::shared_ptr out) { + auto guard = makeScopeGuard([&] { out->finish(); }); + // Initialize the DCtx + std::unique_ptr ctx( + ZSTD_createDStream(), ZSTD_freeDStream); + if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) { + return; + } + { + auto err = ZSTD_initDStream(ctx.get()); + if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { + return; + } + } + + const size_t outSize = ZSTD_DStreamOutSize(); + Buffer inBuffer; + size_t returnCode = 0; + // Read a buffer in from the input queue + while (in->pop(inBuffer) && !errorHolder.hasError()) { + auto zstdInBuffer = makeZstdInBuffer(inBuffer); + // Decompress the whole buffer and send it to the output queue + while (!inBuffer.empty() && !errorHolder.hasError()) { + // Allocate a buffer with at least outSize bytes. + Buffer outBuffer(outSize); + auto zstdOutBuffer = makeZstdOutBuffer(outBuffer); + // Decompress + returnCode = + ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer); + if (!errorHolder.check( + !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) { + return; + } + // Pass the buffer with the decompressed data to the output queue + out->push(split(outBuffer, zstdOutBuffer)); + // Advance past the input we already read + advance(inBuffer, zstdInBuffer); + if (returnCode == 0) { + // The frame is over, prepare to (maybe) start a new frame + ZSTD_initDStream(ctx.get()); + } + } + } + if (!errorHolder.check(returnCode <= 1, "Incomplete block")) { + return; + } + // We've given ZSTD_decompressStream all of our data, but there may still + // be data to read. + while (returnCode == 1) { + // Allocate a buffer with at least outSize bytes. + Buffer outBuffer(outSize); + auto zstdOutBuffer = makeZstdOutBuffer(outBuffer); + // Pass in no input. + ZSTD_inBuffer zstdInBuffer{nullptr, 0, 0}; + // Decompress + returnCode = + ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer); + if (!errorHolder.check( + !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) { + return; + } + // Pass the buffer with the decompressed data to the output queue + out->push(split(outBuffer, zstdOutBuffer)); + } +} + +void asyncDecompressFrames( + ErrorHolder& errorHolder, + WorkQueue>& frames, + ThreadPool& executor, + FILE* fd) { + auto framesGuard = makeScopeGuard([&] { frames.finish(); }); + // Split the source up into its component frames. + // If we find our recognized skippable frame we know the next frames size + // which means that we can decompress each standard frame in independently. + // Otherwise, we will decompress using only one decompression task. + const size_t chunkSize = ZSTD_DStreamInSize(); + auto status = FileStatus::Continue; + while (status == FileStatus::Continue && !errorHolder.hasError()) { + // Make a new input queue that we will put the frames's bytes into. + auto in = std::make_shared(); + auto inGuard = makeScopeGuard([&] { in->finish(); }); + // Make a output queue that decompress will put the decompressed data into + auto out = std::make_shared(); + + size_t frameSize; + { + // Calculate the size of the next frame. + // frameSize is 0 if the frame info can't be decoded. + Buffer buffer(SkippableFrame::kSize); + auto bytesRead = std::fread(buffer.data(), 1, buffer.size(), fd); + status = fileStatus(fd); + if (bytesRead == 0 && status != FileStatus::Continue) { + break; + } + buffer.subtract(buffer.size() - bytesRead); + frameSize = SkippableFrame::tryRead(buffer.range()); + in->push(std::move(buffer)); + } + if (frameSize == 0) { + // We hit a non SkippableFrame, so this will be the last job. + // Make sure that we don't use too much memory + in->setMaxSize(64); + out->setMaxSize(64); + } + // Start decompression in the thread pool + executor.add([&errorHolder, in, out] { + return decompress(errorHolder, std::move(in), std::move(out)); + }); + // Pass the output queue to the writer thread + frames.push(std::move(out)); + if (frameSize == 0) { + // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted + // Pass the rest of the source to this decompression task + while (status == FileStatus::Continue && !errorHolder.hasError()) { + status = readData(*in, chunkSize, chunkSize, fd); + } + break; + } + // Fill the input queue for the decompression job we just started + status = readData(*in, chunkSize, frameSize, fd); + } + errorHolder.check(status != FileStatus::Error, "Error reading input"); +} + +/// Write `data` to `fd`, returns true iff success. +static bool writeData(ByteRange data, FILE* fd) { + while (!data.empty()) { + data.advance(std::fwrite(data.begin(), 1, data.size(), fd)); + if (std::ferror(fd)) { + return false; + } + } + return true; +} + +size_t writeFile( + ErrorHolder& errorHolder, + WorkQueue>& outs, + FILE* outputFd, + bool writeSkippableFrames) { + size_t bytesWritten = 0; + std::shared_ptr out; + // Grab the output queue for each decompression job (in order). + while (outs.pop(out) && !errorHolder.hasError()) { + if (writeSkippableFrames) { + // If we are compressing and want to write skippable frames we can't + // start writing before compression is done because we need to know the + // compressed size. + // Wait for the compressed size to be available and write skippable frame + SkippableFrame frame(out->size()); + if (!writeData(frame.data(), outputFd)) { + errorHolder.setError("Failed to write output"); + return bytesWritten; + } + bytesWritten += frame.kSize; + } + // For each chunk of the frame: Pop it from the queue and write it + Buffer buffer; + while (out->pop(buffer) && !errorHolder.hasError()) { + if (!writeData(buffer.range(), outputFd)) { + errorHolder.setError("Failed to write output"); + return bytesWritten; + } + bytesWritten += buffer.size(); + } + } + return bytesWritten; +} +} diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h new file mode 100644 index 000000000..617aecb3f --- /dev/null +++ b/contrib/pzstd/Pzstd.h @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include "ErrorHolder.h" +#include "Options.h" +#include "utils/Buffer.h" +#include "utils/Range.h" +#include "utils/ThreadPool.h" +#include "utils/WorkQueue.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#undef ZSTD_STATIC_LINKING_ONLY + +#include +#include + +namespace pzstd { +/** + * Runs pzstd with `options` and returns the number of bytes written. + * An error occurred if `errorHandler.hasError()`. + * + * @param options The pzstd options to use for (de)compression + * @param errorHolder Used to report errors and coordinate early shutdown + * if an error occured + * @returns The number of bytes written. + */ +std::size_t pzstdMain(const Options& options, ErrorHolder& errorHolder); + +/** + * Streams input from `fd`, breaks input up into chunks, and compresses each + * chunk independently. Output of each chunk gets streamed to a queue, and + * the output queues get put into `chunks` in order. + * + * @param errorHolder Used to report errors and coordinate early shutdown + * @param chunks Each compression jobs output queue gets `pushed()` here + * as soon as it is available + * @param executor The thread pool to run compression jobs in + * @param fd The input file descriptor + * @param size The size of the input file if known, 0 otherwise + * @param numThreads The number of threads in the thread pool + * @param parameters The zstd parameters to use for compression + */ +void asyncCompressChunks( + ErrorHolder& errorHolder, + WorkQueue>& chunks, + ThreadPool& executor, + FILE* fd, + std::size_t size, + std::size_t numThreads, + ZSTD_parameters parameters); + +/** + * Streams input from `fd`. If pzstd headers are available it breaks the input + * up into independent frames. It sends each frame to an independent + * decompression job. Output of each frame gets streamed to a queue, and + * the output queues get put into `frames` in order. + * + * @param errorHolder Used to report errors and coordinate early shutdown + * @param frames Each decompression jobs output queue gets `pushed()` here + * as soon as it is available + * @param executor The thread pool to run compression jobs in + * @param fd The input file descriptor + */ +void asyncDecompressFrames( + ErrorHolder& errorHolder, + WorkQueue>& frames, + ThreadPool& executor, + FILE* fd); + +/** + * Streams input in from each queue in `outs` in order, and writes the data to + * `outputFd`. + * + * @param errorHolder Used to report errors and coordinate early exit + * @param outs A queue of output queues, one for each + * (de)compression job. + * @param outputFd The file descriptor to write to + * @param writeSkippableFrames Should we write pzstd headers? + * @returns The number of bytes written + */ +std::size_t writeFile( + ErrorHolder& errorHolder, + WorkQueue>& outs, + FILE* outputFd, + bool writeSkippableFrames); +} diff --git a/contrib/pzstd/README.md b/contrib/pzstd/README.md new file mode 100644 index 000000000..eba64085a --- /dev/null +++ b/contrib/pzstd/README.md @@ -0,0 +1,48 @@ +# Parallel Zstandard (PZstandard) + +Parallel Zstandard is a Pigz-like tool for Zstandard. +It provides Zstandard format compatible compression and decompression that is able to utilize multiple cores. +It breaks the input up into equal sized chunks and compresses each chunk independently into a Zstandard frame. +It then concatenates the frames together to produce the final compressed output. +Optionally, with the `-p` option, PZstandard will write a 12 byte header for each frame that is a skippable frame in the Zstandard format, which tells PZstandard the size of the next compressed frame. +When `-p` is specified for compression, PZstandard can decompress the output in parallel. + +## Usage + +Basic usage + + pzstd input-file -o output-file -n num-threads [ -p ] -# # Compression + pzstd -d input-file -o output-file -n num-threads # Decompression + +PZstandard also supports piping and fifo pipes + + cat input-file | pzstd -n num-threads [ -p ] -# -c > /dev/null + +For more options + + pzstd --help + +## Benchmarks + +As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia). + +Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads +------------------------------------------|----------------------------------- +![Compression Speed vs Ratio](images/Cspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed.png "Decompression Speed") + +The test procedure was to run each of the following commands 2 times for each compression level, and take the minimum time. + + time pzstd -# -n 4 -p -c silesia.tar > silesia.tar.zst + time pzstd -d -n 4 -c silesia.tar.zst > /dev/null + + time pigz -# -p 4 -k -c silesia.tar > silesia.tar.gz + time pigz -d -p 4 -k -c silesia.tar.gz > /dev/null + +PZstandard was tested using compression levels 1-19, and Pigz was tested using compression levels 1-9. +Pigz cannot do parallel decompression, it simply does each of reading, decompression, and writing on separate threads. + +## Tests + +Tests require that you have [gtest](https://github.com/google/googletest) installed. +Modify `GTEST_INC` and `GTEST_LIB` in `test/Makefile` and `utils/test/Makefile` to work for your install of gtest. +Then run `make test` in the `contrib/pzstd` directory. diff --git a/contrib/pzstd/SkippableFrame.cpp b/contrib/pzstd/SkippableFrame.cpp new file mode 100644 index 000000000..5dc95e5ab --- /dev/null +++ b/contrib/pzstd/SkippableFrame.cpp @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "SkippableFrame.h" +#include "mem.h" +#include "utils/Range.h" + +#include + +using namespace pzstd; + +SkippableFrame::SkippableFrame(std::uint32_t size) : frameSize_(size) { + MEM_writeLE32(data_.data(), kSkippableFrameMagicNumber); + MEM_writeLE32(data_.data() + 4, kFrameContentsSize); + MEM_writeLE32(data_.data() + 8, frameSize_); +} + +/* static */ std::size_t SkippableFrame::tryRead(ByteRange bytes) { + if (bytes.size() < SkippableFrame::kSize || + MEM_readLE32(bytes.begin()) != kSkippableFrameMagicNumber || + MEM_readLE32(bytes.begin() + 4) != kFrameContentsSize) { + return 0; + } + return MEM_readLE32(bytes.begin() + 8); +} diff --git a/contrib/pzstd/SkippableFrame.h b/contrib/pzstd/SkippableFrame.h new file mode 100644 index 000000000..9dc95c1f5 --- /dev/null +++ b/contrib/pzstd/SkippableFrame.h @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include "utils/Range.h" + +#include +#include +#include +#include + +namespace pzstd { +/** + * We put a skippable frame before each frame. + * It contains a skippable frame magic number, the size of the skippable frame, + * and the size of the next frame. + * Each skippable frame is exactly 12 bytes in little endian format. + * The first 8 bytes are for compatibility with the ZSTD format. + * If we have N threads, the output will look like + * + * [0x184D2A50|4|size1] [frame1 of size size1] + * [0x184D2A50|4|size2] [frame2 of size size2] + * ... + * [0x184D2A50|4|sizeN] [frameN of size sizeN] + * + * Each sizeX is 4 bytes. + * + * These skippable frames should allow us to skip through the compressed file + * and only load at most N pages. + */ +class SkippableFrame { + public: + static constexpr std::size_t kSize = 12; + + private: + std::uint32_t frameSize_; + std::array data_; + static constexpr std::uint32_t kSkippableFrameMagicNumber = 0x184D2A50; + // Could be improved if the size fits in less bytes + static constexpr std::uint32_t kFrameContentsSize = kSize - 8; + + public: + // Write the skippable frame to data_ in LE format. + explicit SkippableFrame(std::uint32_t size); + + // Read the skippable frame from bytes in LE format. + static std::size_t tryRead(ByteRange bytes); + + ByteRange data() const { + return {data_.data(), data_.size()}; + } + + // Size of the next frame. + std::size_t frameSize() const { + return frameSize_; + } +}; +} diff --git a/contrib/pzstd/images/Cspeed.png b/contrib/pzstd/images/Cspeed.png new file mode 100644 index 000000000..aca4f663e Binary files /dev/null and b/contrib/pzstd/images/Cspeed.png differ diff --git a/contrib/pzstd/images/Dspeed.png b/contrib/pzstd/images/Dspeed.png new file mode 100644 index 000000000..e48881bcd Binary files /dev/null and b/contrib/pzstd/images/Dspeed.png differ diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp new file mode 100644 index 000000000..7ff2cef74 --- /dev/null +++ b/contrib/pzstd/main.cpp @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "ErrorHolder.h" +#include "Options.h" +#include "Pzstd.h" +#include "utils/FileSystem.h" +#include "utils/Range.h" +#include "utils/ScopeGuard.h" +#include "utils/ThreadPool.h" +#include "utils/WorkQueue.h" + +using namespace pzstd; + +int main(int argc, const char** argv) { + Options options; + if (!options.parse(argc, argv)) { + return 1; + } + + ErrorHolder errorHolder; + pzstdMain(options, errorHolder); + + if (errorHolder.hasError()) { + std::fprintf(stderr, "Error: %s.\n", errorHolder.getError().c_str()); + return 1; + } + return 0; +} diff --git a/contrib/pzstd/test/Makefile b/contrib/pzstd/test/Makefile new file mode 100644 index 000000000..5fd167d18 --- /dev/null +++ b/contrib/pzstd/test/Makefile @@ -0,0 +1,48 @@ +# ########################################################################## +# Copyright (c) 2016-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ########################################################################## + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + +PZSTDDIR = .. +PROGDIR = ../../../programs +ZSTDDIR = ../../../lib + +# Set GTEST_INC and GTEST_LIB to work with your install of gtest +GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include +GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest + +CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. + +CXXFLAGS ?= -O3 +CXXFLAGS += -std=c++11 +CXXFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) + +datagen.o: $(PROGDIR)/datagen.* + $(CXX) $(FLAGS) $(PROGDIR)/datagen.c -c -o $@ + +%: %.cpp *.h datagen.o + $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) -lgtest -lgtest_main -lpthread + +.PHONY: test clean + +test: OptionsTest PzstdTest + @./OptionsTest$(EXT) + @./PzstdTest$(EXT) + +roundtrip: RoundTripTest + @./RoundTripTest$(EXT) + +clean: + @rm -f datagen.o OptionsTest PzstdTest RoundTripTest diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp new file mode 100644 index 000000000..87e79d59e --- /dev/null +++ b/contrib/pzstd/test/OptionsTest.cpp @@ -0,0 +1,179 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "Options.h" + +#include +#include + +using namespace pzstd; + +namespace pzstd { +bool operator==(const Options& lhs, const Options& rhs) { + return lhs.numThreads == rhs.numThreads && + lhs.maxWindowLog == rhs.maxWindowLog && + lhs.compressionLevel == rhs.compressionLevel && + lhs.decompress == rhs.decompress && lhs.inputFile == rhs.inputFile && + lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite && + lhs.pzstdHeaders == rhs.pzstdHeaders; +} +} + +TEST(Options, ValidInputs) { + { + Options options; + std::array args = { + {nullptr, "--num-threads", "5", "-o", "-", "-f"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = {5, 23, 3, false, "-", "-", true, false}; + EXPECT_EQ(expected, options); + } + { + Options options; + std::array args = { + {nullptr, "-n", "1", "input", "-19", "-p"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = {1, 23, 19, false, "input", "input.zst", false, true}; + EXPECT_EQ(expected, options); + } + { + Options options; + std::array args = {{nullptr, + "--ultra", + "-22", + "-n", + "1", + "--output", + "x", + "-d", + "x.zst", + "-f"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = {1, 0, 22, true, "x.zst", "x", true, false}; + EXPECT_EQ(expected, options); + } + { + Options options; + std::array args = {{nullptr, + "--num-threads", + "100", + "hello.zst", + "--decompress", + "--force"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = {100, 23, 3, true, "hello.zst", "hello", true, false}; + EXPECT_EQ(expected, options); + } + { + Options options; + std::array args = {{nullptr, "-", "-n", "1", "-c"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, false, "-", "-", false, false}; + EXPECT_EQ(expected, options); + } + { + Options options; + std::array args = {{nullptr, "-", "-n", "1", "--stdout"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, false, "-", "-", false, false}; + EXPECT_EQ(expected, options); + } + { + Options options; + std::array args = {{nullptr, + "-n", + "1", + "-", + "-5", + "-o", + "-", + "-u", + "-d", + "--pzstd-headers"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = {1, 0, 5, true, "-", "-", false, true}; + } + { + Options options; + std::array args = { + {nullptr, "silesia.tar", "-o", "silesia.tar.pzstd", "-n", "2"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + Options expected = { + 2, 23, 3, false, "silesia.tar", "silesia.tar.pzstd", false, false}; + } + { + Options options; + std::array args = {{nullptr, "-n", "1"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + } + { + Options options; + std::array args = {{nullptr, "-", "-n", "1"}}; + EXPECT_TRUE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, BadNumThreads) { + { + Options options; + std::array args = {{nullptr, "-o", "-"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } + { + Options options; + std::array args = {{nullptr, "-n", "0", "-o", "-"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } + { + Options options; + std::array args = {{nullptr, "-n", "-o", "-"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, BadCompressionLevel) { + { + Options options; + std::array args = {{nullptr, "x", "-20"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } + { + Options options; + std::array args = {{nullptr, "x", "-u", "-23"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, InvalidOption) { + { + Options options; + std::array args = {{nullptr, "x", "-x"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, BadOutputFile) { + { + Options options; + std::array args = {{nullptr, "notzst", "-d", "-n", "1"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, Extras) { + { + Options options; + std::array args = {{nullptr, "-h"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } + { + Options options; + std::array args = {{nullptr, "-V"}}; + EXPECT_FALSE(options.parse(args.size(), args.data())); + } +} diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp new file mode 100644 index 000000000..9d1256fa5 --- /dev/null +++ b/contrib/pzstd/test/PzstdTest.cpp @@ -0,0 +1,121 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "datagen.h" +#include "Pzstd.h" +#include "test/RoundTrip.h" +#include "utils/ScopeGuard.h" + +#include +#include +#include +#include +#include + +using namespace std; +using namespace pzstd; + +TEST(Pzstd, SmallSizes) { + unsigned seed = std::random_device{}(); + std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed); + std::mt19937 gen(seed); + + for (unsigned len = 1; len < 1028; ++len) { + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + { + static uint8_t buf[1028]; + RDG_genBuffer(buf, len, 0.5, 0.0, gen()); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto written = std::fwrite(buf, 1, len, fd); + std::fclose(fd); + ASSERT_EQ(written, len); + } + for (unsigned headers = 0; headers <= 1; ++headers) { + for (unsigned numThreads = 1; numThreads <= 4; numThreads *= 2) { + for (unsigned level = 1; level <= 8; level *= 8) { + auto errorGuard = makeScopeGuard([&] { + guard.dismiss(); + std::fprintf(stderr, "file: %s\n", inputFile.c_str()); + std::fprintf(stderr, "pzstd headers: %u\n", headers); + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.pzstdHeaders = headers; + options.overwrite = true; + options.inputFile = inputFile; + options.numThreads = numThreads; + options.compressionLevel = level; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); + } + } + } + } +} + +TEST(Pzstd, LargeSizes) { + unsigned seed = std::random_device{}(); + std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed); + std::mt19937 gen(seed); + + for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) { + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + { + std::unique_ptr buf(new uint8_t[len]); + RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen()); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto written = std::fwrite(buf.get(), 1, len, fd); + std::fclose(fd); + ASSERT_EQ(written, len); + } + for (unsigned headers = 0; headers <= 1; ++headers) { + for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { + for (unsigned level = 1; level <= 4; level *= 2) { + auto errorGuard = makeScopeGuard([&] { + guard.dismiss(); + std::fprintf(stderr, "file: %s\n", inputFile.c_str()); + std::fprintf(stderr, "pzstd headers: %u\n", headers); + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.pzstdHeaders = headers; + options.overwrite = true; + options.inputFile = inputFile; + options.numThreads = numThreads; + options.compressionLevel = level; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); + } + } + } + } +} + +TEST(Pzstd, ExtremelyCompressible) { + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + { + std::unique_ptr buf(new uint8_t[10000]); + std::memset(buf.get(), 'a', 10000); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto written = std::fwrite(buf.get(), 1, 10000, fd); + std::fclose(fd); + ASSERT_EQ(written, 10000); + } + Options options; + options.pzstdHeaders = false; + options.overwrite = true; + options.inputFile = inputFile; + options.numThreads = 1; + options.compressionLevel = 1; + ASSERT_TRUE(roundTrip(options)); +} diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h new file mode 100644 index 000000000..829c95cac --- /dev/null +++ b/contrib/pzstd/test/RoundTrip.h @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include "Options.h" +#include "Pzstd.h" +#include "utils/ScopeGuard.h" + +#include +#include +#include +#include + +namespace pzstd { + +inline bool check(std::string source, std::string decompressed) { + std::unique_ptr sBuf(new std::uint8_t[1024]); + std::unique_ptr dBuf(new std::uint8_t[1024]); + + auto sFd = std::fopen(source.c_str(), "rb"); + auto dFd = std::fopen(decompressed.c_str(), "rb"); + auto guard = makeScopeGuard([&] { + std::fclose(sFd); + std::fclose(dFd); + }); + + size_t sRead, dRead; + + do { + sRead = std::fread(sBuf.get(), 1, 1024, sFd); + dRead = std::fread(dBuf.get(), 1, 1024, dFd); + if (std::ferror(sFd) || std::ferror(dFd)) { + return false; + } + if (sRead != dRead) { + return false; + } + + for (size_t i = 0; i < sRead; ++i) { + if (sBuf.get()[i] != dBuf.get()[i]) { + return false; + } + } + } while (sRead == 1024); + if (!std::feof(sFd) || !std::feof(dFd)) { + return false; + } + return true; +} + +inline bool roundTrip(Options& options) { + std::string source = options.inputFile; + std::string compressedFile = std::tmpnam(nullptr); + std::string decompressedFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { + std::remove(compressedFile.c_str()); + std::remove(decompressedFile.c_str()); + }); + + { + options.outputFile = compressedFile; + options.decompress = false; + ErrorHolder errorHolder; + pzstdMain(options, errorHolder); + if (errorHolder.hasError()) { + errorHolder.getError(); + return false; + } + } + { + options.decompress = true; + options.inputFile = compressedFile; + options.outputFile = decompressedFile; + ErrorHolder errorHolder; + pzstdMain(options, errorHolder); + if (errorHolder.hasError()) { + errorHolder.getError(); + return false; + } + } + return check(source, decompressedFile); +} +} diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp new file mode 100644 index 000000000..01c1c8113 --- /dev/null +++ b/contrib/pzstd/test/RoundTripTest.cpp @@ -0,0 +1,88 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "datagen.h" +#include "Options.h" +#include "test/RoundTrip.h" +#include "utils/ScopeGuard.h" + +#include +#include +#include +#include +#include + +using namespace std; +using namespace pzstd; + +namespace { +string +writeData(size_t size, double matchProba, double litProba, unsigned seed) { + std::unique_ptr buf(new uint8_t[size]); + RDG_genBuffer(buf.get(), size, matchProba, litProba, seed); + string file = tmpnam(nullptr); + auto fd = std::fopen(file.c_str(), "wb"); + auto guard = makeScopeGuard([&] { std::fclose(fd); }); + auto bytesWritten = std::fwrite(buf.get(), 1, size, fd); + if (bytesWritten != size) { + std::abort(); + } + return file; +} + +template +string generateInputFile(Generator& gen) { + // Use inputs ranging from 1 Byte to 2^16 Bytes + std::uniform_int_distribution size{1, 1 << 16}; + std::uniform_real_distribution<> prob{0, 1}; + return writeData(size(gen), prob(gen), prob(gen), gen()); +} + +template +Options generateOptions(Generator& gen, const string& inputFile) { + Options options; + options.inputFile = inputFile; + options.overwrite = true; + + std::bernoulli_distribution pzstdHeaders{0.75}; + std::uniform_int_distribution numThreads{1, 32}; + std::uniform_int_distribution compressionLevel{1, 10}; + + options.pzstdHeaders = pzstdHeaders(gen); + options.numThreads = numThreads(gen); + options.compressionLevel = compressionLevel(gen); + + return options; +} +} + +int main(int argc, char** argv) { + std::mt19937 gen(std::random_device{}()); + + auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); }); + for (unsigned i = 0; i < 10000; ++i) { + if (i % 100 == 0) { + std::fprintf(stderr, "Progress: %u%%\r", i / 100); + } + auto inputFile = generateInputFile(gen); + auto inputGuard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + for (unsigned i = 0; i < 10; ++i) { + auto options = generateOptions(gen, inputFile); + if (!roundTrip(options)) { + std::fprintf(stderr, "numThreads: %u\n", options.numThreads); + std::fprintf(stderr, "level: %u\n", options.compressionLevel); + std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress); + std::fprintf( + stderr, "pzstd headers? %u\n", (unsigned)options.pzstdHeaders); + std::fprintf(stderr, "file: %s\n", inputFile.c_str()); + return 1; + } + } + } + return 0; +} diff --git a/contrib/pzstd/utils/Buffer.h b/contrib/pzstd/utils/Buffer.h new file mode 100644 index 000000000..ab25bac9c --- /dev/null +++ b/contrib/pzstd/utils/Buffer.h @@ -0,0 +1,99 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include "utils/Range.h" + +#include +#include +#include + +namespace pzstd { + +/** + * A `Buffer` has a pointer to a shared buffer, and a range of the buffer that + * it owns. + * The idea is that you can allocate one buffer, and write chunks into it + * and break off those chunks. + * The underlying buffer is reference counted, and will be destroyed when all + * `Buffer`s that reference it are destroyed. + */ +class Buffer { + std::shared_ptr buffer_; + MutableByteRange range_; + + static void delete_buffer(unsigned char* buffer) { + delete[] buffer; + } + + public: + /// Construct an empty buffer that owns no data. + explicit Buffer() {} + + /// Construct a `Buffer` that owns a new underlying buffer of size `size`. + explicit Buffer(std::size_t size) + : buffer_(new unsigned char[size], delete_buffer), + range_(buffer_.get(), buffer_.get() + size) {} + + explicit Buffer(std::shared_ptr buffer, MutableByteRange data) + : buffer_(buffer), range_(data) {} + + Buffer(Buffer&&) = default; + Buffer& operator=(Buffer&&) & = default; + + /** + * Splits the data into two pieces: [begin, begin + n), [begin + n, end). + * Their data both points into the same underlying buffer. + * Modifies the original `Buffer` to point to only [begin + n, end). + * + * @param n The offset to split at. + * @returns A buffer that owns the data [begin, begin + n). + */ + Buffer splitAt(std::size_t n) { + auto firstPiece = range_.subpiece(0, n); + range_.advance(n); + return Buffer(buffer_, firstPiece); + } + + /// Modifies the buffer to point to the range [begin + n, end). + void advance(std::size_t n) { + range_.advance(n); + } + + /// Modifies the buffer to point to the range [begin, end - n). + void subtract(std::size_t n) { + range_.subtract(n); + } + + /// Returns a read only `Range` pointing to the `Buffer`s data. + ByteRange range() const { + return range_; + } + /// Returns a mutable `Range` pointing to the `Buffer`s data. + MutableByteRange range() { + return range_; + } + + const unsigned char* data() const { + return range_.data(); + } + + unsigned char* data() { + return range_.data(); + } + + std::size_t size() const { + return range_.size(); + } + + bool empty() const { + return range_.empty(); + } +}; +} diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h new file mode 100644 index 000000000..deae0b5b7 --- /dev/null +++ b/contrib/pzstd/utils/FileSystem.h @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include "utils/Range.h" + +#include +#include +#include + +// A small subset of `std::filesystem`. +// `std::filesystem` should be a drop in replacement. +// See http://en.cppreference.com/w/cpp/filesystem for documentation. + +namespace pzstd { + +using file_status = struct stat; + +/// http://en.cppreference.com/w/cpp/filesystem/status +inline file_status status(StringPiece path, std::error_code& ec) noexcept { + file_status status; + if (stat(path.data(), &status)) { + ec.assign(errno, std::generic_category()); + } else { + ec.clear(); + } + return status; +} + +/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file +inline bool is_regular_file(file_status status) noexcept { + return S_ISREG(status.st_mode); +} + +/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file +inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept { + return is_regular_file(status(path, ec)); +} + +/// http://en.cppreference.com/w/cpp/filesystem/file_size +inline std::uintmax_t file_size( + StringPiece path, + std::error_code& ec) noexcept { + auto stat = status(path, ec); + if (ec) { + return -1; + } + if (!is_regular_file(stat)) { + ec.assign(ENOTSUP, std::generic_category()); + return -1; + } + ec.clear(); + return stat.st_size; +} +} diff --git a/contrib/pzstd/utils/Likely.h b/contrib/pzstd/utils/Likely.h new file mode 100644 index 000000000..c8ea102b1 --- /dev/null +++ b/contrib/pzstd/utils/Likely.h @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/** + * Compiler hints to indicate the fast path of an "if" branch: whether + * the if condition is likely to be true or false. + * + * @author Tudor Bosman (tudorb@fb.com) + */ + +#pragma once + +#undef LIKELY +#undef UNLIKELY + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h new file mode 100644 index 000000000..111e98f58 --- /dev/null +++ b/contrib/pzstd/utils/Range.h @@ -0,0 +1,131 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/** + * A subset of `folly/Range.h`. + * All code copied verbatiam modulo formatting + */ +#pragma once + +#include "utils/Likely.h" + +#include +#include +#include +#include +#include + +namespace pzstd { + +namespace detail { +/* + *Use IsCharPointer::type to enable const char* or char*. + *Use IsCharPointer::const_type to enable only const char*. +*/ +template +struct IsCharPointer {}; + +template <> +struct IsCharPointer { + typedef int type; +}; + +template <> +struct IsCharPointer { + typedef int const_type; + typedef int type; +}; + +} // namespace detail + +template +class Range { + Iter b_; + Iter e_; + + public: + using size_type = std::size_t; + using iterator = Iter; + using const_iterator = Iter; + using value_type = typename std::remove_reference< + typename std::iterator_traits::reference>::type; + using reference = typename std::iterator_traits::reference; + + constexpr Range() : b_(), e_() {} + constexpr Range(Iter begin, Iter end) : b_(begin), e_(end) {} + + constexpr Range(Iter begin, size_type size) : b_(begin), e_(begin + size) {} + + template ::type = 0> + /* implicit */ Range(Iter str) : b_(str), e_(str + std::strlen(str)) {} + + template ::const_type = 0> + /* implicit */ Range(const std::string& str) + : b_(str.data()), e_(b_ + str.size()) {} + + // Allow implicit conversion from Range to Range if From is + // implicitly convertible to To. + template < + class OtherIter, + typename std::enable_if< + (!std::is_same::value && + std::is_convertible::value), + int>::type = 0> + constexpr /* implicit */ Range(const Range& other) + : b_(other.begin()), e_(other.end()) {} + + Range(const Range&) = default; + Range(Range&&) = default; + + Range& operator=(const Range&) & = default; + Range& operator=(Range&&) & = default; + + constexpr size_type size() const { + return e_ - b_; + } + bool empty() const { + return b_ == e_; + } + Iter data() const { + return b_; + } + Iter begin() const { + return b_; + } + Iter end() const { + return e_; + } + + void advance(size_type n) { + if (UNLIKELY(n > size())) { + throw std::out_of_range("index out of range"); + } + b_ += n; + } + + void subtract(size_type n) { + if (UNLIKELY(n > size())) { + throw std::out_of_range("index out of range"); + } + e_ -= n; + } + + Range subpiece(size_type first, size_type length = std::string::npos) const { + if (UNLIKELY(first > size())) { + throw std::out_of_range("index out of range"); + } + + return Range(b_ + first, std::min(length, size() - first)); + } +}; + +using ByteRange = Range; +using MutableByteRange = Range; +using StringPiece = Range; +} diff --git a/contrib/pzstd/utils/ScopeGuard.h b/contrib/pzstd/utils/ScopeGuard.h new file mode 100644 index 000000000..5a333e0ab --- /dev/null +++ b/contrib/pzstd/utils/ScopeGuard.h @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include + +namespace pzstd { + +/** + * Dismissable scope guard. + * `Function` must be callable and take no parameters. + * Unless `dissmiss()` is called, the callable is executed upon destruction of + * `ScopeGuard`. + * + * Example: + * + * auto guard = makeScopeGuard([&] { cleanup(); }); + */ +template +class ScopeGuard { + Function function; + bool dismissed; + + public: + explicit ScopeGuard(Function&& function) + : function(std::move(function)), dismissed(false) {} + + void dismiss() { + dismissed = true; + } + + ~ScopeGuard() noexcept { + if (!dismissed) { + function(); + } + } +}; + +/// Creates a scope guard from `function`. +template +ScopeGuard makeScopeGuard(Function&& function) { + return ScopeGuard(std::forward(function)); +} +} diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h new file mode 100644 index 000000000..a1d1fc0b9 --- /dev/null +++ b/contrib/pzstd/utils/ThreadPool.h @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include "utils/WorkQueue.h" + +#include +#include +#include +#include + +namespace pzstd { +/// A simple thread pool that pulls tasks off its queue in FIFO order. +class ThreadPool { + std::vector threads_; + + WorkQueue> tasks_; + + public: + /// Constructs a thread pool with `numThreads` threads. + explicit ThreadPool(std::size_t numThreads) { + threads_.reserve(numThreads); + for (std::size_t i = 0; i < numThreads; ++i) { + threads_.emplace_back([&] { + std::function task; + while (tasks_.pop(task)) { + task(); + } + }); + } + } + + /// Finishes all tasks currently in the queue. + ~ThreadPool() { + tasks_.finish(); + for (auto& thread : threads_) { + thread.join(); + } + } + + /** + * Adds `task` to the queue of tasks to execute. Since `task` is a + * `std::function<>`, it cannot be a move only type. So any lambda passed must + * not capture move only types (like `std::unique_ptr`). + * + * @param task The task to execute. + */ + void add(std::function task) { + tasks_.push(std::move(task)); + } +}; +} diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h new file mode 100644 index 000000000..538213500 --- /dev/null +++ b/contrib/pzstd/utils/WorkQueue.h @@ -0,0 +1,184 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include "utils/Buffer.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pzstd { + +/// Unbounded thread-safe work queue. +template +class WorkQueue { + // Protects all member variable access + std::mutex mutex_; + std::condition_variable readerCv_; + std::condition_variable writerCv_; + + std::queue queue_; + bool done_; + std::size_t maxSize_; + + // Must have lock to call this function + bool full() const { + if (maxSize_ == 0) { + return false; + } + return queue_.size() >= maxSize_; + } + + public: + /** + * Constructs an empty work queue with an optional max size. + * If `maxSize == 0` the queue size is unbounded. + * + * @param maxSize The maximum allowed size of the work queue. + */ + WorkQueue(std::size_t maxSize = 0) : done_(false), maxSize_(maxSize) {} + + /** + * Push an item onto the work queue. Notify a single thread that work is + * available. If `finish()` has been called, do nothing and return false. + * + * @param item Item to push onto the queue. + * @returns True upon success, false if `finish()` has been called. An + * item was pushed iff `push()` returns true. + */ + bool push(T item) { + { + std::unique_lock lock(mutex_); + while (full() && !done_) { + writerCv_.wait(lock); + } + if (done_) { + return false; + } + queue_.push(std::move(item)); + } + readerCv_.notify_one(); + return true; + } + + /** + * Attempts to pop an item off the work queue. It will block until data is + * available or `finish()` has been called. + * + * @param[out] item If `pop` returns `true`, it contains the popped item. + * If `pop` returns `false`, it is unmodified. + * @returns True upon success. False if the queue is empty and + * `finish()` has been called. + */ + bool pop(T& item) { + { + std::unique_lock lock(mutex_); + while (queue_.empty() && !done_) { + readerCv_.wait(lock); + } + if (queue_.empty()) { + assert(done_); + return false; + } + item = std::move(queue_.front()); + queue_.pop(); + } + writerCv_.notify_one(); + return true; + } + + /** + * Sets the maximum queue size. If `maxSize == 0` then it is unbounded. + * + * @param maxSize The new maximum queue size. + */ + void setMaxSize(std::size_t maxSize) { + { + std::lock_guard lock(mutex_); + maxSize_ = maxSize; + } + writerCv_.notify_all(); + } + + /** + * Promise that `push()` won't be called again, so once the queue is empty + * there will never any more work. + */ + void finish() { + { + std::lock_guard lock(mutex_); + assert(!done_); + done_ = true; + } + readerCv_.notify_all(); + writerCv_.notify_all(); + } + + /// Blocks until `finish()` has been called (but the queue may not be empty). + void waitUntilFinished() { + std::unique_lock lock(mutex_); + while (!done_) { + readerCv_.wait(lock); + // If we were woken by a push, we need to wake a thread waiting on pop(). + if (!done_) { + lock.unlock(); + readerCv_.notify_one(); + lock.lock(); + } + } + } +}; + +/// Work queue for `Buffer`s that knows the total number of bytes in the queue. +class BufferWorkQueue { + WorkQueue queue_; + std::atomic size_; + + public: + BufferWorkQueue(std::size_t maxSize = 0) : queue_(maxSize), size_(0) {} + + void push(Buffer buffer) { + size_.fetch_add(buffer.size()); + queue_.push(std::move(buffer)); + } + + bool pop(Buffer& buffer) { + bool result = queue_.pop(buffer); + if (result) { + size_.fetch_sub(buffer.size()); + } + return result; + } + + void setMaxSize(std::size_t maxSize) { + queue_.setMaxSize(maxSize); + } + + void finish() { + queue_.finish(); + } + + /** + * Blocks until `finish()` has been called. + * + * @returns The total number of bytes of all the `Buffer`s currently in the + * queue. + */ + std::size_t size() { + queue_.waitUntilFinished(); + return size_.load(); + } +}; +} diff --git a/contrib/pzstd/utils/test/BufferTest.cpp b/contrib/pzstd/utils/test/BufferTest.cpp new file mode 100644 index 000000000..66ec961e2 --- /dev/null +++ b/contrib/pzstd/utils/test/BufferTest.cpp @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "utils/Buffer.h" +#include "utils/Range.h" + +#include +#include + +using namespace pzstd; + +namespace { +void deleter(const unsigned char* buf) { + delete[] buf; +} +} + +TEST(Buffer, Constructors) { + Buffer empty; + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(0, empty.size()); + + Buffer sized(5); + EXPECT_FALSE(sized.empty()); + EXPECT_EQ(5, sized.size()); + + Buffer moved(std::move(sized)); + EXPECT_FALSE(sized.empty()); + EXPECT_EQ(5, sized.size()); + + Buffer assigned; + assigned = std::move(moved); + EXPECT_FALSE(sized.empty()); + EXPECT_EQ(5, sized.size()); +} + +TEST(Buffer, BufferManagement) { + std::shared_ptr buf(new unsigned char[10], deleter); + { + Buffer acquired(buf, MutableByteRange(buf.get(), buf.get() + 10)); + EXPECT_EQ(2, buf.use_count()); + Buffer moved(std::move(acquired)); + EXPECT_EQ(2, buf.use_count()); + Buffer assigned; + assigned = std::move(moved); + EXPECT_EQ(2, buf.use_count()); + + Buffer split = assigned.splitAt(5); + EXPECT_EQ(3, buf.use_count()); + + split.advance(1); + assigned.subtract(1); + EXPECT_EQ(3, buf.use_count()); + } + EXPECT_EQ(1, buf.use_count()); +} + +TEST(Buffer, Modifiers) { + Buffer buf(10); + { + unsigned char i = 0; + for (auto& byte : buf.range()) { + byte = i++; + } + } + + auto prefix = buf.splitAt(2); + + ASSERT_EQ(2, prefix.size()); + EXPECT_EQ(0, *prefix.data()); + + ASSERT_EQ(8, buf.size()); + EXPECT_EQ(2, *buf.data()); + + buf.advance(2); + EXPECT_EQ(4, *buf.data()); + + EXPECT_EQ(9, *(buf.range().end() - 1)); + + buf.subtract(2); + EXPECT_EQ(7, *(buf.range().end() - 1)); + + EXPECT_EQ(4, buf.size()); +} diff --git a/contrib/pzstd/utils/test/Makefile b/contrib/pzstd/utils/test/Makefile new file mode 100644 index 000000000..b9ea73e32 --- /dev/null +++ b/contrib/pzstd/utils/test/Makefile @@ -0,0 +1,42 @@ +# ########################################################################## +# Copyright (c) 2016-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ########################################################################## + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + +PZSTDDIR = ../.. + +# Set GTEST_INC and GTEST_LIB to work with your install of gtest +GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include +GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest + +CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) +CXXFLAGS ?= -O3 +CXXFLAGS += -std=c++11 +CXXFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) + +%: %.cpp + $(CXX) $(FLAGS) $^ -o $@$(EXT) -lgtest -lgtest_main -lpthread + +.PHONY: test clean + +test: BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest + @./BufferTest$(EXT) + @./RangeTest$(EXT) + @./ScopeGuardTest$(EXT) + @./ThreadPoolTest$(EXT) + @./WorkQueueTest$(EXT) + +clean: + @rm -f BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest diff --git a/contrib/pzstd/utils/test/RangeTest.cpp b/contrib/pzstd/utils/test/RangeTest.cpp new file mode 100644 index 000000000..c761c8aff --- /dev/null +++ b/contrib/pzstd/utils/test/RangeTest.cpp @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "utils/Range.h" + +#include +#include + +using namespace pzstd; + +// Range is directly copied from folly. +// Just some sanity tests to make sure everything seems to work. + +TEST(Range, Constructors) { + StringPiece empty; + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(0, empty.size()); + + std::string str = "hello"; + { + Range piece(str.begin(), str.end()); + EXPECT_EQ(5, piece.size()); + EXPECT_EQ('h', *piece.data()); + EXPECT_EQ('o', *(piece.end() - 1)); + } + + { + StringPiece piece(str.data(), str.size()); + EXPECT_EQ(5, piece.size()); + EXPECT_EQ('h', *piece.data()); + EXPECT_EQ('o', *(piece.end() - 1)); + } + + { + StringPiece piece(str); + EXPECT_EQ(5, piece.size()); + EXPECT_EQ('h', *piece.data()); + EXPECT_EQ('o', *(piece.end() - 1)); + } + + { + StringPiece piece(str.c_str()); + EXPECT_EQ(5, piece.size()); + EXPECT_EQ('h', *piece.data()); + EXPECT_EQ('o', *(piece.end() - 1)); + } +} + +TEST(Range, Modifiers) { + StringPiece range("hello world"); + ASSERT_EQ(11, range.size()); + + { + auto hello = range.subpiece(0, 5); + EXPECT_EQ(5, hello.size()); + EXPECT_EQ('h', *hello.data()); + EXPECT_EQ('o', *(hello.end() - 1)); + } + { + auto hello = range; + hello.subtract(6); + EXPECT_EQ(5, hello.size()); + EXPECT_EQ('h', *hello.data()); + EXPECT_EQ('o', *(hello.end() - 1)); + } + { + auto world = range; + world.advance(6); + EXPECT_EQ(5, world.size()); + EXPECT_EQ('w', *world.data()); + EXPECT_EQ('d', *(world.end() - 1)); + } + + std::string expected = "hello world"; + EXPECT_EQ(expected, std::string(range.begin(), range.end())); + EXPECT_EQ(expected, std::string(range.data(), range.size())); +} diff --git a/contrib/pzstd/utils/test/ScopeGuardTest.cpp b/contrib/pzstd/utils/test/ScopeGuardTest.cpp new file mode 100644 index 000000000..0c4dc0357 --- /dev/null +++ b/contrib/pzstd/utils/test/ScopeGuardTest.cpp @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "utils/ScopeGuard.h" + +#include + +using namespace pzstd; + +TEST(ScopeGuard, Dismiss) { + { + auto guard = makeScopeGuard([&] { EXPECT_TRUE(false); }); + guard.dismiss(); + } +} + +TEST(ScopeGuard, Executes) { + bool executed = false; + { + auto guard = makeScopeGuard([&] { executed = true; }); + } + EXPECT_TRUE(executed); +} diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp new file mode 100644 index 000000000..9b9868cb1 --- /dev/null +++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "utils/ThreadPool.h" + +#include +#include +#include +#include + +using namespace pzstd; + +TEST(ThreadPool, Ordering) { + std::vector results; + + { + ThreadPool executor(1); + for (int i = 0; i < 100; ++i) { + executor.add([ &results, i ] { results.push_back(i); }); + } + } + + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(i, results[i]); + } +} + +TEST(ThreadPool, AllJobsFinished) { + std::atomic numFinished{0}; + std::atomic start{false}; + { + ThreadPool executor(5); + for (int i = 0; i < 1000; ++i) { + executor.add([ &numFinished, &start ] { + while (!start.load()) { + // spin + } + ++numFinished; + }); + } + start.store(true); + } + EXPECT_EQ(1000, numFinished.load()); +} + +TEST(ThreadPool, AddJobWhileJoining) { + std::atomic done{false}; + { + ThreadPool executor(1); + executor.add([&executor, &done] { + while (!done.load()) { + std::this_thread::yield(); + } + // Sleep for a second to be sure that we are joining + std::this_thread::sleep_for(std::chrono::seconds(1)); + executor.add([] { + EXPECT_TRUE(false); + }); + }); + done.store(true); + } +} diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp new file mode 100644 index 000000000..84d8573c3 --- /dev/null +++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp @@ -0,0 +1,262 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "utils/Buffer.h" +#include "utils/WorkQueue.h" + +#include +#include +#include +#include + +using namespace pzstd; + +namespace { +struct Popper { + WorkQueue* queue; + int* results; + std::mutex* mutex; + + void operator()() { + int result; + while (queue->pop(result)) { + std::lock_guard lock(*mutex); + results[result] = result; + } + } +}; +} + +TEST(WorkQueue, SingleThreaded) { + WorkQueue queue; + int result; + + queue.push(5); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(5, result); + + queue.push(1); + queue.push(2); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(1, result); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(2, result); + + queue.push(1); + queue.push(2); + queue.finish(); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(1, result); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(2, result); + EXPECT_FALSE(queue.pop(result)); + + queue.waitUntilFinished(); +} + +TEST(WorkQueue, SPSC) { + WorkQueue queue; + const int max = 100; + + for (int i = 0; i < 10; ++i) { + queue.push(i); + } + + std::thread thread([ &queue, max ] { + int result; + for (int i = 0;; ++i) { + if (!queue.pop(result)) { + EXPECT_EQ(i, max); + break; + } + EXPECT_EQ(i, result); + } + }); + + std::this_thread::yield(); + for (int i = 10; i < max; ++i) { + queue.push(i); + } + queue.finish(); + + thread.join(); +} + +TEST(WorkQueue, SPMC) { + WorkQueue queue; + std::vector results(10000, -1); + std::mutex mutex; + std::vector threads; + for (int i = 0; i < 100; ++i) { + threads.emplace_back(Popper{&queue, results.data(), &mutex}); + } + + for (int i = 0; i < 10000; ++i) { + queue.push(i); + } + queue.finish(); + + for (auto& thread : threads) { + thread.join(); + } + + for (int i = 0; i < 10000; ++i) { + EXPECT_EQ(i, results[i]); + } +} + +TEST(WorkQueue, MPMC) { + WorkQueue queue; + std::vector results(10000, -1); + std::mutex mutex; + std::vector popperThreads; + for (int i = 0; i < 100; ++i) { + popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); + } + + std::vector pusherThreads; + for (int i = 0; i < 10; ++i) { + auto min = i * 1000; + auto max = (i + 1) * 1000; + pusherThreads.emplace_back( + [ &queue, min, max ] { + for (int i = min; i < max; ++i) { + queue.push(i); + } + }); + } + + for (auto& thread : pusherThreads) { + thread.join(); + } + queue.finish(); + + for (auto& thread : popperThreads) { + thread.join(); + } + + for (int i = 0; i < 10000; ++i) { + EXPECT_EQ(i, results[i]); + } +} + +TEST(WorkQueue, BoundedSizeWorks) { + WorkQueue queue(1); + int result; + queue.push(5); + queue.pop(result); + queue.push(5); + queue.pop(result); + queue.push(5); + queue.finish(); + queue.pop(result); + EXPECT_EQ(5, result); +} + +TEST(WorkQueue, BoundedSizePushAfterFinish) { + WorkQueue queue(1); + int result; + queue.push(5); + std::thread pusher([&queue] { + queue.push(6); + }); + // Dirtily try and make sure that pusher has run. + std::this_thread::sleep_for(std::chrono::seconds(1)); + queue.finish(); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(5, result); + EXPECT_FALSE(queue.pop(result)); + + pusher.join(); +} + +TEST(WorkQueue, SetMaxSize) { + WorkQueue queue(2); + int result; + queue.push(5); + queue.push(6); + queue.setMaxSize(1); + std::thread pusher([&queue] { + queue.push(7); + }); + // Dirtily try and make sure that pusher has run. + std::this_thread::sleep_for(std::chrono::seconds(1)); + queue.finish(); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(5, result); + EXPECT_TRUE(queue.pop(result)); + EXPECT_EQ(6, result); + EXPECT_FALSE(queue.pop(result)); + + pusher.join(); +} + +TEST(WorkQueue, BoundedSizeMPMC) { + WorkQueue queue(100); + std::vector results(10000, -1); + std::mutex mutex; + std::vector popperThreads; + for (int i = 0; i < 10; ++i) { + popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); + } + + std::vector pusherThreads; + for (int i = 0; i < 100; ++i) { + auto min = i * 100; + auto max = (i + 1) * 100; + pusherThreads.emplace_back( + [ &queue, min, max ] { + for (int i = min; i < max; ++i) { + queue.push(i); + } + }); + } + + for (auto& thread : pusherThreads) { + thread.join(); + } + queue.finish(); + + for (auto& thread : popperThreads) { + thread.join(); + } + + for (int i = 0; i < 10000; ++i) { + EXPECT_EQ(i, results[i]); + } +} + +TEST(BufferWorkQueue, SizeCalculatedCorrectly) { + { + BufferWorkQueue queue; + queue.finish(); + EXPECT_EQ(0, queue.size()); + } + { + BufferWorkQueue queue; + queue.push(Buffer(10)); + queue.finish(); + EXPECT_EQ(10, queue.size()); + } + { + BufferWorkQueue queue; + queue.push(Buffer(10)); + queue.push(Buffer(5)); + queue.finish(); + EXPECT_EQ(15, queue.size()); + } + { + BufferWorkQueue queue; + queue.push(Buffer(10)); + queue.push(Buffer(5)); + queue.finish(); + Buffer buffer; + queue.pop(buffer); + EXPECT_EQ(5, queue.size()); + } +} diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index 032e65771..7492a3832 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -42,12 +42,15 @@ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c index f462c57f8..29e4fa628 100644 --- a/lib/common/xxhash.c +++ b/lib/common/xxhash.c @@ -115,7 +115,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # define FORCE_INLINE static __forceinline #else -# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # ifdef __GNUC__ # define FORCE_INLINE static inline __attribute__((always_inline)) # else diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 386b2c010..679dbdb83 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -41,12 +41,15 @@ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index c2dd13c87..b7d3d77a2 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -35,24 +35,8 @@ /* ************************************************************** * Compiler specifics ****************************************************************/ -#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -/* inline is defined */ -#elif defined(_MSC_VER) -# define inline __inline -#else -# define inline /* disable inline */ -#endif - - #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0116136c0..9e733b8f2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -8,7 +8,6 @@ */ - /*-******************************************************* * Compiler specifics *********************************************************/ @@ -17,11 +16,15 @@ # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index a5521bd36..e94fa83cc 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -43,16 +43,8 @@ # define inline /* disable inline */ #endif - #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index fb1ee35a0..762e972ce 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -67,11 +67,15 @@ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ # pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ #else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index adfe55cf7..cfabb20ba 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -71,26 +71,18 @@ static const size_t g_min_fast_dictContent = 192; * Console display ***************************************/ #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if (ZDICT_clockSpan(g_time) > refreshRate) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC * 3 / 10; -static clock_t g_time = 0; +#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } -static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length) +static void ZDICT_printHex(const void* ptr, size_t length) { const BYTE* const b = (const BYTE*)ptr; size_t u; for (u=0; u126) c = '.'; /* non-printable char */ - DISPLAYLEVEL(dlevel, "%c", c); + DISPLAY("%c", c); } } @@ -211,7 +203,7 @@ static void ZDICT_initDictItem(dictItem* d) static dictItem ZDICT_analyzePos( BYTE* doneMarks, const int* suffix, U32 start, - const void* buffer, U32 minRatio) + const void* buffer, U32 minRatio, U32 notificationLevel) { U32 lengthList[LLIMIT] = {0}; U32 cumulLength[LLIMIT] = {0}; @@ -473,7 +465,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList) static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */ const size_t* fileSizes, unsigned nbFiles, - U32 minRatio) + U32 minRatio, U32 notificationLevel) { int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0)); int* const suffix = suffix0+1; @@ -481,6 +473,13 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks)); /* +16 for overflow security */ U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos)); size_t result = 0; + clock_t displayClock = 0; + clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10; + +# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ + if (ZDICT_clockSpan(displayClock) > refreshRate) \ + { displayClock = clock(); DISPLAY(__VA_ARGS__); \ + if (notificationLevel>=4) fflush(stdout); } } /* init */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ @@ -518,7 +517,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, { U32 cursor; for (cursor=0; cursor < bufferSize; ) { dictItem solution; if (doneMarks[cursor]) { cursor++; continue; } - solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio); + solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel); if (solution.length==0) { cursor++; continue; } ZDICT_insertDictItem(dictList, dictListSize, solution); cursor += solution.length; @@ -558,15 +557,15 @@ typedef struct static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, - const void* src, size_t srcSize) + const void* src, size_t srcSize, U32 notificationLevel) { size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog); size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); - if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } - } + { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } + } cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize); if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; } @@ -647,9 +646,10 @@ static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val, #define OFFCODE_MAX 30 /* only applicable to first block */ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, - unsigned compressionLevel, - const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, - const void* dictBuffer, size_t dictBufferSize) + unsigned compressionLevel, + const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize, + unsigned notificationLevel) { U32 countLit[256]; HUF_CREATE_STATIC_CTABLE(hufTable, 255); @@ -690,18 +690,19 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, memset(bestRepOffset, 0, sizeof(bestRepOffset)); if (compressionLevel==0) compressionLevel=g_compressionLevel_default; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); - { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); - if (ZSTD_isError(beginResult)) { - eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); - goto _cleanup; - } } + { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); + if (ZSTD_isError(beginResult)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); + goto _cleanup; + } } /* collect stats on all files */ for (u=0; u= 3) { + if (params.notificationLevel>= 3) { U32 const nb = MIN(25, dictList[0].pos); U32 const dictContentSize = ZDICT_dictSize(dictList); U32 u; @@ -901,7 +903,7 @@ size_t ZDICT_trainFromBuffer_unsafe( U32 printedLength = MIN(40, length); DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |", u, length, pos, dictList[u].savings); - ZDICT_printHex(3, (const char*)samplesBuffer+pos, printedLength); + ZDICT_printHex((const char*)samplesBuffer+pos, printedLength); DISPLAYLEVEL(3, "| \n"); } } diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index c84aedd1f..642a43516 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -68,7 +68,7 @@ typedef struct { int compressionLevel; /* 0 means default; target a specific zstd compression level */ unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ - unsigned reserved[2]; /* space for future parameters */ + unsigned reserved[2]; /* reserved space for future parameters */ } ZDICT_params_t; diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 94847d5c8..1181cf6e9 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -139,11 +139,15 @@ typedef struct # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif @@ -1300,17 +1304,9 @@ typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes; /* expo #endif #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ -#else -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index af1366208..b306db9a2 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -1122,12 +1122,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx; # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif @@ -1595,15 +1598,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif @@ -2790,17 +2785,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_ #endif #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ -#else -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 637204deb..64cfdf49a 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -1122,12 +1122,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx; # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif @@ -1588,25 +1591,13 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) /* inline is defined */ #elif defined(_MSC_VER) +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # define inline __inline #else # define inline /* disable inline */ #endif -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - /**************************************************************** * Includes ****************************************************************/ @@ -2432,17 +2423,11 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_ #endif #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ #else # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 1239bdb3a..b2c3ab00f 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1380,12 +1380,15 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif @@ -2021,15 +2024,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif @@ -2837,17 +2832,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_ * Compiler specifics *********************************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ -#else -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 96fdf35e9..8434f613c 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1380,12 +1380,15 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr) # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif @@ -2002,14 +2005,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif @@ -2858,17 +2854,9 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS * Compiler specifics *********************************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ -#else -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 6bd463b6e..882361484 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -1665,12 +1665,15 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif @@ -2238,14 +2241,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif @@ -3054,16 +3050,9 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er * Compiler specifics *********************************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index f948069e0..b634d6243 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -1680,17 +1680,18 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ #else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif - - /* ************************************************************** * Error Management ****************************************************************/ @@ -2006,14 +2007,7 @@ size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif @@ -3176,17 +3170,10 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction * Compiler specifics *********************************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ # pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in index 28afc3add..9399363dd 100644 --- a/lib/libzstd.pc.in +++ b/lib/libzstd.pc.in @@ -8,7 +8,7 @@ includedir=@INCLUDEDIR@ Name: zstd Description: lossless compression algorithm library -URL: https://github.com/Cyan4973/zstd +URL: https://github.com/facebook/zstd Version: @VERSION@ Libs: -L@LIBDIR@ -lzstd Cflags: -I@INCLUDEDIR@ diff --git a/lib/zstd.h b/lib/zstd.h index d768ded35..f6c5564dd 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -33,7 +33,7 @@ extern "C" { /*======= Version =======*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 0 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str diff --git a/programs/.gitignore b/programs/.gitignore index 5875fc792..24f96cf4b 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -7,6 +7,7 @@ zstd-decompress # Object files *.o *.ko +default.profraw # Executables *.exe diff --git a/programs/Makefile b/programs/Makefile index fc634b60e..280c2727d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -97,6 +97,7 @@ zstd-pgo : clean zstd ./zstd -b7i2 $(PROFILE_WITH) ./zstd -b5 $(PROFILE_WITH) $(RM) zstd + $(RM) $(ZSTDDIR)/decompress/zstd_decompress.o $(MAKE) zstd MOREFLAGS=-fprofile-use zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c @@ -116,9 +117,10 @@ zstd-small: clean clean: $(MAKE) -C ../lib clean - @$(RM) ../lib/decompress/*.o + @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ - zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) + zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ + *.gcda default.profraw @echo Cleaning completed diff --git a/programs/zstd.1 b/programs/zstd.1 index 0a0ad629d..a23616529 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -135,7 +135,7 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB). .SH BUGS -Report bugs at:- https://github.com/Cyan4973/zstd/issues +Report bugs at:- https://github.com/facebook/zstd/issues .SH AUTHOR Yann Collet diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 2d6d45214..e829c93d6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -52,7 +52,7 @@ # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -# if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) +# if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) # else @@ -83,7 +83,7 @@ static const char* g_defaultDictName = "dictionary"; static const unsigned g_defaultMaxDictSize = 110 KB; -static const int g_defaultDictCLevel = 5; +static const int g_defaultDictCLevel = 3; static const unsigned g_defaultSelectivityLevel = 9; @@ -182,7 +182,7 @@ static void waitEnter(void) /*! readU32FromChar() : @return : unsigned integer value reach from input in `char` format Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : this function can overflow if result > MAX_UINT */ + Note : this function can overflow if digit string > MAX_UINT */ static unsigned readU32FromChar(const char** stringPtr) { unsigned result = 0; @@ -289,13 +289,13 @@ int main(int argCount, char** argv) argument++; while (argument[0]!=0) { - #ifndef ZSTD_NOCOMPRESS +#ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { dictCLevel = cLevel = readU32FromChar(&argument); continue; } - #endif +#endif switch(argument[0]) { @@ -337,7 +337,7 @@ int main(int argCount, char** argv) /* recursive */ case 'r': recursive=1; argument++; break; - #ifndef ZSTD_NOBENCH +#ifndef ZSTD_NOBENCH /* Benchmark */ case 'b': bench=1; argument++; break; @@ -368,7 +368,7 @@ int main(int argCount, char** argv) BMK_SetBlockSize(bSize); } break; - #endif /* ZSTD_NOBENCH */ +#endif /* ZSTD_NOBENCH */ /* Dictionary Selection level */ case 's': @@ -378,11 +378,11 @@ int main(int argCount, char** argv) /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ case 'p': argument++; - #ifndef ZSTD_NOBENCH +#ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { BMK_setAdditionalParam(readU32FromChar(&argument)); } else - #endif +#endif main_pause=1; break; /* unknown command */ @@ -470,7 +470,7 @@ int main(int argCount, char** argv) /* Check if input/output defined as console; trigger an error in this case */ if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); - if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode)) + if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && strcmp(filenameTable[0], stdinmark) && !(forceStdout && decode)) CLEAN_RETURN(badusage(programName)); /* user-selected output filename, only possible with a single file */ diff --git a/projects/README.md b/projects/README.md index 39742fb1a..dd60b56e8 100644 --- a/projects/README.md +++ b/projects/README.md @@ -14,7 +14,7 @@ The following projects are included with the zstd distribution: #### How to compile zstd with Visual Studio 1. Install Visual Studio e.g. VS 2015 Community Edition (it's free). -2. Download the latest version of zstd from https://github.com/Cyan4973/zstd/releases +2. Download the latest version of zstd from https://github.com/facebook/zstd/releases 3. Decompress ZIP archive. 4. Go to decompressed directory then to `projects` then `VS2010` and open `zstd.sln` 5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree. diff --git a/projects/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt index 86178bf97..b8f5d18e4 100644 --- a/projects/cmake/CMakeLists.txt +++ b/projects/cmake/CMakeLists.txt @@ -1,34 +1,10 @@ # ################################################################ -# zstd - Makefile -# Copyright (C) Yann Collet 2014-2016 +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. # All rights reserved. -# -# BSD license # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, this -# list of conditions and the following disclaimer in the documentation and/or -# other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# You can contact the author at : -# - zstd source repository : https://github.com/Cyan4973/zstd -# - Public forum : https://groups.google.com/forum/#!forum/lz4c +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. # ################################################################ PROJECT(zstd) diff --git a/tests/Makefile b/tests/Makefile index 6b22e6dbd..3ce9f317e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -160,10 +160,10 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest $(VALGRIND) ./datagen -g50M > $(VOID) $(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi ./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID) + ./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID) ./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp - $(VALGRIND) $(PRGDIR)/zstd -vdf tmp -o $(VOID) - ./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID) + $(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID) + ./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID) @rm tmp $(VALGRIND) ./fuzzer -T1mn -t1 $(VALGRIND) ./fullbench -i1 diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 18948b913..b8b7f0c93 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -23,7 +23,6 @@ **************************************/ #include /* free */ #include /* fgets, sscanf */ -#include /* timeb */ #include /* strcmp */ #include /* clock_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ diff --git a/tests/playTests.sh b/tests/playTests.sh index b887613f7..64b3fd956 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -41,6 +41,7 @@ $ECHO "\nStarting playTests.sh isWindows=$isWindows" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" file $ZSTD + $ECHO "\n**** simple tests **** " ./datagen > tmp @@ -51,6 +52,8 @@ $ZSTD -99 -f tmp # too large compression level, automatic sized down $ECHO "test : compress to stdout" $ZSTD tmp -c > tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format +$ECHO "test : implied stdout when input is stdin" +$ECHO bob | $ZSTD | $ZSTD -d $ECHO "test : null-length file roundtrip" $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout $ECHO "test : decompress file with wrong suffix (must fail)" diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 45cfe865d..8d4d172b4 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -18,7 +18,7 @@ import traceback import hashlib script_version = 'v0.8.0 (2016-08-03)' -default_repo_url = 'https://github.com/Cyan4973/zstd.git' +default_repo_url = 'https://github.com/facebook/zstd.git' working_dir_name = 'speedTest' working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest clone_path = working_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 5a24bc4d7..a5a713009 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -19,7 +19,7 @@ import sys import subprocess from subprocess import Popen, PIPE -repo_url = 'https://github.com/Cyan4973/zstd.git' +repo_url = 'https://github.com/facebook/zstd.git' tmp_dir_name = 'tests/versionsTest' make_cmd = 'make' git_cmd = 'git' diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 528979930..9dc164eaf 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -23,7 +23,7 @@ **************************************/ #include /* free */ #include /* fgets, sscanf */ -#include /* timeb */ +#include /* clock_t, clock() */ #include /* strcmp */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ @@ -58,13 +58,13 @@ static const U32 prime2 = 2246822519U; static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \ - { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \ + if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const U32 g_refreshRate = 150; -static U32 g_displayTime = 0; +static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_displayClock = 0; -static U32 g_testTime = 0; +static clock_t g_clockTime = 0; /*-******************************************************* @@ -72,23 +72,9 @@ static U32 g_testTime = 0; *********************************************************/ #define MAX(a,b) ((a)>(b)?(a):(b)) -static U32 FUZ_GetMilliStart(void) +static clock_t FUZ_GetClockSpan(clock_t clockStart) { - struct timeb tb; - U32 nCount; - ftime( &tb ); - nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); - return nCount; -} - - -static U32 FUZ_GetMilliSpan(U32 nTimeStart) -{ - U32 const nCurrent = FUZ_GetMilliStart(); - U32 nSpan = nCurrent - nTimeStart; - if (nTimeStart > nCurrent) - nSpan += 0x100000 * 1000; - return nSpan; + return clock() - clockStart; /* works even when overflow. Max span ~ 30 mn */ } /*! FUZ_rand() : @@ -291,7 +277,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres U32 coreSeed = seed; ZBUFF_CCtx* zc; ZBUFF_DCtx* zd; - U32 startTime = FUZ_GetMilliStart(); + clock_t startClock = clock(); /* allocations */ zc = ZBUFF_createCCtx(); @@ -321,7 +307,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres FUZ_rand(&coreSeed); /* test loop */ - for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) { + for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) { U32 lseed; const BYTE* srcBuffer; const BYTE* dict; @@ -543,7 +529,7 @@ int main(int argc, const char** argv) case 'i': argument++; - nbTests=0; g_testTime=0; + nbTests=0; g_clockTime=0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; nbTests += *argument - '0'; @@ -553,15 +539,15 @@ int main(int argc, const char** argv) case 'T': argument++; - nbTests=0; g_testTime=0; + nbTests=0; g_clockTime=0; while ((*argument>='0') && (*argument<='9')) { - g_testTime *= 10; - g_testTime += *argument - '0'; + g_clockTime *= 10; + g_clockTime += *argument - '0'; argument++; } - if (*argument=='m') g_testTime *=60, argument++; + if (*argument=='m') g_clockTime *=60, argument++; if (*argument=='n') argument++; - g_testTime *= 1000; + g_clockTime *= CLOCKS_PER_SEC; break; case 's': @@ -605,7 +591,11 @@ int main(int argc, const char** argv) /* Get Seed */ DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); - if (!seedset) seed = FUZ_GetMilliStart() % 10000; + if (!seedset) { + time_t const t = time(NULL); + U32 const h = XXH32(&t, sizeof(t), 1); + seed = h % 10000; + } DISPLAY("Seed = %u\n", seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index f02197491..c49e9d93e 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -23,13 +23,13 @@ **************************************/ #include /* free */ #include /* fgets, sscanf */ -#include /* timeb */ +#include /* clock_t, clock() */ #include /* strcmp */ #include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem */ #include "zstd.h" /* ZSTD_compressBound */ #include "datagen.h" /* RDG_genBuffer */ -#define XXH_STATIC_LINKING_ONLY +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH64_* */ @@ -55,13 +55,13 @@ static const U32 prime2 = 2246822519U; static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \ - { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \ + if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const U32 g_refreshRate = 150; -static U32 g_displayTime = 0; +static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_displayClock = 0; -static U32 g_testTime = 0; +static clock_t g_clockTime = 0; /*-******************************************************* @@ -69,22 +69,9 @@ static U32 g_testTime = 0; *********************************************************/ #define MAX(a,b) ((a)>(b)?(a):(b)) -static U32 FUZ_GetMilliStart(void) +static clock_t FUZ_GetClockSpan(clock_t clockStart) { - struct timeb tb; - U32 nCount; - ftime( &tb ); - nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); - return nCount; -} - -static U32 FUZ_GetMilliSpan(U32 nTimeStart) -{ - U32 const nCurrent = FUZ_GetMilliStart(); - U32 nSpan = nCurrent - nTimeStart; - if (nTimeStart > nCurrent) - nSpan += 0x100000 * 1000; - return nSpan; + return clock() - clockStart; /* works even when overflow. Max span ~ 30 mn */ } /*! FUZ_rand() : @@ -336,7 +323,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres U32 coreSeed = seed; ZSTD_CStream* zc; ZSTD_DStream* zd; - U32 startTime = FUZ_GetMilliStart(); + clock_t startClock = clock(); /* allocations */ zc = ZSTD_createCStream(); @@ -366,7 +353,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres FUZ_rand(&coreSeed); /* test loop */ - for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) { + for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) { U32 lseed; const BYTE* srcBuffer; const BYTE* dict; @@ -594,7 +581,7 @@ int main(int argc, const char** argv) case 'i': argument++; - nbTests=0; g_testTime=0; + nbTests=0; g_clockTime=0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; nbTests += *argument - '0'; @@ -604,15 +591,15 @@ int main(int argc, const char** argv) case 'T': argument++; - nbTests=0; g_testTime=0; + nbTests=0; g_clockTime=0; while ((*argument>='0') && (*argument<='9')) { - g_testTime *= 10; - g_testTime += *argument - '0'; + g_clockTime *= 10; + g_clockTime += *argument - '0'; argument++; } - if (*argument=='m') g_testTime *=60, argument++; + if (*argument=='m') g_clockTime *=60, argument++; if (*argument=='n') argument++; - g_testTime *= 1000; + g_clockTime *= CLOCKS_PER_SEC; break; case 's': @@ -656,7 +643,12 @@ int main(int argc, const char** argv) /* Get Seed */ DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); - if (!seedset) seed = FUZ_GetMilliStart() % 10000; + if (!seedset) { + time_t const t = time(NULL); + U32 const h = XXH32(&t, sizeof(t), 1); + seed = h % 10000; + } + DISPLAY("Seed = %u\n", seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); diff --git a/zstd.rb b/zstd.rb deleted file mode 100644 index 0aa72a213..000000000 --- a/zstd.rb +++ /dev/null @@ -1,18 +0,0 @@ -class Zstd < Formula - desc "Zstandard - Fast real-time compression algorithm" - homepage "http://www.zstd.net/" - url "https://github.com/Cyan4973/zstd/archive/v1.0.0.tar.gz" - sha256 "197e6ef74da878cbf72844f38461bb18129d144fd5221b3598e973ecda6f5963" - - def install - system "make", "install", "PREFIX=#{prefix}" - end - - test do - (testpath/"input.txt").write("Hello, world." * 10) - system "#{bin}/zstd", "input.txt", "-o", "compressed.zst" - system "#{bin}/zstd", "--test", "compressed.zst" - system "#{bin}/zstd", "-d", "compressed.zst", "-o", "decompressed.txt" - system "cmp", "input.txt", "decompressed.txt" - end -end diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 3facb3210..8a5d7b77e 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -551,7 +551,7 @@ Let's presume the following Huffman tree must be described : The tree depth is 4, since its smallest element uses 4 bits. Value `5` will not be listed, nor will values above `5`. Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`. -Weight formula is : +Weight formula is : ``` Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0 ``` @@ -779,7 +779,7 @@ which specifies `Baseline` and `Number_of_Bits` to add. _Codes_ are FSE compressed, and interleaved with raw additional bits in the same bitstream. -##### Literals length codes +##### Literals length codes Literals length codes are values ranging from `0` to `35` included. They define lengths from 0 to 131071 bytes. @@ -1126,10 +1126,10 @@ When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`. Dictionary format ----------------- -`zstd` is compatible with "pure content" dictionaries, free of any format restriction. +`zstd` is compatible with "raw content" dictionaries, free of any format restriction. But dictionaries created by `zstd --train` follow a format, described here. -__Pre-requisites__ : a dictionary has a known length, +__Pre-requisites__ : a dictionary has a size, defined either by a buffer limit, or a file size. | `Magic_Number` | `Dictionary_ID` | `Entropy_Tables` | `Content` | @@ -1151,20 +1151,21 @@ _Reserved ranges :_ - high range : >= (2^31) __`Entropy_Tables`__ : following the same format as a [compressed blocks]. - They are stored in following order : - Huffman tables for literals, FSE table for offsets, - FSE table for match lengths, and FSE table for literals lengths. - It's finally followed by 3 offset values, populating recent offsets, - stored in order, 4-bytes little-endian each, for a total of 12 bytes. + They are stored in following order : + Huffman tables for literals, FSE table for offsets, + FSE table for match lengths, and FSE table for literals lengths. + It's finally followed by 3 offset values, populating recent offsets, + stored in order, 4-bytes little-endian each, for a total of 12 bytes. -__`Content`__ : Where the actual dictionary content is. - Content size depends on Dictionary size. +__`Content`__ : The rest of the dictionary is its content. + The content act as a "past" in front of data to compress or decompress. [compressed blocks]: #the-format-of-compressed_block Version changes --------------- +- 0.2.1 : clarify field names, by Przemyslaw Skibinski - 0.2.0 : numerous format adjustments for zstd v0.8 - 0.1.2 : limit Huffman tree depth to 11 bits - 0.1.1 : reserved dictID ranges