";
+ for (l=0; l
" << endl << endl;
+ } else { /* comments of type /** and /*- mean: this is a comment; use a header for the first line */
+ if (comments.empty()) continue;
+
+ trim(comments[0], " ");
+ sout << "" << comments[0] << "
";
+ chapters.push_back(comments[0]);
+ chapter++;
+
+ for (l=1; l 1)
+ sout << "
" << endl << endl;
+ else
+ sout << "" << endl << endl;
+ }
+ }
+
+ ostream << "\n\n\n" << version << "\n\n" << endl;
+ ostream << "" << version << "
\n";
+
+ ostream << "
\nContents
\n\n";
+ for (size_t i=0; i" << chapters[i].c_str() << "\n";
+ ostream << "
\n
\n";
+
+ ostream << sout.str();
+ ostream << "" << endl << "" << endl;
+
+ return 0;
+}
\ No newline at end of file
diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h
new file mode 100644
index 000000000..76c982ab2
--- /dev/null
+++ b/contrib/pzstd/Logging.h
@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include
+#include
+
+namespace pzstd {
+
+constexpr int ERROR = 1;
+constexpr int INFO = 2;
+constexpr int DEBUG = 3;
+constexpr int VERBOSE = 4;
+
+class Logger {
+ std::mutex mutex_;
+ FILE* out_;
+ const int level_;
+
+ using Clock = std::chrono::system_clock;
+ Clock::time_point lastUpdate_;
+ std::chrono::milliseconds refreshRate_;
+
+ public:
+ explicit Logger(int level, FILE* out = stderr)
+ : out_(out), level_(level), lastUpdate_(Clock::now()),
+ refreshRate_(150) {}
+
+
+ bool logsAt(int level) {
+ return level <= level_;
+ }
+
+ template
+ void operator()(int level, const char *fmt, Args... args) {
+ if (level > level_) {
+ return;
+ }
+ std::lock_guard lock(mutex_);
+ std::fprintf(out_, fmt, args...);
+ }
+
+ template
+ void update(int level, const char *fmt, Args... args) {
+ if (level > level_) {
+ return;
+ }
+ std::lock_guard lock(mutex_);
+ auto now = Clock::now();
+ if (now - lastUpdate_ > refreshRate_) {
+ lastUpdate_ = now;
+ std::fprintf(out_, "\r");
+ std::fprintf(out_, fmt, args...);
+ }
+ }
+
+ void clear(int level) {
+ if (level > level_) {
+ return;
+ }
+ std::lock_guard lock(mutex_);
+ std::fprintf(out_, "\r%79s\r", "");
+ }
+};
+
+}
diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index e30be0bed..2de50416c 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -7,20 +7,71 @@
# of patent rights can be found in the PATENTS file in the same directory.
# ##########################################################################
+# Standard variables for installation
+DESTDIR ?=
+PREFIX ?= /usr/local
+BINDIR := $(DESTDIR)$(PREFIX)/bin
+
ZSTDDIR = ../../lib
PROGDIR = ../../programs
-CPPFLAGS = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
-CXXFLAGS ?= -O3
-CXXFLAGS += -std=c++11
-CXXFLAGS += $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
+# External program to use to run tests, e.g. qemu or valgrind
+TESTPROG ?=
+# Flags to pass to the tests
+TESTFLAGS ?=
+
+# We use gcc/clang to generate the header dependencies of files
+DEPFLAGS = -MMD -MP -MF $*.Td
+POSTCOMPILE = mv -f $*.Td $*.d
+
+# CFLAGS, CXXFLAGS, CPPFLAGS, and LDFLAGS are for the users to override
+CFLAGS ?= -O3 -Wall -Wextra
+CXXFLAGS ?= -O3 -Wall -Wextra -pedantic -std=c++11
+CPPFLAGS ?=
+LDFLAGS ?=
+
+# Include flags
+PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+GTEST_INC = -isystem googletest/googletest/include
+
+PZSTD_CPPFLAGS = $(PZSTD_INC) $(GTEST_INC)
+PZSTD_CCXXFLAGS =
+PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS)
+PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS)
+PZSTD_LDFLAGS =
+EXTRA_FLAGS =
+ALL_CFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS) $(PZSTD_CFLAGS)
+ALL_CXXFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS)
+ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
-ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
-ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+# gtest libraries need to go before "-lpthread" because they depend on it.
+GTEST_LIB = -L googletest/build/googlemock/gtest
+LIBS = $(GTEST_LIB) -lpthread
+
+# Compilation commands
+LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -o $@
+CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@
+CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@
+
+# Get a list of all zstd files so we rebuild the static library when we need to
+ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \
+ $(wildcard $(ZSTDDIR)/common/*.h)
+ZSTDCOMP_FILES := $(wildcard $(ZSTDDIR)/compress/*.c) \
+ $(wildcard $(ZSTDDIR)/compress/*.h)
+ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \
+ $(wildcard $(ZSTDDIR)/decompress/*.h)
+ZSTDPROG_FILES := $(wildcard $(PROGDIR)/*.c) \
+ $(wildcard $(PROGDIR)/*.h)
+ZSTD_FILES := $(wildcard $(ZSTDDIR)/*.h) \
+ $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
+ $(ZSTDPROG_FILES)
+
+# List all the pzstd source files so we can determine their dependencies
+PZSTD_SRCS := $(wildcard *.cpp)
+PZSTD_TESTS := $(wildcard test/*.cpp)
+UTILS_TESTS := $(wildcard utils/test/*.cpp)
+ALL_SRCS := $(PZSTD_SRCS) $(PZSTD_TESTS) $(UTILS_TESTS)
# Define *.exe as extension for Windows systems
@@ -30,89 +81,169 @@ else
EXT =
endif
-.PHONY: default all test clean test32 googletest googletest32
+# Standard targets
+.PHONY: default
+default: all
-default: pzstd
+.PHONY: check
+check:
+ $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/ResourcePoolTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./test/OptionsTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./test/PzstdTest$(EXT) $(TESTFLAGS)
-all: pzstd
+.PHONY: install
+install: PZSTD_CPPFLAGS += -DNDEBUG
+install: pzstd$(EXT)
+ install -d -m 755 $(BINDIR)/
+ install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT)
+
+.PHONY: uninstall
+uninstall:
+ $(RM) $(BINDIR)/pzstd$(EXT)
+
+# Targets for many different builds
+.PHONY: all
+all: PZSTD_CPPFLAGS += -DNDEBUG
+all: pzstd$(EXT) tests roundtrip
+
+.PHONY: debug
+debug: EXTRA_FLAGS += -g
+debug: pzstd$(EXT) tests roundtrip
+
+.PHONY: tsan
+tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC
+tsan: PZSTD_LDFLAGS += -fsanitize=thread -pie
+tsan: debug
+
+.PHONY: asan
+asan: EXTRA_FLAGS += -fsanitize=address
+asan: debug
+
+.PHONY: ubsan
+ubsan: EXTRA_FLAGS += -fsanitize=undefined
+ubsan: debug
+
+.PHONY: all32
+all32: EXTRA_FLAGS += -m32
+all32: all
+
+.PHONY: debug32
+debug32: EXTRA_FLAGS += -m32
+debug32: debug
+
+.PHONY: asan32
+asan32: EXTRA_FLAGS += -m32
+asan32: asan
+
+.PHONY: tsan32
+tsan32: EXTRA_FLAGS += -m32
+tsan32: tsan
+
+.PHONY: ubsan32
+ubsan32: EXTRA_FLAGS += -m32
+ubsan32: ubsan
+
+# Run long round trip tests
+.PHONY: roundtripcheck
+roundtripcheck: roundtrip check
+ $(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
+
+# Build the main binary
+pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+ $(LD_COMMAND)
+
+# Target that depends on all the tests
+.PHONY: tests
+tests: EXTRA_FLAGS += -Wno-deprecated-declarations
+tests: $(patsubst %,%$(EXT),$(basename $(PZSTD_TESTS) $(UTILS_TESTS)))
+
+# Build the round trip tests
+.PHONY: roundtrip
+roundtrip: EXTRA_FLAGS += -Wno-deprecated-declarations
+roundtrip: test/RoundTripTest$(EXT)
+
+# Use the static library that zstd builds for simplicity and
+# so we get the compiler options correct
+$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
+ $(MAKE) -C $(ZSTDDIR) libzstd CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)"
-libzstd.a: $(ZSTD_FILES)
- $(MAKE) -C $(ZSTDDIR) libzstd
- @cp $(ZSTDDIR)/libzstd.a .
+# Rules to build the tests
+test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \
+ Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+ $(LD_COMMAND)
-Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
- $(CXX) $(FLAGS) -c Pzstd.cpp -o $@
+test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main
+test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \
+ SkippableFrame.o $(ZSTDDIR)/libzstd.a
+ $(LD_COMMAND)
-SkippableFrame.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
- $(CXX) $(FLAGS) -c SkippableFrame.cpp -o $@
+utils/test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main
+utils/test/%Test$(EXT): utils/test/%Test.o
+ $(LD_COMMAND)
-Options.o: Options.h Options.cpp
- $(CXX) $(FLAGS) -c Options.cpp -o $@
-main.o: main.cpp *.h utils/*.h
- $(CXX) $(FLAGS) -c main.cpp -o $@
-
-pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a
- $(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread
-
-libzstd32.a: $(ZSTD_FILES)
- $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32"
- @cp $(ZSTDDIR)/libzstd.a libzstd32.a
-
-Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
- $(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@
-
-SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
- $(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@
-
-Options32.o: Options.h Options.cpp
- $(CXX) -m32 $(FLAGS) -c Options.cpp -o $@
-
-main32.o: main.cpp *.h utils/*.h
- $(CXX) -m32 $(FLAGS) -c main.cpp -o $@
-
-pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a
- $(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread
+GTEST_CMAKEFLAGS =
+# Install googletest
+.PHONY: googletest
+googletest: PZSTD_CCXXFLAGS += -fPIC
googletest:
@$(RM) -rf googletest
@git clone https://github.com/google/googletest
@mkdir -p googletest/build
- @cd googletest/build && cmake .. && make
+ @cd googletest/build && cmake $(GTEST_CMAKEFLAGS) -DCMAKE_CXX_FLAGS="$(ALL_CXXFLAGS)" .. && $(MAKE)
-googletest32:
- @$(RM) -rf googletest
- @git clone https://github.com/google/googletest
- @mkdir -p googletest/build
- @cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make
-
-googletest-mingw64:
- $(RM) -rf googletest
- git clone https://github.com/google/googletest
- mkdir -p googletest/build
- cd googletest/build && cmake -G "MSYS Makefiles" .. && $(MAKE)
-
-test:
- $(MAKE) libzstd.a
- $(MAKE) pzstd MOREFLAGS="-Wall -Wextra -pedantic -Werror"
- $(MAKE) -C utils/test clean
- $(MAKE) -C utils/test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
- $(MAKE) -C test clean
- $(MAKE) -C test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
-
-test32:
- $(MAKE) libzstd.a MOREFLAGS="-m32"
- $(MAKE) pzstd MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
- $(MAKE) -C utils/test clean
- $(MAKE) -C utils/test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
- $(MAKE) -C test clean
- $(MAKE) -C test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+.PHONY: googletest32
+googletest32: PZSTD_CCXXFLAGS += -m32
+googletest32: googletest
+.PHONY: googletest-mingw64
+googletest-mingw64: GTEST_CMAKEFLAGS += -G "MSYS Makefiles"
+googletest-mingw64: googletest
+.PHONY: clean
clean:
+ $(RM) -f *.o pzstd$(EXT) *.Td *.d
+ $(RM) -f test/*.o test/*Test$(EXT) test/*.Td test/*.d
+ $(RM) -f utils/test/*.o utils/test/*Test$(EXT) utils/test/*.Td utils/test/*.d
+ $(RM) -f $(PROGDIR)/*.o $(PROGDIR)/*.Td $(PROGDIR)/*.d
$(MAKE) -C $(ZSTDDIR) clean
- $(MAKE) -C utils/test clean
- $(MAKE) -C test clean
- @$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT)
@echo Cleaning completed
+
+
+# Cancel implicit rules
+%.o: %.c
+%.o: %.cpp
+
+# Object file rules
+%.o: %.c
+ $(CC_COMMAND)
+ $(POSTCOMPILE)
+
+$(PROGDIR)/%.o: $(PROGDIR)/%.c
+ $(CC_COMMAND)
+ $(POSTCOMPILE)
+
+%.o: %.cpp
+ $(CXX_COMMAND)
+ $(POSTCOMPILE)
+
+test/%.o: test/%.cpp
+ $(CXX_COMMAND)
+ $(POSTCOMPILE)
+
+utils/test/%.o: utils/test/%.cpp
+ $(CXX_COMMAND)
+ $(POSTCOMPILE)
+
+# Dependency file stuff
+.PRECIOUS: %.d test/%.d utils/test/%.d
+
+# Include rules that specify header file dependencies
+-include $(patsubst %,%.d,$(basename $(ALL_SRCS)))
diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index ece8c0782..18c069eae 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -303,6 +303,12 @@ Options::Status Options::parse(int argc, const char **argv) {
} // while (*options != 0);
} // for (int i = 1; i < argc; ++i);
+ // Set options for test mode
+ if (test) {
+ outputFile = nullOutput;
+ keepSource = true;
+ }
+
// Input file defaults to standard input if not provided.
if (localInputFiles.empty()) {
localInputFiles.emplace_back(kStdIn);
@@ -399,11 +405,6 @@ Options::Status Options::parse(int argc, const char **argv) {
verbosity = 1;
}
- // Set options for test mode
- if (test) {
- outputFile = nullOutput;
- keepSource = true;
- }
return Status::Success;
}
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index e0826b9d8..c5b4ce4cb 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -15,6 +15,7 @@
#include "utils/WorkQueue.h"
#include
+#include
#include
#include
#include
@@ -58,26 +59,24 @@ static std::uint64_t handleOneInput(const Options &options,
FILE* inputFd,
const std::string &outputFile,
FILE* outputFd,
- ErrorHolder &errorHolder) {
+ SharedState& state) {
auto inputSize = fileSizeOrZero(inputFile);
// WorkQueue outlives ThreadPool so in the case of error we are certain
- // we don't accidently try to call push() on it after it is destroyed.
+ // we don't accidently try to call push() on it after it is destroyed
WorkQueue> outs{options.numThreads + 1};
std::uint64_t bytesRead;
std::uint64_t bytesWritten;
{
- // Initialize the thread pool with numThreads + 1
- // We add one because the read thread spends most of its time waiting.
- // This also sets the minimum number of threads to 2, so the algorithm
- // doesn't deadlock.
- ThreadPool executor(options.numThreads + 1);
+ // Initialize the (de)compression thread pool with numThreads
+ ThreadPool executor(options.numThreads);
+ // Run the reader thread on an extra thread
+ ThreadPool readExecutor(1);
if (!options.decompress) {
// Add a job that reads the input and starts all the compression jobs
- executor.add(
- [&errorHolder, &outs, &executor, inputFd, inputSize, &options,
- &bytesRead] {
+ readExecutor.add(
+ [&state, &outs, &executor, inputFd, inputSize, &options, &bytesRead] {
bytesRead = asyncCompressChunks(
- errorHolder,
+ state,
outs,
executor,
inputFd,
@@ -86,29 +85,28 @@ static std::uint64_t handleOneInput(const Options &options,
options.determineParameters());
});
// Start writing
- bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress,
- options.verbosity);
+ bytesWritten = writeFile(state, outs, outputFd, options.decompress);
} else {
// Add a job that reads the input and starts all the decompression jobs
- executor.add([&errorHolder, &outs, &executor, inputFd, &bytesRead] {
- bytesRead = asyncDecompressFrames(errorHolder, outs, executor, inputFd);
+ readExecutor.add([&state, &outs, &executor, inputFd, &bytesRead] {
+ bytesRead = asyncDecompressFrames(state, outs, executor, inputFd);
});
// Start writing
- bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress,
- options.verbosity);
+ bytesWritten = writeFile(state, outs, outputFd, options.decompress);
}
}
- if (options.verbosity > 1 && !errorHolder.hasError()) {
+ if (!state.errorHolder.hasError()) {
std::string inputFileName = inputFile == "-" ? "stdin" : inputFile;
std::string outputFileName = outputFile == "-" ? "stdout" : outputFile;
if (!options.decompress) {
double ratio = static_cast(bytesWritten) /
static_cast(bytesRead + !bytesRead);
- std::fprintf(stderr, "%-20s :%6.2f%% (%6llu => %6llu bytes, %s)\n",
+ state.log(INFO, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64
+ " bytes, %s)\n",
inputFileName.c_str(), ratio * 100, bytesRead, bytesWritten,
outputFileName.c_str());
} else {
- std::fprintf(stderr, "%-20s: %llu bytes \n",
+ state.log(INFO, "%-20s: %" PRIu64 " bytes \n",
inputFileName.c_str(),bytesWritten);
}
}
@@ -138,7 +136,7 @@ static FILE *openInputFile(const std::string &inputFile,
static FILE *openOutputFile(const Options &options,
const std::string &outputFile,
- ErrorHolder &errorHolder) {
+ SharedState& state) {
if (outputFile == "-") {
SET_BINARY_MODE(stdout);
return stdout;
@@ -148,82 +146,78 @@ static FILE *openOutputFile(const Options &options,
auto outputFd = std::fopen(outputFile.c_str(), "rb");
if (outputFd != nullptr) {
std::fclose(outputFd);
- if (options.verbosity <= 1) {
- errorHolder.setError("Output file exists");
+ if (!state.log.logsAt(INFO)) {
+ state.errorHolder.setError("Output file exists");
return nullptr;
}
- std::fprintf(
- stderr,
+ state.log(
+ INFO,
"pzstd: %s already exists; do you wish to overwrite (y/n) ? ",
outputFile.c_str());
int c = getchar();
if (c != 'y' && c != 'Y') {
- errorHolder.setError("Not overwritten");
+ state.errorHolder.setError("Not overwritten");
return nullptr;
}
}
}
auto outputFd = std::fopen(outputFile.c_str(), "wb");
- if (!errorHolder.check(
+ if (!state.errorHolder.check(
outputFd != nullptr, "Failed to open output file")) {
- return 0;
+ return nullptr;
}
return outputFd;
}
int pzstdMain(const Options &options) {
int returnCode = 0;
+ SharedState state(options);
for (const auto& input : options.inputFiles) {
- // Setup the error holder
- ErrorHolder errorHolder;
+ // Setup the shared state
auto printErrorGuard = makeScopeGuard([&] {
- if (errorHolder.hasError()) {
+ if (state.errorHolder.hasError()) {
returnCode = 1;
- if (options.verbosity > 0) {
- std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(),
- errorHolder.getError().c_str());
- }
- } else {
-
+ state.log(ERROR, "pzstd: %s: %s.\n", input.c_str(),
+ state.errorHolder.getError().c_str());
}
});
// Open the input file
- auto inputFd = openInputFile(input, errorHolder);
+ auto inputFd = openInputFile(input, state.errorHolder);
if (inputFd == nullptr) {
continue;
}
auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); });
// Open the output file
auto outputFile = options.getOutputFile(input);
- if (!errorHolder.check(outputFile != "",
+ if (!state.errorHolder.check(outputFile != "",
"Input file does not have extension .zst")) {
continue;
}
- auto outputFd = openOutputFile(options, outputFile, errorHolder);
+ auto outputFd = openOutputFile(options, outputFile, state);
if (outputFd == nullptr) {
continue;
}
auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); });
// (de)compress the file
- handleOneInput(options, input, inputFd, outputFile, outputFd, errorHolder);
- if (errorHolder.hasError()) {
+ handleOneInput(options, input, inputFd, outputFile, outputFd, state);
+ if (state.errorHolder.hasError()) {
continue;
}
// Delete the input file if necessary
if (!options.keepSource) {
// Be sure that we are done and have written everything before we delete
- if (!errorHolder.check(std::fclose(inputFd) == 0,
+ if (!state.errorHolder.check(std::fclose(inputFd) == 0,
"Failed to close input file")) {
continue;
}
closeInputGuard.dismiss();
- if (!errorHolder.check(std::fclose(outputFd) == 0,
+ if (!state.errorHolder.check(std::fclose(outputFd) == 0,
"Failed to close output file")) {
continue;
}
closeOutputGuard.dismiss();
if (std::remove(input.c_str()) != 0) {
- errorHolder.setError("Failed to remove input file");
+ state.errorHolder.setError("Failed to remove input file");
continue;
}
}
@@ -269,27 +263,25 @@ Buffer split(Buffer& buffer, ZSTD_outBuffer& outBuffer) {
/**
* Stream chunks of input from `in`, compress it, and stream it out to `out`.
*
- * @param errorHolder Used to report errors and check if an error occured
+ * @param state The shared state
* @param in Queue that we `pop()` input buffers from
* @param out Queue that we `push()` compressed output buffers to
* @param maxInputSize An upper bound on the size of the input
- * @param parameters The zstd parameters to use for compression
*/
static void compress(
- ErrorHolder& errorHolder,
+ SharedState& state,
std::shared_ptr in,
std::shared_ptr out,
- size_t maxInputSize,
- ZSTD_parameters parameters) {
+ size_t maxInputSize) {
+ auto& errorHolder = state.errorHolder;
auto guard = makeScopeGuard([&] { out->finish(); });
// Initialize the CCtx
- std::unique_ptr ctx(
- ZSTD_createCStream(), ZSTD_freeCStream);
+ auto ctx = state.cStreamPool->get();
if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
return;
}
{
- auto err = ZSTD_initCStream_advanced(ctx.get(), nullptr, 0, parameters, 0);
+ auto err = ZSTD_resetCStream(ctx.get(), 0);
if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
return;
}
@@ -396,7 +388,7 @@ readData(BufferWorkQueue& queue, size_t chunkSize, size_t size, FILE* fd,
}
std::uint64_t asyncCompressChunks(
- ErrorHolder& errorHolder,
+ SharedState& state,
WorkQueue>& chunks,
ThreadPool& executor,
FILE* fd,
@@ -410,23 +402,23 @@ std::uint64_t asyncCompressChunks(
// independently.
size_t step = calculateStep(size, numThreads, params);
auto status = FileStatus::Continue;
- while (status == FileStatus::Continue && !errorHolder.hasError()) {
+ while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
// Make a new input queue that we will put the chunk's input data into.
auto in = std::make_shared();
auto inGuard = makeScopeGuard([&] { in->finish(); });
// Make a new output queue that compress will put the compressed data into.
auto out = std::make_shared();
// Start compression in the thread pool
- executor.add([&errorHolder, in, out, step, params] {
+ executor.add([&state, in, out, step] {
return compress(
- errorHolder, std::move(in), std::move(out), step, params);
+ state, std::move(in), std::move(out), step);
});
// Pass the output queue to the writer thread.
chunks.push(std::move(out));
// Fill the input queue for the compression job we just started
status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead);
}
- errorHolder.check(status != FileStatus::Error, "Error reading input");
+ state.errorHolder.check(status != FileStatus::Error, "Error reading input");
return bytesRead;
}
@@ -434,24 +426,24 @@ std::uint64_t asyncCompressChunks(
* Decompress a frame, whose data is streamed into `in`, and stream the output
* to `out`.
*
- * @param errorHolder Used to report errors and check if an error occured
+ * @param state The shared state
* @param in Queue that we `pop()` input buffers from. It contains
* exactly one compressed frame.
* @param out Queue that we `push()` decompressed output buffers to
*/
static void decompress(
- ErrorHolder& errorHolder,
+ SharedState& state,
std::shared_ptr in,
std::shared_ptr out) {
+ auto& errorHolder = state.errorHolder;
auto guard = makeScopeGuard([&] { out->finish(); });
// Initialize the DCtx
- std::unique_ptr ctx(
- ZSTD_createDStream(), ZSTD_freeDStream);
+ auto ctx = state.dStreamPool->get();
if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
return;
}
{
- auto err = ZSTD_initDStream(ctx.get());
+ auto err = ZSTD_resetDStream(ctx.get());
if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
return;
}
@@ -509,7 +501,7 @@ static void decompress(
}
std::uint64_t asyncDecompressFrames(
- ErrorHolder& errorHolder,
+ SharedState& state,
WorkQueue>& frames,
ThreadPool& executor,
FILE* fd) {
@@ -522,7 +514,7 @@ std::uint64_t asyncDecompressFrames(
// Otherwise, we will decompress using only one decompression task.
const size_t chunkSize = ZSTD_DStreamInSize();
auto status = FileStatus::Continue;
- while (status == FileStatus::Continue && !errorHolder.hasError()) {
+ while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
// Make a new input queue that we will put the frames's bytes into.
auto in = std::make_shared();
auto inGuard = makeScopeGuard([&] { in->finish(); });
@@ -551,15 +543,15 @@ std::uint64_t asyncDecompressFrames(
out->setMaxSize(64);
}
// Start decompression in the thread pool
- executor.add([&errorHolder, in, out] {
- return decompress(errorHolder, std::move(in), std::move(out));
+ executor.add([&state, in, out] {
+ return decompress(state, std::move(in), std::move(out));
});
// Pass the output queue to the writer thread
frames.push(std::move(out));
if (frameSize == 0) {
// We hit a non SkippableFrame ==> not compressed by pzstd or corrupted
// Pass the rest of the source to this decompression task
- while (status == FileStatus::Continue && !errorHolder.hasError()) {
+ while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead);
}
break;
@@ -567,7 +559,7 @@ std::uint64_t asyncDecompressFrames(
// Fill the input queue for the decompression job we just started
status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead);
}
- errorHolder.check(status != FileStatus::Error, "Error reading input");
+ state.errorHolder.check(status != FileStatus::Error, "Error reading input");
return totalBytesRead;
}
@@ -582,32 +574,14 @@ static bool writeData(ByteRange data, FILE* fd) {
return true;
}
-void updateWritten(int verbosity, std::uint64_t bytesWritten) {
- if (verbosity <= 1) {
- return;
- }
- using Clock = std::chrono::system_clock;
- static Clock::time_point then;
- constexpr std::chrono::milliseconds refreshRate{150};
-
- auto now = Clock::now();
- if (now - then > refreshRate) {
- then = now;
- std::fprintf(stderr, "\rWritten: %u MB ",
- static_cast(bytesWritten >> 20));
- }
-}
-
std::uint64_t writeFile(
- ErrorHolder& errorHolder,
+ SharedState& state,
WorkQueue>& outs,
FILE* outputFd,
- bool decompress,
- int verbosity) {
- auto lineClearGuard = makeScopeGuard([verbosity] {
- if (verbosity > 1) {
- std::fprintf(stderr, "\r%79s\r", "");
- }
+ bool decompress) {
+ auto& errorHolder = state.errorHolder;
+ auto lineClearGuard = makeScopeGuard([&state] {
+ state.log.clear(INFO);
});
std::uint64_t bytesWritten = 0;
std::shared_ptr out;
@@ -633,7 +607,8 @@ std::uint64_t writeFile(
return bytesWritten;
}
bytesWritten += buffer.size();
- updateWritten(verbosity, bytesWritten);
+ state.log.update(INFO, "Written: %u MB ",
+ static_cast(bytesWritten >> 20));
}
}
return bytesWritten;
diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h
index fe44ccfde..9fb2c4884 100644
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@@ -9,9 +9,11 @@
#pragma once
#include "ErrorHolder.h"
+#include "Logging.h"
#include "Options.h"
#include "utils/Buffer.h"
#include "utils/Range.h"
+#include "utils/ResourcePool.h"
#include "utils/ThreadPool.h"
#include "utils/WorkQueue.h"
#define ZSTD_STATIC_LINKING_ONLY
@@ -32,12 +34,58 @@ namespace pzstd {
*/
int pzstdMain(const Options& options);
+class SharedState {
+ public:
+ SharedState(const Options& options) : log(options.verbosity) {
+ if (!options.decompress) {
+ auto parameters = options.determineParameters();
+ cStreamPool.reset(new ResourcePool{
+ [parameters]() -> ZSTD_CStream* {
+ auto zcs = ZSTD_createCStream();
+ if (zcs) {
+ auto err = ZSTD_initCStream_advanced(
+ zcs, nullptr, 0, parameters, 0);
+ if (ZSTD_isError(err)) {
+ ZSTD_freeCStream(zcs);
+ return nullptr;
+ }
+ }
+ return zcs;
+ },
+ [](ZSTD_CStream *zcs) {
+ ZSTD_freeCStream(zcs);
+ }});
+ } else {
+ dStreamPool.reset(new ResourcePool{
+ []() -> ZSTD_DStream* {
+ auto zds = ZSTD_createDStream();
+ if (zds) {
+ auto err = ZSTD_initDStream(zds);
+ if (ZSTD_isError(err)) {
+ ZSTD_freeDStream(zds);
+ return nullptr;
+ }
+ }
+ return zds;
+ },
+ [](ZSTD_DStream *zds) {
+ ZSTD_freeDStream(zds);
+ }});
+ }
+ }
+
+ Logger log;
+ ErrorHolder errorHolder;
+ std::unique_ptr> cStreamPool;
+ std::unique_ptr> dStreamPool;
+};
+
/**
* Streams input from `fd`, breaks input up into chunks, and compresses each
* chunk independently. Output of each chunk gets streamed to a queue, and
* the output queues get put into `chunks` in order.
*
- * @param errorHolder Used to report errors and coordinate early shutdown
+ * @param state The shared state
* @param chunks Each compression jobs output queue gets `pushed()` here
* as soon as it is available
* @param executor The thread pool to run compression jobs in
@@ -48,7 +96,7 @@ int pzstdMain(const Options& options);
* @returns The number of bytes read from the file
*/
std::uint64_t asyncCompressChunks(
- ErrorHolder& errorHolder,
+ SharedState& state,
WorkQueue>& chunks,
ThreadPool& executor,
FILE* fd,
@@ -62,7 +110,7 @@ std::uint64_t asyncCompressChunks(
* decompression job. Output of each frame gets streamed to a queue, and
* the output queues get put into `frames` in order.
*
- * @param errorHolder Used to report errors and coordinate early shutdown
+ * @param state The shared state
* @param frames Each decompression jobs output queue gets `pushed()` here
* as soon as it is available
* @param executor The thread pool to run compression jobs in
@@ -70,7 +118,7 @@ std::uint64_t asyncCompressChunks(
* @returns The number of bytes read from the file
*/
std::uint64_t asyncDecompressFrames(
- ErrorHolder& errorHolder,
+ SharedState& state,
WorkQueue>& frames,
ThreadPool& executor,
FILE* fd);
@@ -79,18 +127,16 @@ std::uint64_t asyncDecompressFrames(
* Streams input in from each queue in `outs` in order, and writes the data to
* `outputFd`.
*
- * @param errorHolder Used to report errors and coordinate early exit
+ * @param state The shared state
* @param outs A queue of output queues, one for each
* (de)compression job.
* @param outputFd The file descriptor to write to
* @param decompress Are we decompressing?
- * @param verbosity The verbosity level to log at
* @returns The number of bytes written
*/
std::uint64_t writeFile(
- ErrorHolder& errorHolder,
+ SharedState& state,
WorkQueue>& outs,
FILE* outputFd,
- bool decompress,
- int verbosity);
+ bool decompress);
}
diff --git a/contrib/pzstd/README.md b/contrib/pzstd/README.md
index 05ceb5599..3fe7b0b9d 100644
--- a/contrib/pzstd/README.md
+++ b/contrib/pzstd/README.md
@@ -10,7 +10,7 @@ When decompressing files compressed with Zstandard, PZstandard does IO in one th
## Usage
-PZstandard supports the same command line interface as Zstandard, but also provies the `-p` option to specify the number of threads.
+PZstandard supports the same command line interface as Zstandard, but also provides the `-p` option to specify the number of threads.
Dictionary mode is not currently supported.
Basic usage
diff --git a/contrib/pzstd/test/Makefile b/contrib/pzstd/test/Makefile
deleted file mode 100644
index 4f6ba9997..000000000
--- a/contrib/pzstd/test/Makefile
+++ /dev/null
@@ -1,48 +0,0 @@
-# ##########################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ##########################################################################
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
-else
-EXT =
-endif
-
-PZSTDDIR = ..
-PROGDIR = ../../../programs
-ZSTDDIR = ../../../lib
-
-# Set GTEST_INC and GTEST_LIB to work with your install of gtest
-GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
-GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
-GTEST_FLAGS = $(GTEST_INC) $(GTEST_LIB)
-CPPFLAGS = -I$(PZSTDDIR) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
-
-CXXFLAGS ?= -O3
-CXXFLAGS += -std=c++11 -Wno-deprecated-declarations
-CXXFLAGS += $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
-
-datagen.o: $(PROGDIR)/datagen.*
- $(CC) $(CPPFLAGS) -O3 $(MOREFLAGS) $(LDFLAGS) -Wno-long-long -Wno-variadic-macros $(PROGDIR)/datagen.c -c -o $@
-
-%: %.cpp *.h datagen.o
- $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) $(GTEST_FLAGS) -lgtest -lgtest_main -lpthread
-
-.PHONY: test clean
-
-test: OptionsTest PzstdTest
- @./OptionsTest$(EXT)
- @./PzstdTest$(EXT)
-
-roundtrip: RoundTripTest
- @./RoundTripTest$(EXT)
-
-clean:
- @rm -f datagen.o OptionsTest PzstdTest RoundTripTest
diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp
index e7d4b2b3e..b3efe2b7e 100644
--- a/contrib/pzstd/test/OptionsTest.cpp
+++ b/contrib/pzstd/test/OptionsTest.cpp
@@ -180,12 +180,6 @@ TEST(Options, GetOutputFile) {
EXPECT_SUCCESS(options.parse(args.size(), args.data()));
EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0]));
}
- {
- Options options;
- auto args = makeArray("-o-");
- EXPECT_FAILURE(options.parse(args.size(), args.data()));
- EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0]));
- }
{
Options options;
auto args = makeArray("x", "y", "-o", nullOutput);
diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h
new file mode 100644
index 000000000..ed011306b
--- /dev/null
+++ b/contrib/pzstd/utils/ResourcePool.h
@@ -0,0 +1,96 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+
+namespace pzstd {
+
+/**
+ * An unbounded pool of resources.
+ * A `ResourcePool` requires a factory function that takes allocates `T*` and
+ * a free function that frees a `T*`.
+ * Calling `ResourcePool::get()` will give you a new `ResourcePool::UniquePtr`
+ * to a `T`, and when it goes out of scope the resource will be returned to the
+ * pool.
+ * The `ResourcePool` *must* survive longer than any resources it hands out.
+ * Remember that `ResourcePool` hands out mutable `T`s, so make sure to clean
+ * up the resource before or after every use.
+ */
+template
+class ResourcePool {
+ public:
+ class Deleter;
+ using Factory = std::function;
+ using Free = std::function;
+ using UniquePtr = std::unique_ptr;
+
+ private:
+ std::mutex mutex_;
+ Factory factory_;
+ Free free_;
+ std::vector resources_;
+ unsigned inUse_;
+
+ public:
+ /**
+ * Creates a `ResourcePool`.
+ *
+ * @param factory The function to use to create new resources.
+ * @param free The function to use to free resources created by `factory`.
+ */
+ ResourcePool(Factory factory, Free free)
+ : factory_(std::move(factory)), free_(std::move(free)), inUse_(0) {}
+
+ /**
+ * @returns A unique pointer to a resource. The resource is null iff
+ * there are no avaiable resources and `factory()` returns null.
+ */
+ UniquePtr get() {
+ std::lock_guard lock(mutex_);
+ if (!resources_.empty()) {
+ UniquePtr resource{resources_.back(), Deleter{*this}};
+ resources_.pop_back();
+ ++inUse_;
+ return resource;
+ }
+ UniquePtr resource{factory_(), Deleter{*this}};
+ ++inUse_;
+ return resource;
+ }
+
+ ~ResourcePool() noexcept {
+ assert(inUse_ == 0);
+ for (const auto resource : resources_) {
+ free_(resource);
+ }
+ }
+
+ class Deleter {
+ ResourcePool *pool_;
+ public:
+ explicit Deleter(ResourcePool &pool) : pool_(&pool) {}
+
+ void operator() (T *resource) {
+ std::lock_guard lock(pool_->mutex_);
+ // Make sure we don't put null resources into the pool
+ if (resource) {
+ pool_->resources_.push_back(resource);
+ }
+ assert(pool_->inUse_ > 0);
+ --pool_->inUse_;
+ }
+ };
+};
+
+}
diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h
index a1d1fc0b9..99b3ecfa5 100644
--- a/contrib/pzstd/utils/ThreadPool.h
+++ b/contrib/pzstd/utils/ThreadPool.h
@@ -27,7 +27,7 @@ class ThreadPool {
explicit ThreadPool(std::size_t numThreads) {
threads_.reserve(numThreads);
for (std::size_t i = 0; i < numThreads; ++i) {
- threads_.emplace_back([&] {
+ threads_.emplace_back([this] {
std::function task;
while (tasks_.pop(task)) {
task();
diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h
index 538213500..780e5360f 100644
--- a/contrib/pzstd/utils/WorkQueue.h
+++ b/contrib/pzstd/utils/WorkQueue.h
@@ -28,6 +28,7 @@ class WorkQueue {
std::mutex mutex_;
std::condition_variable readerCv_;
std::condition_variable writerCv_;
+ std::condition_variable finishCv_;
std::queue queue_;
bool done_;
@@ -53,12 +54,13 @@ class WorkQueue {
/**
* Push an item onto the work queue. Notify a single thread that work is
* available. If `finish()` has been called, do nothing and return false.
+ * If `push()` returns false, then `item` has not been moved from.
*
* @param item Item to push onto the queue.
* @returns True upon success, false if `finish()` has been called. An
* item was pushed iff `push()` returns true.
*/
- bool push(T item) {
+ bool push(T&& item) {
{
std::unique_lock lock(mutex_);
while (full() && !done_) {
@@ -124,19 +126,14 @@ class WorkQueue {
}
readerCv_.notify_all();
writerCv_.notify_all();
+ finishCv_.notify_all();
}
/// Blocks until `finish()` has been called (but the queue may not be empty).
void waitUntilFinished() {
std::unique_lock lock(mutex_);
while (!done_) {
- readerCv_.wait(lock);
- // If we were woken by a push, we need to wake a thread waiting on pop().
- if (!done_) {
- lock.unlock();
- readerCv_.notify_one();
- lock.lock();
- }
+ finishCv_.wait(lock);
}
}
};
diff --git a/contrib/pzstd/utils/test/Makefile b/contrib/pzstd/utils/test/Makefile
deleted file mode 100644
index b9ea73e32..000000000
--- a/contrib/pzstd/utils/test/Makefile
+++ /dev/null
@@ -1,42 +0,0 @@
-# ##########################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ##########################################################################
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
-else
-EXT =
-endif
-
-PZSTDDIR = ../..
-
-# Set GTEST_INC and GTEST_LIB to work with your install of gtest
-GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
-GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
-
-CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB)
-CXXFLAGS ?= -O3
-CXXFLAGS += -std=c++11
-CXXFLAGS += $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
-
-%: %.cpp
- $(CXX) $(FLAGS) $^ -o $@$(EXT) -lgtest -lgtest_main -lpthread
-
-.PHONY: test clean
-
-test: BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest
- @./BufferTest$(EXT)
- @./RangeTest$(EXT)
- @./ScopeGuardTest$(EXT)
- @./ThreadPoolTest$(EXT)
- @./WorkQueueTest$(EXT)
-
-clean:
- @rm -f BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest
diff --git a/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
new file mode 100644
index 000000000..a6a86b345
--- /dev/null
+++ b/contrib/pzstd/utils/test/ResourcePoolTest.cpp
@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "utils/ResourcePool.h"
+
+#include
+#include
+#include
+
+using namespace pzstd;
+
+TEST(ResourcePool, FullTest) {
+ unsigned numCreated = 0;
+ unsigned numDeleted = 0;
+ {
+ ResourcePool pool(
+ [&numCreated] { ++numCreated; return new int{5}; },
+ [&numDeleted](int *x) { ++numDeleted; delete x; });
+
+ {
+ auto i = pool.get();
+ EXPECT_EQ(5, *i);
+ *i = 6;
+ }
+ {
+ auto i = pool.get();
+ EXPECT_EQ(6, *i);
+ auto j = pool.get();
+ EXPECT_EQ(5, *j);
+ *j = 7;
+ }
+ {
+ auto i = pool.get();
+ EXPECT_EQ(6, *i);
+ auto j = pool.get();
+ EXPECT_EQ(7, *j);
+ }
+ }
+ EXPECT_EQ(2, numCreated);
+ EXPECT_EQ(numCreated, numDeleted);
+}
+
+TEST(ResourcePool, ThreadSafe) {
+ std::atomic numCreated{0};
+ std::atomic numDeleted{0};
+ {
+ ResourcePool pool(
+ [&numCreated] { ++numCreated; return new int{0}; },
+ [&numDeleted](int *x) { ++numDeleted; delete x; });
+ auto push = [&pool] {
+ for (int i = 0; i < 100; ++i) {
+ auto x = pool.get();
+ ++*x;
+ }
+ };
+ std::thread t1{push};
+ std::thread t2{push};
+ t1.join();
+ t2.join();
+
+ auto x = pool.get();
+ auto y = pool.get();
+ EXPECT_EQ(200, *x + *y);
+ }
+ EXPECT_GE(2, numCreated);
+ EXPECT_EQ(numCreated, numDeleted);
+}
diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp
index ebf375a84..7f58ccb3f 100644
--- a/contrib/pzstd/utils/test/WorkQueueTest.cpp
+++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp
@@ -10,6 +10,7 @@
#include "utils/WorkQueue.h"
#include
+#include
#include
#include
#include
@@ -64,7 +65,7 @@ TEST(WorkQueue, SPSC) {
const int max = 100;
for (int i = 0; i < 10; ++i) {
- queue.push(i);
+ queue.push(int{i});
}
std::thread thread([ &queue, max ] {
@@ -80,7 +81,7 @@ TEST(WorkQueue, SPSC) {
std::this_thread::yield();
for (int i = 10; i < max; ++i) {
- queue.push(i);
+ queue.push(int{i});
}
queue.finish();
@@ -97,7 +98,7 @@ TEST(WorkQueue, SPMC) {
}
for (int i = 0; i < 50; ++i) {
- queue.push(i);
+ queue.push(int{i});
}
queue.finish();
@@ -126,7 +127,7 @@ TEST(WorkQueue, MPMC) {
pusherThreads.emplace_back(
[ &queue, min, max ] {
for (int i = min; i < max; ++i) {
- queue.push(i);
+ queue.push(int{i});
}
});
}
@@ -212,7 +213,7 @@ TEST(WorkQueue, BoundedSizeMPMC) {
pusherThreads.emplace_back(
[ &queue, min, max ] {
for (int i = min; i < max; ++i) {
- queue.push(i);
+ queue.push(int{i});
}
});
}
@@ -231,6 +232,18 @@ TEST(WorkQueue, BoundedSizeMPMC) {
}
}
+TEST(WorkQueue, FailedPush) {
+ WorkQueue> queue;
+ std::unique_ptr x(new int{5});
+ EXPECT_TRUE(queue.push(std::move(x)));
+ EXPECT_EQ(nullptr, x);
+ queue.finish();
+ x.reset(new int{6});
+ EXPECT_FALSE(queue.push(std::move(x)));
+ EXPECT_NE(nullptr, x);
+ EXPECT_EQ(6, *x);
+}
+
TEST(BufferWorkQueue, SizeCalculatedCorrectly) {
{
BufferWorkQueue queue;
diff --git a/images/Cspeed4.png b/doc/images/Cspeed4.png
similarity index 100%
rename from images/Cspeed4.png
rename to doc/images/Cspeed4.png
diff --git a/images/DCspeed5.png b/doc/images/DCspeed5.png
similarity index 100%
rename from images/DCspeed5.png
rename to doc/images/DCspeed5.png
diff --git a/images/Dspeed4.png b/doc/images/Dspeed4.png
similarity index 100%
rename from images/Dspeed4.png
rename to doc/images/Dspeed4.png
diff --git a/images/smallData.png b/doc/images/smallData.png
similarity index 100%
rename from images/smallData.png
rename to doc/images/smallData.png
diff --git a/zstd_compression_format.md b/doc/zstd_compression_format.md
similarity index 100%
rename from zstd_compression_format.md
rename to doc/zstd_compression_format.md
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
new file mode 100644
index 000000000..2470a4555
--- /dev/null
+++ b/doc/zstd_manual.html
@@ -0,0 +1,531 @@
+
+
+
+zstd 1.1.1 Manual
+
+
+zstd 1.1.1 Manual
+
+Contents
+
+- Introduction
+- Version
+- Simple API
+- Explicit memory management
+- Simple dictionary API
+- Fast dictionary API
+- Streaming
+- Streaming compression - HowTo
+- Streaming decompression - HowTo
+- START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
+- Advanced types
+- Advanced compression functions
+- Advanced decompression functions
+- Advanced streaming functions
+- Buffer-less and synchronous inner streaming functions
+- Buffer-less streaming compression (synchronous mode)
+- Buffer-less streaming decompression (synchronous mode)
+- Block functions
+
+
+Introduction
+ zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+ at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
+ decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+ Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+ Compression can be done in:
+ - a single step (described as Simple API)
+ - a single step, reusing a context (described as Explicit memory management)
+ - unbounded multiple steps (described as Streaming compression)
+ The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
+ - a single step (described as Simple dictionary API)
+ - a single step, reusing a dictionary (described as Fast dictionary API)
+
+ Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+ These APIs shall never be used with a dynamic library.
+ They are not "stable", their definition may change in the future. Only static linking is allowed.
+
+
+Version
+
+unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */
+
+Simple API
+
+size_t ZSTD_compress( void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+ Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
+ Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ @return : compressed size written into `dst` (<= `dstCapacity),
+ or an error code if it fails (which can be tested using ZSTD_isError())
+
+
+size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
+ `compressedSize` : must be the _exact_ size of a single compressed frame.
+ `dstCapacity` is an upper bound of originalSize.
+ If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
+ @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ or an errorCode if it fails (which can be tested using ZSTD_isError())
+
+
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+ 'src' is the start of a zstd compressed frame.
+ @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
+ note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ When `return==0`, data to decompress could be any size.
+ In which case, it's necessary to use streaming mode to decompress data.
+ Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+ (For example, data may be necessarily cut into blocks <= 16 KB).
+ note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ note 3 : decompressed size can be very large (64-bits value),
+ potentially larger than what local system can handle as a single memory segment.
+ In which case, it's necessary to use streaming mode to decompress data.
+ note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ Always ensure result fits within application's authorized limits.
+ Each application can set its own limits.
+ note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more.
+
+
+Helper functions
int ZSTD_maxCLevel(void); /*!< maximum compression level available */
+size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
+unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
+
+Explicit memory management
+
+size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
+ Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx())
+
+
+Decompression context
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTD_DCtx* ZSTD_createDCtx(void);
+size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx())
+
+
+Simple dictionary API
+
+size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ int compressionLevel);
+ Compression using a predefined Dictionary (see dictBuilder/zdict.h).
+ Note : This function load the dictionary, resulting in significant startup delay.
+
+
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
+ Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
+ Dictionary must be identical to the one used during compression.
+ Note : This function load the dictionary, resulting in significant startup delay
+
+
+Fast dictionary API
+
+ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+ When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
+ `dict` can be released after ZSTD_CDict creation
+
+
+size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
+ Function frees memory allocated by ZSTD_createCDict()
+
+
+size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict);
+ Compression using a digested Dictionary.
+ Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+ Note that compression level is decided during dictionary creation
+
+
+ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+ Create a digested dictionary, ready to start decompression operation without startup delay.
+ `dict` can be released after creation
+
+
+size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
+ Function frees memory allocated with ZSTD_createDDict()
+
+
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_DDict* ddict);
+ Decompression using a digested Dictionary
+ Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times.
+
+
+Streaming
+
+typedef struct ZSTD_inBuffer_s {
+ const void* src; /**< start of input buffer */
+ size_t size; /**< size of input buffer */
+ size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+ void* dst; /**< start of output buffer */
+ size_t size; /**< size of output buffer */
+ size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+Streaming compression - HowTo
+ A ZSTD_CStream object is required to track streaming operation.
+ Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+ ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+ It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
+ since it will play nicer with system's memory, by re-using already allocated memory.
+ Use one separate ZSTD_CStream per thread for parallel execution.
+
+ Start a new compression by initializing ZSTD_CStream.
+ Use ZSTD_initCStream() to start a new compression operation.
+ Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+
+ Use ZSTD_compressStream() repetitively to consume input stream.
+ The function will automatically update both `pos` fields.
+ Note that it may not consume the entire input, in which case `pos < size`,
+ and it's up to the caller to present again remaining data.
+ @return : a size hint, preferred nb of bytes to use as input for next function call
+ (it's just a hint, to help latency a little, any other value will work fine)
+ (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
+ or an error code, which can be tested using ZSTD_isError().
+
+ At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+ `output->pos` will be updated.
+ Note some content might still be left within internal buffer if `output->size` is too small.
+ @return : nb of bytes still present within internal buffer (0 if it's empty)
+ or an error code, which can be tested using ZSTD_isError().
+
+ ZSTD_endStream() instructs to finish a frame.
+ It will perform a flush and write frame epilogue.
+ The epilogue is required for decoders to consider a frame completed.
+ Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+ In which case, call again ZSTD_endStream() to complete the flush.
+ @return : nb of bytes still present within internal buffer (0 if it's empty)
+ or an error code, which can be tested using ZSTD_isError().
+
+
+
+
+Streaming compression functions
typedef struct ZSTD_CStream_s ZSTD_CStream;
+ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
+
+size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+
+Streaming decompression - HowTo
+ A ZSTD_DStream object is required to track streaming operations.
+ Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+ ZSTD_DStream objects can be re-used multiple times.
+
+ Use ZSTD_initDStream() to start a new decompression operation,
+ or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
+ @return : recommended first input size
+
+ Use ZSTD_decompressStream() repetitively to consume your input.
+ The function will update both `pos` fields.
+ If `input.pos < input.size`, some input has not been consumed.
+ It's up to the caller to present again remaining data.
+ If `output.pos < output.size`, decoder has flushed everything it could.
+ @return : 0 when a frame is completely decoded and fully flushed,
+ an error code, which can be tested using ZSTD_isError(),
+ any other value > 0, which means there is still some work to do to complete the frame.
+ The return value is a suggested next input size (just an hint, to help latency).
+
+
+
+Streaming decompression functions
typedef struct ZSTD_DStream_s ZSTD_DStream;
+ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
+
+size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+
+START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
The definitions in this section are considered experimental.
+ They should never be used with a dynamic library, as they may change in the future.
+ They are provided for advanced usages.
+ Use them only in association with static linking.
+
+
+
+Advanced types
+
+typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
+
+typedef struct {
+ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
+ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+ unsigned hashLog; /**< dispatch table : larger == faster, more memory */
+ unsigned searchLog; /**< nb of searches : larger == more compression, slower */
+ unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
+ unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+ ZSTD_strategy strategy;
+} ZSTD_compressionParameters;
+
+typedef struct {
+ unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
+ unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
+ unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
+} ZSTD_frameParameters;
+
+typedef struct {
+ ZSTD_compressionParameters cParams;
+ ZSTD_frameParameters fParams;
+} ZSTD_parameters;
+
+Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+
+Advanced compression functions
+
+size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+ Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
+ `frameContentSize` is an optional parameter, provide `0` if unknown
+
+
+ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+ Create a ZSTD compression context using external alloc and free functions
+
+
+size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ Gives the amount of memory used by a given ZSTD_CCtx
+
+
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+ ZSTD_parameters params, ZSTD_customMem customMem);
+ Create a ZSTD_CDict using external alloc and free, and customized compression parameters
+
+
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ Gives the amount of memory used by a given ZSTD_sizeof_CDict
+
+
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+ same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
+ All fields of `ZSTD_frameParameters` are set to default (0)
+
+
+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+ @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
+ `srcSize` value is optional, select 0 if not known
+
+
+size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+ Ensure param values remain within authorized range
+
+
+ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+ optimize params for a given `srcSize` and `dictSize`.
+ both values are optional, select `0` if unknown.
+
+
+size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params);
+ Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter
+
+
+Advanced decompression functions
+
+size_t ZSTD_estimateDCtxSize(void);
+ Gives the potential amount of memory allocated to create a ZSTD_DCtx
+
+
+ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+ Create a ZSTD decompression context using external alloc and free functions
+
+
+size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ Gives the amount of memory used by a given ZSTD_DCtx
+
+
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+ Gives the amount of memory used by a given ZSTD_DDict
+
+
+Advanced streaming functions
+
+Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+
+Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
+size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+
+Buffer-less and synchronous inner streaming functions
+ This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+ But it's also a complex one, with many restrictions (documented below).
+ Prefer using normal streaming API for an easier experience
+
+
+
+Buffer-less streaming compression (synchronous mode)
+ A ZSTD_CCtx object is required to track streaming operations.
+ Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
+ ZSTD_CCtx object can be re-used multiple times within successive compression operations.
+
+ Start by initializing a context.
+ Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
+ or ZSTD_compressBegin_advanced(), for finer parameter control.
+ It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
+
+ Then, consume your input using ZSTD_compressContinue().
+ There are some important considerations to keep in mind when using this advanced function :
+ - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
+ - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks.
+ - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+ Worst case evaluation is provided by ZSTD_compressBound().
+ ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+ - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+ It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+ - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+ In which case, it will "discard" the relevant memory section from its history.
+
+ Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
+ It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
+ Without last block mark, frames will be considered unfinished (broken) by decoders.
+
+ You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+
+
+Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+Buffer-less streaming decompression (synchronous mode)
+ A ZSTD_DCtx object is required to track streaming operations.
+ Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+ A ZSTD_DCtx object can be re-used multiple times.
+
+ First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
+ It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
+ such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
+ and the dictionary ID used.
+ (Note : content size is optional, it may not be present. 0 means : content size unknown).
+ Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
+ As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
+ Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
+ Frame parameters are extracted from the beginning of the compressed frame.
+ Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
+ @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
+ >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+ errorCode, which can be tested using ZSTD_isError().
+
+ Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
+ Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
+
+ Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
+ ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+ ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
+
+ @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+ It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some metadata item.
+ It can also be an error code, which can be tested with ZSTD_isError().
+
+ ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
+ They should preferably be located contiguously, prior to current block.
+ Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters.
+ ZSTD_decompressContinue() is very sensitive to contiguity,
+ if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
+ or that previous contiguous segment is large enough to properly handle maximum back-reference.
+
+ A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+ Context can then be reset to start a new decompression.
+
+ Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
+ This information is not required to properly decode a frame.
+
+ == Special case : skippable frames ==
+
+ Skippable frames allow integration of user-defined data into a flow of concatenated frames.
+ Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
+ a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
+ b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
+ c) Frame Content - any content (User Data) of length equal to Frame Size
+ For skippable frames ZSTD_decompressContinue() always returns 0.
+ For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+ It also returns Frame Size as fparamsPtr->frameContentSize.
+
+
+typedef struct {
+ unsigned long long frameContentSize;
+ unsigned windowSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+} ZSTD_frameParams;
+
+Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
+size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+
+Block functions
+ Block functions produce and decode raw zstd blocks, without frame metadata.
+ Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+ User will have to take in charge required information to regenerate data, such as compressed and content sizes.
+
+ A few rules to respect :
+ - Compressing and decompressing require a context structure
+ + Use ZSTD_createCCtx() and ZSTD_createDCtx()
+ - It is necessary to init context before starting
+ + compression : ZSTD_compressBegin()
+ + decompression : ZSTD_decompressBegin()
+ + variants _usingDict() are also allowed
+ + copyCCtx() and copyDCtx() work too
+ - Block size is limited, it must be <= ZSTD_getBlockSizeMax()
+ + If you need to compress more, cut data into multiple blocks
+ + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
+ - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
+ In which case, nothing is produced into `dst`.
+ + User must test for such outcome and deal directly with uncompressed data
+ + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
+ + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
+ Use ZSTD_insertBlock() in such a case.
+
+
+Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
+
+
+
diff --git a/examples/.gitignore b/examples/.gitignore
index 1c98e1884..0711813d3 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -5,6 +5,7 @@ dictionary_compression
dictionary_decompression
streaming_compression
streaming_decompression
+multiple_streaming_compression
#test artefact
tmp*
diff --git a/examples/Makefile b/examples/Makefile
index 54602dfe0..741022869 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,26 +1,11 @@
-# ##########################################################################
-# ZSTD educational examples - Makefile
-# Copyright (C) Yann Collet 2016
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
#
-# GPL v2 License
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# You can contact the author at :
-# - zstd homepage : http://www.zstd.net/
-# ##########################################################################
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
# This Makefile presumes libzstd is installed, using `sudo make install`
@@ -32,7 +17,8 @@ default: all
all: simple_compression simple_decompression \
dictionary_compression dictionary_decompression \
- streaming_compression streaming_decompression
+ streaming_compression streaming_decompression \
+ multiple_streaming_compression
simple_compression : simple_compression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -49,6 +35,9 @@ dictionary_decompression : dictionary_decompression.c
streaming_compression : streaming_compression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+multiple_streaming_compression : multiple_streaming_compression.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
streaming_decompression : streaming_decompression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -56,7 +45,8 @@ clean:
@rm -f core *.o tmp* result* *.zst \
simple_compression simple_decompression \
dictionary_compression dictionary_decompression \
- streaming_compression streaming_decompression
+ streaming_compression streaming_decompression \
+ multiple_streaming_compression
@echo Cleaning completed
test: all
@@ -69,7 +59,10 @@ test: all
@echo starting streaming compression
./streaming_compression tmp
./streaming_decompression tmp.zst > /dev/null
+ @echo starting multiple streaming compression
+ ./multiple_streaming_compression *.c
@echo starting dictionary compression
./dictionary_compression tmp2 tmp README.md
./dictionary_decompression tmp2.zst tmp.zst README.md
+ $(RM) tmp* *.zst
@echo tests completed
diff --git a/examples/README.md b/examples/README.md
index ba132f6c3..8a40443ea 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -15,6 +15,11 @@ Zstandard library : usage examples
Compress a single file.
Introduces usage of : `ZSTD_compressStream()`
+- [Multiple Streaming compression](multiple_streaming_compression.c) :
+ Compress multiple files in a single command line.
+ Introduces memory usage preservation technique,
+ reducing impact of malloc()/free() and memset() by re-using existing resources.
+
- [Streaming decompression](streaming_decompression.c) :
Decompress a single file compressed by zstd.
Compatible with both simple and streaming compression.
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
new file mode 100644
index 000000000..61699104c
--- /dev/null
+++ b/examples/multiple_streaming_compression.c
@@ -0,0 +1,163 @@
+/**
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
+
+
+/* The objective of this example is to show of to compress multiple successive files
+* while preserving memory management.
+* All structures and buffers will be created only once,
+* and shared across all compression operations */
+
+#include // malloc, exit
+#include // fprintf, perror, feof
+#include // strerror
+#include // errno
+#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being
+#include // presumes zstd library is installed
+
+
+static void* malloc_orDie(size_t size)
+{
+ void* const buff = malloc(size);
+ if (buff) return buff;
+ /* error */
+ perror("malloc:");
+ exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+ FILE* const inFile = fopen(filename, instruction);
+ if (inFile) return inFile;
+ /* error */
+ perror(filename);
+ exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+ size_t const readSize = fread(buffer, 1, sizeToRead, file);
+ if (readSize == sizeToRead) return readSize; /* good */
+ if (feof(file)) return readSize; /* good, reached end of file */
+ /* error */
+ perror("fread");
+ exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+ size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+ if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
+ /* error */
+ perror("fwrite");
+ exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+ if (!fclose(file)) return 0;
+ /* error */
+ perror("fclose");
+ exit(6);
+}
+
+
+typedef struct {
+ void* buffIn;
+ void* buffOut;
+ size_t buffInSize;
+ size_t buffOutSize;
+ ZSTD_CStream* cstream;
+} resources ;
+
+static resources createResources_orDie()
+{
+ resources ress;
+ ress.buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */
+ ress.buffOutSize= ZSTD_CStreamOutSize(); /* can always flush a full block */
+ ress.buffIn = malloc_orDie(ress.buffInSize);
+ ress.buffOut= malloc_orDie(ress.buffOutSize);
+ ress.cstream = ZSTD_createCStream();
+ if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
+ return ress;
+}
+
+static void freeResources(resources ress)
+{
+ ZSTD_freeCStream(ress.cstream);
+ free(ress.buffIn);
+ free(ress.buffOut);
+}
+
+
+static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel)
+{
+ FILE* const fin = fopen_orDie(fname, "rb");
+ FILE* const fout = fopen_orDie(outName, "wb");
+
+ size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel);
+ if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+
+ size_t read, toRead = ress.buffInSize;
+ while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
+ ZSTD_inBuffer input = { ress.buffIn, read, 0 };
+ while (input.pos < input.size) {
+ ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+ toRead = ZSTD_compressStream(ress.cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
+ if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+ if (toRead > ress.buffInSize) toRead = ress.buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */
+ fwrite_orDie(ress.buffOut, output.pos, fout);
+ }
+ }
+
+ ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+ size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output); /* close frame */
+ if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
+ fwrite_orDie(ress.buffOut, output.pos, fout);
+
+ fclose_orDie(fout);
+ fclose_orDie(fin);
+}
+
+
+int main(int argc, const char** argv)
+{
+ const char* const exeName = argv[0];
+
+ if (argc<2) {
+ printf("wrong arguments\n");
+ printf("usage:\n");
+ printf("%s FILE(s)\n", exeName);
+ return 1;
+ }
+
+ resources const ress = createResources_orDie();
+ void* ofnBuffer = NULL;
+ size_t ofnbSize = 0;
+
+ int argNb;
+ for (argNb = 1; argNb < argc; argNb++) {
+ const char* const ifn = argv[argNb];
+ size_t const ifnSize = strlen(ifn);
+ size_t const ofnSize = ifnSize + 5;
+ if (ofnbSize <= ofnSize) {
+ ofnbSize = ofnSize + 16;
+ free(ofnBuffer);
+ ofnBuffer = malloc_orDie(ofnbSize);
+ }
+ memset(ofnBuffer, 0, ofnSize);
+ strcat(ofnBuffer, ifn);
+ strcat(ofnBuffer, ".zst");
+ compressFile_orDie(ress, ifn, ofnBuffer, 7);
+ }
+
+ freeResources(ress);
+ /* success */
+ printf("compressed %i files \n", argc-1);
+
+ return 0;
+}
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index 907ee3387..62a881fcb 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -65,7 +65,7 @@ static void decompress(const char* fname)
void* const cBuff = loadFile_X(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
if (rSize==0) {
- printf("%s : original size unknown \n", fname);
+ printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
exit(5);
}
void* const rBuff = malloc_X((size_t)rSize);
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 1ba1a3d81..400aa673d 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -71,9 +71,12 @@ static void decompressFile_orDie(const char* fname)
ZSTD_DStream* const dstream = ZSTD_createDStream();
if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
+
+ /* In more complex scenarios, a file may consist of multiple appended frames (ex : pzstd).
+ * The following example decompresses only the first frame.
+ * It is compatible with other provided streaming examples */
size_t const initResult = ZSTD_initDStream(dstream);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
-
size_t read, toRead = initResult;
while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
diff --git a/lib/Makefile b/lib/Makefile
index 4fb8ed9d2..1117b491c 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -97,19 +97,21 @@ install: libzstd libzstd.pc
@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
- @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
+ @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+ @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed
uninstall:
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
- $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
- $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
- $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h
- $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
+ @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd libraries successfully uninstalled
endif
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index acd966999..18bba0e87 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -168,9 +168,11 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -198,6 +200,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
new file mode 100644
index 000000000..a0fa1724a
--- /dev/null
+++ b/lib/common/error_private.c
@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/* The purpose of this file is to have a single list of error strings embedded in binary */
+
+#include "error_private.h"
+
+const char* ERR_getErrorString(ERR_enum code)
+{
+ static const char* const notErrorCode = "Unspecified error code";
+ switch( code )
+ {
+ case PREFIX(no_error): return "No error detected";
+ case PREFIX(GENERIC): return "Error (generic)";
+ case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+ case PREFIX(version_unsupported): return "Version not supported";
+ case PREFIX(parameter_unknown): return "Unknown parameter type";
+ case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
+ case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
+ case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
+ case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
+ case PREFIX(init_missing): return "Context should be init first";
+ case PREFIX(memory_allocation): return "Allocation error : not enough memory";
+ case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
+ case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+ case PREFIX(srcSize_wrong): return "Src size incorrect";
+ case PREFIX(corruption_detected): return "Corrupted block detected";
+ case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+ case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
+ case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
+ case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
+ case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
+ case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+ case PREFIX(maxCode):
+ default: return notErrorCode;
+ }
+}
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index d27e15af8..1bc2e4954 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -21,7 +21,7 @@ extern "C" {
* Dependencies
******************************************/
#include /* size_t */
-#include "error_public.h" /* enum list */
+#include "zstd_errors.h" /* enum list */
/* ****************************************
@@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu
* Error Strings
******************************************/
-ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
-{
- static const char* notErrorCode = "Unspecified error code";
- switch( code )
- {
- case PREFIX(no_error): return "No error detected";
- case PREFIX(GENERIC): return "Error (generic)";
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
- case PREFIX(version_unsupported): return "Version not supported";
- case PREFIX(parameter_unknown): return "Unknown parameter type";
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
- case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
- case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
- case PREFIX(init_missing): return "Context should be init first";
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
- case PREFIX(srcSize_wrong): return "Src size incorrect";
- case PREFIX(corruption_detected): return "Corrupted block detected";
- case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
- case PREFIX(dictionary_wrong): return "Dictionary mismatch";
- case PREFIX(maxCode):
- default: return notErrorCode;
- }
-}
+const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
diff --git a/lib/common/fse.h b/lib/common/fse.h
index 720d54b11..cecb1aef7 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
BIT_flushBits(bitC);
}
+
/* ====== Decompression ====== */
typedef struct {
@@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
-#define FSE_MAX_MEMORY_USAGE 14
-#define FSE_DEFAULT_MEMORY_USAGE 13
+#ifndef FSE_MAX_MEMORY_USAGE
+# define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+# define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
-#define FSE_MAX_SYMBOL_VALUE 255
-
+#ifndef FSE_MAX_SYMBOL_VALUE
+# define FSE_MAX_SYMBOL_VALUE 255
+#endif
/* **************************************************************
* template functions type & suffix
diff --git a/lib/common/error_public.h b/lib/common/zstd_errors.h
similarity index 91%
rename from lib/common/error_public.h
rename to lib/common/zstd_errors.h
index d46abd2c7..50dc4f720 100644
--- a/lib/common/error_public.h
+++ b/lib/common/zstd_errors.h
@@ -7,8 +7,8 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
#if defined (__cplusplus)
extern "C" {
@@ -29,6 +29,7 @@ typedef enum {
ZSTD_error_parameter_unknown,
ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits,
+ ZSTD_error_frameParameter_windowTooLarge,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
@@ -56,4 +57,4 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
}
#endif
-#endif /* ERROR_PUBLIC_H_MODULE */
+#endif /* ZSTD_ERRORS_H_398273423 */
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f40e00aab..d889c840f 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -31,6 +31,16 @@
# endif /* __STDC_VERSION__ */
#endif
+#ifdef _MSC_VER
+# define FORCE_NOINLINE static __declspec(noinline)
+#else
+# ifdef __GNUC__
+# define FORCE_NOINLINE static __attribute__((__noinline__))
+# else
+# define FORCE_NOINLINE static
+# endif
+#endif
+
/*-*************************************
* Dependencies
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index b7d3d77a2..78784aa36 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
} }
/* fill val */
- { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
- U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
+ { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
+ U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U32 n; for (n=0; n0; n--) {
- valPerRank[n] = min; /* get starting value within each rank */
+ U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
+ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 94f4b5a25..e7f7d9911 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -122,6 +122,11 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface *
return &(ctx->seqStore);
}
+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
+{
+ return cctx->params;
+}
+
/** ZSTD_checkParams() :
ensure param values remain within authorized range.
@@ -137,7 +142,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1;
CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); }
CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
- if ((U32)(cParams.strategy) > (U32)ZSTD_btopt) return ERROR(compressionParameter_unsupported);
+ if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported);
return 0;
}
@@ -160,7 +165,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
- { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt);
+ { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt) | (cPar.strategy == ZSTD_btopt2);
U32 const maxChainLog = cPar.windowLog+btPlus;
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
@@ -186,7 +191,7 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<workSpaceSize < neededSpace) {
ZSTD_free(zc->workSpace, zc->customMem);
zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
@@ -276,7 +281,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->frameContentSize = frameContentSize;
{ int i; for (i=0; irep[i] = repStartValue[i]; }
- if (params.cParams.strategy == ZSTD_btopt) {
+ if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
zc->seqStore.litFreq = (U32*)ptr;
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
@@ -377,7 +382,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
* Block entropic compression
*********************************************************/
-/* See zstd_compression_format.md for detailed format description */
+/* See doc/zstd_compression_format.md for detailed format description */
size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
@@ -1458,7 +1463,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
- const BYTE* match = base + matchIndex;
+ const BYTE* match;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
@@ -2170,7 +2175,17 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
#ifdef ZSTD_OPT_H_91842398743
- ZSTD_compressBlock_opt_generic(ctx, src, srcSize);
+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
+#else
+ (void)ctx; (void)src; (void)srcSize;
+ return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
#else
(void)ctx; (void)src; (void)srcSize;
return;
@@ -2180,7 +2195,17 @@ static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t src
static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
#ifdef ZSTD_OPT_H_91842398743
- ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize);
+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
+#else
+ (void)ctx; (void)src; (void)srcSize;
+ return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
#else
(void)ctx; (void)src; (void)srcSize;
return;
@@ -2192,9 +2217,9 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{
- static const ZSTD_blockCompressor blockCompressor[2][7] = {
- { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt },
- { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict }
+ static const ZSTD_blockCompressor blockCompressor[2][8] = {
+ { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 },
+ { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict }
};
return blockCompressor[extDict][(U32)strat];
@@ -2235,7 +2260,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
BYTE* op = ostart;
U32 const maxDist = 1 << cctx->params.cParams.windowLog;
- if (cctx->params.fParams.checksumFlag)
+ if (cctx->params.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
@@ -2247,7 +2272,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
/* preemptive overflow correction */
if (cctx->lowLimit > (1<<30)) {
- U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt);
+ U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt) | (cctx->params.cParams.strategy == ZSTD_btopt2);
U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */);
U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */
@@ -2436,6 +2461,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
case ZSTD_btlazy2:
case ZSTD_btopt:
+ case ZSTD_btopt2:
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
break;
@@ -2448,14 +2474,28 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
}
+/* Dictionaries that assign zero probability to symbols that show up causes problems
+ when FSE encoding. Refuse dictionaries that assign zero probability to symbols
+ that we may encounter during compression.
+ NOTE: This behavior is not standard and could be improved in the future. */
+static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) {
+ U32 s;
+ if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted);
+ for (s = 0; s <= maxSymbolValue; ++s) {
+ if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted);
+ }
+ return 0;
+}
+
+
/* Dictionary format :
- Magic == ZSTD_DICT_MAGIC (4 bytes)
- HUF_writeCTable(256)
- FSE_writeNCount(off)
- FSE_writeNCount(ml)
- FSE_writeNCount(ll)
- RepOffsets
- Dictionary content
+ Magic == ZSTD_DICT_MAGIC (4 bytes)
+ HUF_writeCTable(256)
+ FSE_writeNCount(off)
+ FSE_writeNCount(ml)
+ FSE_writeNCount(ll)
+ RepOffsets
+ Dictionary content
*/
/*! ZSTD_loadDictEntropyStats() :
@return : size read from dictionary
@@ -2464,32 +2504,41 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
{
const BYTE* dictPtr = (const BYTE*)dict;
const BYTE* const dictEnd = dictPtr + dictSize;
+ short offcodeNCount[MaxOff+1];
+ unsigned offcodeMaxValue = MaxOff;
{ size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
dictPtr += hufHeaderSize;
}
- { short offcodeNCount[MaxOff+1];
- unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
+ { unsigned offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
+ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
+ /* Every match length code must have non-zero probability */
+ CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize;
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
+ /* Every literal length code must have non-zero probability */
+ CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize;
}
@@ -2500,6 +2549,16 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
+ { U32 offcodeMax = MaxOff;
+ if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) {
+ U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */
+ /* Calculate minimum offset code required to represent maxOffset */
+ offcodeMax = ZSTD_highbit32(maxOffset);
+ }
+ /* Every possible supported offset <= dictContentSize + 128 KB must be representable */
+ CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
+ }
+
cctx->flagStaticTables = 1;
return dictPtr - (const BYTE*)dict;
}
@@ -2688,7 +2747,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
return NULL;
}
- memcpy(dictContent, dict, dictSize);
+ if (dictSize) {
+ memcpy(dictContent, dict, dictSize);
+ }
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
@@ -2723,6 +2784,10 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
}
}
+static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
+ return ZSTD_getParamsFromCCtx(cdict->refContext);
+}
+
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
@@ -2759,7 +2824,8 @@ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s {
ZSTD_CCtx* cctx;
- ZSTD_CDict* cdict;
+ ZSTD_CDict* cdictLocal;
+ const ZSTD_CDict* cdict;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
@@ -2773,6 +2839,7 @@ struct ZSTD_CStream_s {
ZSTD_cStreamStage stage;
U32 checksum;
U32 frameEnded;
+ ZSTD_parameters params;
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CStream within "zstd.h" */
@@ -2802,7 +2869,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
if (zcs==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->cctx);
- ZSTD_freeCDict(zcs->cdict);
+ ZSTD_freeCDict(zcs->cdictLocal);
ZSTD_free(zcs->inBuff, cMem);
ZSTD_free(zcs->outBuff, cMem);
ZSTD_free(zcs, cMem);
@@ -2818,7 +2885,10 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
- CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize));
+ if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */
+
+ if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
+ else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
@@ -2850,15 +2920,28 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
- ZSTD_freeCDict(zcs->cdict);
- zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
- if (zcs->cdict == NULL) return ERROR(memory_allocation);
+ if (dict) {
+ ZSTD_freeCDict(zcs->cdictLocal);
+ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
+ if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+ zcs->cdict = zcs->cdictLocal;
+ } else zcs->cdict = NULL;
zcs->checksum = params.fParams.checksumFlag > 0;
+ zcs->params = params;
return ZSTD_resetCStream(zcs, pledgedSrcSize);
}
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+ ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
+ size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
+ zcs->cdict = cdict;
+ return initError;
+}
+
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
@@ -2873,7 +2956,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{
if (zcs==NULL) return 0; /* support sizeof on NULL */
- return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize;
+ return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
}
/*====== Compression ======*/
@@ -3064,9 +3147,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
{ 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */
- { 25, 25, 23, 7, 3, 64, ZSTD_btopt }, /* level 20 */
- { 26, 26, 23, 7, 3,256, ZSTD_btopt }, /* level 21 */
- { 27, 27, 25, 9, 3,512, ZSTD_btopt }, /* level 22 */
+ { 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */
+ { 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */
+ { 27, 27, 25, 9, 3,512, ZSTD_btopt2 }, /* level 22 */
},
{ /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */
@@ -3090,9 +3173,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/
- { 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/
- { 18, 19, 18, 12, 3,512, ZSTD_btopt }, /* level 21.*/
- { 18, 19, 18, 13, 3,512, ZSTD_btopt }, /* level 22.*/
+ { 18, 19, 18, 11, 3,512, ZSTD_btopt2 }, /* level 20.*/
+ { 18, 19, 18, 12, 3,512, ZSTD_btopt2 }, /* level 21.*/
+ { 18, 19, 18, 13, 3,512, ZSTD_btopt2 }, /* level 22.*/
},
{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */
@@ -3116,9 +3199,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/
- { 17, 18, 17, 9, 3,256, ZSTD_btopt }, /* level 20.*/
- { 17, 18, 17, 10, 3,256, ZSTD_btopt }, /* level 21.*/
- { 17, 18, 17, 11, 3,512, ZSTD_btopt }, /* level 22.*/
+ { 17, 18, 17, 9, 3,256, ZSTD_btopt2 }, /* level 20.*/
+ { 17, 18, 17, 10, 3,256, ZSTD_btopt2 }, /* level 21.*/
+ { 17, 18, 17, 11, 3,512, ZSTD_btopt2 }, /* level 22.*/
},
{ /* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */
@@ -3142,9 +3225,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/
{ 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/
- { 14, 15, 15, 8, 3,256, ZSTD_btopt }, /* level 20.*/
- { 14, 15, 15, 9, 3,256, ZSTD_btopt }, /* level 21.*/
- { 14, 15, 15, 10, 3,256, ZSTD_btopt }, /* level 22.*/
+ { 14, 15, 15, 8, 3,256, ZSTD_btopt2 }, /* level 20.*/
+ { 14, 15, 15, 9, 3,256, ZSTD_btopt2 }, /* level 21.*/
+ { 14, 15, 15, 10, 3,256, ZSTD_btopt2 }, /* level 22.*/
},
};
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index cb5872908..90d511c7e 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
@@ -16,6 +16,7 @@
#define ZSTD_FREQ_DIV 5
+#define ZSTD_MAX_PRICE (1<<30)
/*-*************************************
* Price functions for optimal parser
@@ -120,12 +121,14 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
}
-FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
+FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
{
/* offset */
BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
+ if (!ultra && offCode >= 20) price += (offCode-19)*2;
+
/* match Length */
{ const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
@@ -171,7 +174,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
{ \
- while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \
+ while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \
opt[pos].mlen = mlen_; \
opt[pos].off = offset_; \
opt[pos].litlen = litlen_; \
@@ -375,7 +378,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
*********************************/
FORCE_INLINE
void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, const int ultra)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
@@ -401,7 +404,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
{ U32 i; for (i=0; irep[i]; }
- inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -424,7 +426,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
}
best_off = i - (ip == anchor);
do {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--;
@@ -449,7 +451,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
best_mlen = matches[u].len;
while (mlen <= best_mlen) {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
mlen++;
@@ -496,7 +498,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
}
- best_mlen = minMatch;
+ best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i=(opt[cur].mlen != 1); i best_mlen) best_mlen = mlen;
- if (opt[cur].mlen == 1) {
- litlen = opt[cur].litlen;
- if (cur > litlen) {
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
- } else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
- } else {
- litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
- }
+ do {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
+ } else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+ }
- if (mlen > best_mlen) best_mlen = mlen;
-
- do {
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
@@ -549,12 +550,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen)
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
} else {
litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
}
if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
@@ -626,7 +627,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
FORCE_INLINE
void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, const int ultra)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
@@ -657,7 +658,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
- inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -666,7 +666,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 current = (U32)(ip-base);
memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0;
- inr = ip;
opt[0].litlen = (U32)(ip - anchor);
/* check repCode */
@@ -691,7 +690,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
best_off = i - (ip==anchor);
litlen = opt[0].litlen;
do {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--;
@@ -721,7 +720,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
best_mlen = matches[u].len;
litlen = opt[0].litlen;
while (mlen <= best_mlen) {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++;
@@ -765,8 +764,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
}
- best_mlen = 0;
-
+ best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i = (mlen != 1); i litlen) {
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
- } else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
- } else {
- litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
- }
-
- best_mlen = mlen;
+ if (mlen > best_mlen) best_mlen = mlen;
do {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
+ } else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+ }
+
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
@@ -815,8 +813,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
goto _storeSequence;
}
- best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch;
-
/* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
@@ -826,12 +822,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen)
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
} else {
litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
}
if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 47b5f42c7..4c47930cd 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -34,7 +34,7 @@
* Frames requiring more memory will be rejected.
*/
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-# define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20) /* 257 MB */
+# define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
#endif
@@ -111,7 +111,7 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
-size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); } /* support sizeof on NULL */
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); }
size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
@@ -170,20 +170,22 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{
ZSTD_decompressBegin(dstDCtx); /* init */
- dstDCtx->dictEnd = srcDCtx->dictEnd;
- dstDCtx->vBase = srcDCtx->vBase;
- dstDCtx->base = srcDCtx->base;
- dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
- dstDCtx->dictID = srcDCtx->dictID;
- dstDCtx->litEntropy = srcDCtx->litEntropy;
- dstDCtx->fseEntropy = srcDCtx->fseEntropy;
- dstDCtx->LLTptr = srcDCtx->LLTable;
- dstDCtx->MLTptr = srcDCtx->MLTable;
- dstDCtx->OFTptr = srcDCtx->OFTable;
- dstDCtx->HUFptr = srcDCtx->hufTable;
- dstDCtx->rep[0] = srcDCtx->rep[0];
- dstDCtx->rep[1] = srcDCtx->rep[1];
- dstDCtx->rep[2] = srcDCtx->rep[2];
+ if (srcDCtx) { /* support refDCtx on NULL */
+ dstDCtx->dictEnd = srcDCtx->dictEnd;
+ dstDCtx->vBase = srcDCtx->vBase;
+ dstDCtx->base = srcDCtx->base;
+ dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
+ dstDCtx->dictID = srcDCtx->dictID;
+ dstDCtx->litEntropy = srcDCtx->litEntropy;
+ dstDCtx->fseEntropy = srcDCtx->fseEntropy;
+ dstDCtx->LLTptr = srcDCtx->LLTable;
+ dstDCtx->MLTptr = srcDCtx->MLTable;
+ dstDCtx->OFTptr = srcDCtx->OFTable;
+ dstDCtx->HUFptr = srcDCtx->hufTable;
+ dstDCtx->rep[0] = srcDCtx->rep[0];
+ dstDCtx->rep[1] = srcDCtx->rep[1];
+ dstDCtx->rep[2] = srcDCtx->rep[2];
+ }
}
@@ -191,7 +193,7 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
* Decompression section
***************************************************************/
-/* See compression format details in : zstd_compression_format.md */
+/* See compression format details in : doc/zstd_compression_format.md */
/** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_prefix.
@@ -248,7 +250,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
if (!singleSegment) {
BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
- if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
+ if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7);
}
@@ -270,7 +272,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
}
if (!windowSize) windowSize = (U32)frameContentSize;
- if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
+ if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID;
@@ -301,14 +303,16 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
/** ZSTD_decodeFrameHeader() :
-* `srcSize` must be the size provided by ZSTD_frameHeaderSize().
+* `headerSize` must be the size provided by ZSTD_frameHeaderSize().
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
-static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
{
- size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
+ size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
+ if (ZSTD_isError(result)) return result; /* invalid header */
+ if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
- return result;
+ return 0;
}
@@ -710,10 +714,13 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
{ int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) {
- if (nbSeq == 0xFF)
+ if (nbSeq == 0xFF) {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
- else
+ } else {
+ if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
}
*nbSeqPtr = nbSeq;
}
@@ -807,7 +814,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
if (ofCode <= 1) {
offset += (llCode==0);
if (offset) {
- size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp;
@@ -839,6 +847,53 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
}
+FORCE_NOINLINE
+size_t ZSTD_execSequenceLast7(BYTE* op,
+ BYTE* const oend, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit_w,
+ const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+ /* check */
+ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */
+
+ /* copy literals */
+ if (op < oend_w) {
+ ZSTD_wildcopy(op, *litPtr, oend_w - op);
+ *litPtr += oend_w - op;
+ op = oend_w;
+ }
+ while (op < oLitEnd) *op++ = *(*litPtr)++;
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - base)) {
+ /* offset beyond prefix */
+ if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
+ match = dictEnd - (base-match);
+ if (match + sequence.matchLength <= dictEnd) {
+ memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = base;
+ } }
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+}
+
+
FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
@@ -853,8 +908,9 @@ size_t ZSTD_execSequence(BYTE* op,
const BYTE* match = oLitEnd - sequence.offset;
/* check */
- if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit_w, base, vBase, dictEnd);
/* copy Literals */
ZSTD_copy8(op, *litPtr);
@@ -878,7 +934,13 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_w) {
+ U32 i;
+ for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {
@@ -1310,25 +1372,28 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
}
{ short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize;
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize;
}
@@ -1397,7 +1462,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
return NULL;
}
- memcpy(dictContent, dict, dictSize);
+ if (dictSize) {
+ memcpy(dictContent, dict, dictSize);
+ }
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
@@ -1467,7 +1534,8 @@ typedef enum { zdss_init, zdss_loadHeader,
/* *** Resource management *** */
struct ZSTD_DStream_s {
ZSTD_DCtx* dctx;
- ZSTD_DDict* ddict;
+ ZSTD_DDict* ddictLocal;
+ const ZSTD_DDict* ddict;
ZSTD_frameParams fParams;
ZSTD_dStreamStage stage;
char* inBuff;
@@ -1517,7 +1585,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
if (zds==NULL) return 0; /* support free on null */
{ ZSTD_customMem const cMem = zds->customMem;
ZSTD_freeDCtx(zds->dctx);
- ZSTD_freeDDict(zds->ddict);
+ ZSTD_freeDDict(zds->ddictLocal);
ZSTD_free(zds->inBuff, cMem);
ZSTD_free(zds->outBuff, cMem);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
@@ -1539,9 +1607,12 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
{
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
- ZSTD_freeDDict(zds->ddict);
- zds->ddict = ZSTD_createDDict(dict, dictSize);
- if (zds->ddict == NULL) return ERROR(memory_allocation);
+ ZSTD_freeDDict(zds->ddictLocal);
+ if (dict) {
+ zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
+ if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
+ } else zds->ddictLocal = NULL;
+ zds->ddict = zds->ddictLocal;
zds->legacyVersion = 0;
zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix;
@@ -1552,9 +1623,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0);
}
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) /**< note : ddict will just be referenced, and must outlive decompression session */
+{
+ size_t const initResult = ZSTD_initDStream(zds);
+ zds->ddict = ddict;
+ return initResult;
+}
+
size_t ZSTD_resetDStream(ZSTD_DStream* zds)
{
- if (zds->ddict == NULL) return ERROR(stage_wrong); /* must be init at least once */
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
zds->legacyVersion = 0;
@@ -1568,7 +1645,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
switch(paramType)
{
default : return ERROR(parameter_unknown);
- case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
+ case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
}
return 0;
}
@@ -1577,7 +1654,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{
if (zds==NULL) return 0; /* support sizeof on NULL */
- return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize;
+ return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize;
}
@@ -1618,15 +1695,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) {
+ const void* const dict = zds->ddict ? zds->ddict->dict : NULL;
+ size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
- zds->ddict->dict, zds->ddict->dictSize));
+ dict, dictSize));
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
} else {
return hSize; /* error */
} }
#else
- return hSize;
+ return hSize;
#endif
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
@@ -1641,7 +1720,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
/* Consume header */
- ZSTD_refDCtx(zds->dctx, zds->ddict->refContext);
+ { const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL;
+ ZSTD_refDCtx(zds->dctx, refContext);
+ }
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
{ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
@@ -1649,7 +1730,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
- if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
+ if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 47a82af14..b3f20b12b 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -371,21 +371,22 @@ static dictItem ZDICT_analyzePos(
static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
{
const U32 tableSize = table->pos;
- const U32 max = elt.pos + (elt.length-1);
+ const U32 eltEnd = elt.pos + elt.length;
/* tail overlap */
U32 u; for (u=1; u elt.pos) && (table[u].pos < max)) { /* overlap */
+ if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */
/* append */
U32 addedLength = table[u].pos - elt.pos;
table[u].length += addedLength;
table[u].pos = elt.pos;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
- table[u].savings += elt.length / 8; /* rough approx */
+ table[u].savings += elt.length / 8; /* rough approx bonus */
elt = table[u];
+ /* sort : improve rank */
while ((u>1) && (table[u-1].savings < elt.savings))
- table[u] = table[u-1], u--;
+ table[u] = table[u-1], u--;
table[u] = elt;
return u;
} }
@@ -393,14 +394,15 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
/* front overlap */
for (u=1; u elt.pos) && (table[u].pos < elt.pos)) { /* overlap */
+ if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */
- int addedLength = (elt.pos + elt.length) - (table[u].pos + table[u].length);
- table[u].savings += elt.length / 8; /* rough approx */
- if (addedLength > 0) { /* otherwise, already included */
+ int addedLength = (int)eltEnd - (table[u].pos + table[u].length);
+ table[u].savings += elt.length / 8; /* rough approx bonus */
+ if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += addedLength;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
}
+ /* sort : improve rank */
elt = table[u];
while ((u>1) && (table[u-1].savings < elt.savings))
table[u] = table[u-1], u--;
@@ -811,7 +813,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
MEM_writeLE32(dstPtr+4, repStartValue[1]);
MEM_writeLE32(dstPtr+8, repStartValue[2]);
#endif
- dstPtr += 12;
+ //dstPtr += 12;
eSize += 12;
_cleanup:
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index fe9c5ccdd..5c36c2108 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -958,13 +958,16 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
U32 weightTotal;
U32 maxBits;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
U32 nextRankStart;
void* ptr = DTable+1;
HUF_DElt* const dt = (HUF_DElt*)ptr;
+ if (!srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
+ iSize = ip[0];
+
FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -1005,6 +1008,7 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
rankVal[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return (size_t)-FSE_ERROR_corruptionDetected;
/* get last non-null symbol weight (implied, total must be 2^n) */
maxBits = FSE_highbit32(weightTotal) + 1;
@@ -1533,6 +1537,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
{
size_t rleSize = litbp.origSize;
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
+ if (!srcSize) return ERROR(srcSize_wrong);
memset(oend - rleSize, *ip, rleSize);
*litStart = oend - rleSize;
*litSize = rleSize;
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index de1592e18..24498fedf 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -1607,10 +1607,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -1652,6 +1654,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1;
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index caad331d3..a3bd1da23 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -1604,10 +1604,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -1649,6 +1651,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1;
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index c9dcb94e0..0a740baca 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -1896,10 +1896,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -1941,6 +1943,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1;
@@ -3107,8 +3110,13 @@ static size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
}
}
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8)
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 5027e2b8b..201bf3c6b 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -1873,10 +1873,12 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -1910,6 +1912,7 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BITv05_highbit32(weightTotal) + 1;
@@ -2032,13 +2035,14 @@ size_t HUFv05_decompress1X2_usingDTable(
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize;
- size_t errorCode;
const U32 dtLog = DTable[0];
const void* dtPtr = DTable;
const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1;
BITv05_DStream_t bitD;
- errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
- if (HUFv05_isError(errorCode)) return errorCode;
+
+ if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall);
+ { size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
+ if (HUFv05_isError(errorCode)) return errorCode; }
HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog);
@@ -2942,6 +2946,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
{
size_t litSize, litCSize, singleStream=0;
U32 lhSize = ((istart[0]) >> 4) & 3;
+ if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
switch(lhSize)
{
case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */
@@ -2965,6 +2970,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break;
}
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUFv05_isError(singleStream ?
HUFv05_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
@@ -2990,6 +2996,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
lhSize=3;
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2];
+ if (litCSize + litSize > srcSize) return ERROR(corruption_detected);
errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUFv05_isError(errorCode)) return ERROR(corruption_detected);
@@ -3046,6 +3053,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break;
case 3:
litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2];
+ if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break;
}
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
@@ -3063,7 +3071,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, U32 flagStaticTable)
{
const BYTE* const istart = (const BYTE* const)src;
const BYTE* ip = istart;
@@ -3079,17 +3087,22 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
/* SeqHead */
*nbSeq = *ip++;
if (*nbSeq==0) return 1;
- if (*nbSeq >= 128)
+ if (*nbSeq >= 128) {
+ if (ip >= iend) return ERROR(srcSize_wrong);
*nbSeq = ((nbSeq[0]-128)<<8) + *ip++;
+ }
+ if (ip >= iend) return ERROR(srcSize_wrong);
LLtype = *ip >> 6;
Offtype = (*ip >> 4) & 3;
MLtype = (*ip >> 2) & 3;
if (*ip & 2) {
+ if (ip+3 > iend) return ERROR(srcSize_wrong);
dumpsLength = ip[2];
dumpsLength += ip[1] << 8;
ip += 3;
} else {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
dumpsLength = ip[1];
dumpsLength += (ip[0] & 1) << 8;
ip += 2;
@@ -3118,6 +3131,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableLL, LLbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3141,6 +3155,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableOffb, Offbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3164,6 +3179,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableML, MLbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3312,7 +3328,12 @@ static size_t ZSTDv05_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {
@@ -3371,7 +3392,7 @@ static size_t ZSTDv05_decompressSequences(
/* Build Decoding Tables */
errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
DTableLL, DTableML, DTableOffb,
- ip, seqSize);
+ ip, seqSize, dctx->flagStaticTables);
if (ZSTDv05_isError(errorCode)) return errorCode;
ip += errorCode;
@@ -3660,11 +3681,11 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
{
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSEv05Log;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSEv05Log;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSEv05Log;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
hSize = HUFv05_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUFv05_isError(hSize)) return ERROR(dictionary_corrupted);
@@ -3673,6 +3694,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
offcodeHeaderSize = FSEv05_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSEv05_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSEv05Log) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + offcodeHeaderSize;
@@ -3680,12 +3702,14 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
matchlengthHeaderSize = FSEv05_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSEv05_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSEv05Log) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + matchlengthHeaderSize;
dictSize -= matchlengthHeaderSize;
litlengthHeaderSize = FSEv05_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
+ if (litlengthLog > LLFSEv05Log) return ERROR(dictionary_corrupted);
if (FSEv05_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index d9e89f806..b6fde3aa6 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -1932,9 +1932,11 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -1969,6 +1971,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BITv06_highbit32(weightTotal) + 1;
@@ -3183,6 +3186,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
lhSize=3;
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2];
+ if (litCSize + litSize > srcSize) return ERROR(corruption_detected);
{ size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUFv06_isError(errorCode)) return ERROR(corruption_detected);
@@ -3302,10 +3306,13 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
{ int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) {
- if (nbSeq == 0xFF)
+ if (nbSeq == 0xFF) {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
- else
+ } else {
+ if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
}
*nbSeqPtr = nbSeq;
}
@@ -3466,7 +3473,12 @@ size_t ZSTDv06_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {
@@ -3822,9 +3834,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
dictSize -= hSize;
{ short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + offcodeHeaderSize;
@@ -3832,9 +3845,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + matchlengthHeaderSize;
@@ -3842,9 +3856,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
}
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index f4c8073f9..c7693f240 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -1382,9 +1382,11 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -1419,6 +1421,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BITv07_highbit32(weightTotal) + 1;
@@ -3529,10 +3532,13 @@ size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
{ int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) {
- if (nbSeq == 0xFF)
+ if (nbSeq == 0xFF) {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
- else
+ } else {
+ if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
}
*nbSeqPtr = nbSeq;
}
@@ -3690,7 +3696,12 @@ size_t ZSTDv07_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_w) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {
@@ -4097,27 +4108,30 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si
}
{ short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += offcodeHeaderSize;
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += matchlengthHeaderSize;
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += litlengthHeaderSize;
diff --git a/lib/zstd.h b/lib/zstd.h
index dd3f5df4c..ea5caf148 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1,4 +1,4 @@
-/**
+/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
@@ -14,12 +14,12 @@
extern "C" {
#endif
-/*====== Dependency ======*/
+/* ====== Dependency ======*/
#include /* size_t */
-/*====== Export for Windows ======*/
-/*!
+/* ====== Export for Windows ======*/
+/*
* ZSTD_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
@@ -30,10 +30,32 @@ extern "C" {
#endif
-/*======= Version =======*/
+/*******************************************************************************************************
+ Introduction
+
+ zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+ at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
+ decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+ Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+ Compression can be done in:
+ - a single step (described as Simple API)
+ - a single step, reusing a context (described as Explicit memory management)
+ - unbounded multiple steps (described as Streaming compression)
+ The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
+ - a single step (described as Simple dictionary API)
+ - a single step, reusing a dictionary (described as Fast dictionary API)
+
+ Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+ These APIs shall never be used with a dynamic library.
+ They are not "stable", their definition may change in the future. Only static linking is allowed.
+*********************************************************************************************************/
+
+/*------ Version ------*/
+ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */
+
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
-#define ZSTD_VERSION_RELEASE 0
+#define ZSTD_VERSION_RELEASE 1
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@@ -41,10 +63,9 @@ extern "C" {
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
-ZSTDLIB_API unsigned ZSTD_versionNumber (void);
-/* *************************************
+/***************************************
* Simple API
***************************************/
/*! ZSTD_compress() :
@@ -91,29 +112,33 @@ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `siz
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
-/*-*************************************
+/***************************************
* Explicit memory management
***************************************/
-/** Compression context */
+/*= Compression context
+* When compressing many messages / blocks,
+* it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+* This will make the situation much easier for the system's memory.
+* Use one context per thread for parallel execution in multi-threaded environments. */
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
-/** ZSTD_compressCCtx() :
+/*! ZSTD_compressCCtx() :
Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
-/** Decompression context */
+/*= Decompression context */
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-/** ZSTD_decompressDCtx() :
+/*! ZSTD_decompressDCtx() :
* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-/*-************************
+/**************************
* Simple dictionary API
***************************/
/*! ZSTD_compress_usingDict() :
@@ -135,14 +160,20 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
const void* dict,size_t dictSize);
-/*-**************************
-* Fast Dictionary API
+/****************************
+* Fast dictionary API
****************************/
-/*! ZSTD_createCDict() :
-* Create a digested dictionary, ready to start compression operation without startup delay.
-* `dict` can be released after ZSTD_CDict creation */
typedef struct ZSTD_CDict_s ZSTD_CDict;
+
+/*! ZSTD_createCDict() :
+* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
+* `dict` can be released after ZSTD_CDict creation */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+
+/*! ZSTD_freeCDict() :
+* Function frees memory allocated by ZSTD_createCDict() */
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
/*! ZSTD_compress_usingCDict() :
@@ -154,11 +185,16 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict);
+
+typedef struct ZSTD_DDict_s ZSTD_DDict;
+
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation */
-typedef struct ZSTD_DDict_s ZSTD_DDict;
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_freeDDict() :
+* Function frees memory allocated with ZSTD_createDDict() */
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
/*! ZSTD_decompress_usingDDict() :
@@ -170,7 +206,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict);
-/*-**************************
+/****************************
* Streaming
****************************/
@@ -187,16 +223,18 @@ typedef struct ZSTD_outBuffer_s {
} ZSTD_outBuffer;
-/*====== streaming compression ======*/
/*-***********************************************************************
-* Streaming compression - howto
+* Streaming compression - HowTo
*
* A ZSTD_CStream object is required to track streaming operation.
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+* It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
+* since it will play nicer with system's memory, by re-using already allocated memory.
+* Use one separate ZSTD_CStream per thread for parallel execution.
*
-* Start by initializing ZSTD_CStream.
+* Start a new compression by initializing ZSTD_CStream.
* Use ZSTD_initCStream() to start a new compression operation.
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
*
@@ -225,23 +263,22 @@ typedef struct ZSTD_outBuffer_s {
*
* *******************************************************************/
+/*===== Streaming compression functions ======*/
typedef struct ZSTD_CStream_s ZSTD_CStream;
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-
-ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
-
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+
-/*====== decompression ======*/
/*-***************************************************************************
-* Streaming decompression howto
+* Streaming decompression - HowTo
*
* A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
@@ -262,28 +299,29 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
* The return value is a suggested next input size (just an hint, to help latency).
* *******************************************************************************/
+/*===== Streaming decompression functions =====*/
typedef struct ZSTD_DStream_s ZSTD_DStream;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
-ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
-ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-
#ifdef ZSTD_STATIC_LINKING_ONLY
-/* ====================================================================================
+/****************************************************************************************
+ * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
* The definitions in this section are considered experimental.
* They should never be used with a dynamic library, as they may change in the future.
* They are provided for advanced usages.
* Use them only in association with static linking.
- * ==================================================================================== */
+ * ***************************************************************************************/
-/*--- Constants ---*/
+/* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
@@ -310,8 +348,8 @@ static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
-/*--- Types ---*/
-typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */
+/*--- Advanced types ---*/
+typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
typedef struct {
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -334,13 +372,13 @@ typedef struct {
ZSTD_frameParameters fParams;
} ZSTD_parameters;
-/* custom memory allocation functions */
+/*= Custom memory allocation functions */
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-/*-*************************************
+/***************************************
* Advanced compression functions
***************************************/
/*! ZSTD_estimateCCtxSize() :
@@ -393,7 +431,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
ZSTD_parameters params);
-/*--- Advanced Decompression functions ---*/
+/*--- Advanced decompression functions ---*/
/*! ZSTD_estimateDCtxSize() :
* Gives the potential amount of memory allocated to create a ZSTD_DCtx */
@@ -412,47 +450,41 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-/* ******************************************************************
-* Advanced Streaming functions
+/********************************************************************
+* Advanced streaming functions
********************************************************************/
-/*====== compression ======*/
-
+/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
-ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
+ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-/*====== decompression ======*/
-
+/*===== Advanced Streaming decompression functions =====*/
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
-
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-/* ******************************************************************
+/*********************************************************************
* Buffer-less and synchronous inner streaming functions
-********************************************************************/
-/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+*
+* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
* But it's also a complex one, with many restrictions (documented below).
-* Prefer using normal streaming API for an easier experience */
+* Prefer using normal streaming API for an easier experience
+********************************************************************* */
-ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+/**
+ Buffer-less streaming compression (synchronous mode)
-ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-
-/*
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -481,26 +513,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
*/
-typedef struct {
- unsigned long long frameContentSize;
- unsigned windowSize;
- unsigned dictID;
- unsigned checksumFlag;
-} ZSTD_frameParams;
+/*===== Buffer-less streaming compression functions =====*/
+ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
-ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
-ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
-ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
-ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-
-typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
-ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-
-/*
+/*-
Buffer-less streaming decompression (synchronous mode)
A ZSTD_DCtx object is required to track streaming operations.
@@ -557,11 +580,27 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
It also returns Frame Size as fparamsPtr->frameContentSize.
*/
+typedef struct {
+ unsigned long long frameContentSize;
+ unsigned windowSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+} ZSTD_frameParams;
-/* **************************************
-* Block functions
-****************************************/
-/*! Block functions produce and decode raw zstd blocks, without frame metadata.
+/*===== Buffer-less streaming decompression functions =====*/
+ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
+ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+
+/**
+ Block functions
+
+ Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -585,6 +624,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
*/
#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
+/*===== Raw zstd block functions =====*/
ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
diff --git a/programs/Makefile b/programs/Makefile
index 6e78d0ea9..f5e625f06 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -57,8 +57,8 @@ endif
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
VOID = nul
-RES64_FILE = ..\build\VS2010\zstd\generate_res\zstd64.res
-RES32_FILE = ..\build\VS2010\zstd\generate_res\zstd32.res
+RES64_FILE = windres\zstd64.res
+RES32_FILE = windres\zstd32.res
ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
RES_FILE = $(RES64_FILE)
else
@@ -70,29 +70,36 @@ VOID = /dev/null
endif
-.PHONY: default all clean install uninstall
+.PHONY: default all clean clean_decomp_o install uninstall generate_res
default: zstd
all: zstd
-
+$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
- $(CC) $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
+
+zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
+ zstdcli.c fileio.c bench.c datagen.c dibio.c
+ifneq (,$(filter Windows%,$(OS)))
+ windres\generate_res.bat
+endif
+ $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT)
+
$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
$(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
-zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
- zstdcli.c fileio.c bench.c datagen.c dibio.c
- $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT)
-
zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
+ifneq (,$(filter Windows%,$(OS)))
+ windres\generate_res.bat
+endif
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT)
-zstd_nolegacy :
+zstd_nolegacy : clean_decomp_o
$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
zstd-pgo : MOREFLAGS = -fprofile-generate
@@ -108,19 +115,22 @@ zstd-pgo : clean zstd
$(MAKE) zstd MOREFLAGS=-fprofile-use
zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
- $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
+ $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT)
-zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
- zstdcli.c fileio.c
- $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
+zstd-small: clean_decomp_o
+ ZSTD_LEGACY_SUPPORT=0 CFLAGS="-Os -s" $(MAKE) zstd-frugal
-zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
- zstdcli.c fileio.c
- $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
+zstd-decompress-clean: $(ZSTDDECOMP_O) $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
+ $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT)
-zstd-small: clean
- CFLAGS="-Os -s" $(MAKE) zstd-frugal
+zstd-decompress: clean_decomp_o
+ ZSTD_LEGACY_SUPPORT=0 $(MAKE) zstd-decompress-clean
+zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
+ $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
+
+generate_res:
+ windres\generate_res.bat
clean:
$(MAKE) -C ../lib clean
@@ -130,6 +140,10 @@ clean:
*.gcda default.profraw
@echo Cleaning completed
+clean_decomp_o:
+ @$(RM) $(ZSTDDECOMP_O)
+ @$(RM) $(ZSTDDECOMP32_O)
+
#----------------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
@@ -148,11 +162,11 @@ install: zstd
@echo zstd installation completed
uninstall:
- $(RM) $(DESTDIR)$(BINDIR)/zstdcat
- $(RM) $(DESTDIR)$(BINDIR)/unzstd
- $(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
- $(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
- $(RM) $(DESTDIR)$(MANDIR)/unzstd.1
- $(RM) $(DESTDIR)$(MANDIR)/zstd.1
+ @$(RM) $(DESTDIR)$(BINDIR)/zstdcat
+ @$(RM) $(DESTDIR)$(BINDIR)/unzstd
+ @$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
+ @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
+ @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
+ @$(RM) $(DESTDIR)$(MANDIR)/zstd.1
@echo zstd programs successfully uninstalled
endif
diff --git a/programs/fileio.c b/programs/fileio.c
index 56f22fe75..c4c308e03 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -8,13 +8,6 @@
*/
-/*
- Note : this file is part of zstd command line, which is not library.
- The license of ZSTD library is BSD.
- The license of this file is GPLv2.
-*/
-
-
/* *************************************
* Tuning options
***************************************/
@@ -127,6 +120,9 @@ static U32 g_checksumFlag = 1;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
static U32 g_removeSrcFile = 0;
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
+static U32 g_memLimit = 0;
+void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
+
/*-*************************************
@@ -487,6 +483,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* Allocation */
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
+ ZSTD_setDStreamParameter(ress.dctx, ZSTDdsp_maxWindowSize, g_memLimit);
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_DStreamOutSize();
diff --git a/programs/fileio.h b/programs/fileio.h
index 1e89aec27..60a7e0de8 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -15,7 +15,6 @@
extern "C" {
#endif
-
/* *************************************
* Special i/o constants
**************************************/
@@ -37,6 +36,7 @@ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdo
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setChecksumFlag(unsigned checksumFlag);
void FIO_setRemoveSrcFile(unsigned flag);
+void FIO_setMemLimit(unsigned memLimit);
/*-*************************************
diff --git a/programs/windres/generate_res.bat b/programs/windres/generate_res.bat
new file mode 100644
index 000000000..76e47fa0a
--- /dev/null
+++ b/programs/windres/generate_res.bat
@@ -0,0 +1,11 @@
+@echo off
+REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
+
+where /q windres.exe
+IF ERRORLEVEL 1 (
+ ECHO The windres.exe is missing. Ensure it is installed and placed in your PATH.
+ EXIT /B
+) ELSE (
+ windres.exe -I ..\lib -I windres -i windres\zstd.rc -O coff -F pe-x86-64 -o windres\zstd64.res
+ windres.exe -I ..\lib -I windres -i windres\zstd.rc -O coff -F pe-i386 -o windres\zstd32.res
+)
diff --git a/programs/windres/verrsrc.h b/programs/windres/verrsrc.h
new file mode 100644
index 000000000..e282add02
--- /dev/null
+++ b/programs/windres/verrsrc.h
@@ -0,0 +1,8 @@
+/* minimal set of defines required to generate zstd.res from zstd.rc */
+
+#define VS_VERSION_INFO 1
+
+#define VS_FFI_FILEFLAGSMASK 0x0000003FL
+#define VOS_NT_WINDOWS32 0x00040004L
+#define VFT_DLL 0x00000002L
+#define VFT2_UNKNOWN 0x00000000L
diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc
new file mode 100644
index 000000000..f5e404730
--- /dev/null
+++ b/programs/windres/zstd.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "zstd.h" /* ZSTD_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+ PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+ BLOCK "StringFileInfo"
+ BEGIN
+ BLOCK "040904B0"
+ BEGIN
+ VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+ VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
+ VALUE "FileVersion", ZSTD_VERSION_STRING
+ VALUE "InternalName", "zstd.exe"
+ VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+ VALUE "OriginalFilename", "zstd.exe"
+ VALUE "ProductName", "Zstandard"
+ VALUE "ProductVersion", ZSTD_VERSION_STRING
+ END
+ END
+ BLOCK "VarFileInfo"
+ BEGIN
+ VALUE "Translation", 0x0409, 1200
+ END
+END
+
+#endif
diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res
new file mode 100644
index 000000000..aec8fcf24
Binary files /dev/null and b/programs/windres/zstd32.res differ
diff --git a/programs/windres/zstd64.res b/programs/windres/zstd64.res
new file mode 100644
index 000000000..0fa5040fd
Binary files /dev/null and b/programs/windres/zstd64.res differ
diff --git a/programs/zstd.1 b/programs/zstd.1
index c262a0c6a..9d91b5100 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -27,30 +27,148 @@ is equivalent to
.SH DESCRIPTION
.PP
-\fBzstd\fR is a fast lossless compression algorithm.
+\fBzstd\fR is a fast lossless compression algorithm
+and data compression tool,
+with command line syntax similar to \fB gzip (1) \fR and \fB xz (1) \fR .
It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages.
-\fBzstd\fR offers configurable compression speed, with fast modes at > 200 MB/s per core.
-It also features a very fast decoder, with speed > 500 MB/s per core.
+\fBzstd\fR offers highly configurable compression speed,
+with fast modes at > 200 MB/s per core,
+and strong modes nearing lzma compression ratios.
+It also features a very fast decoder, with speeds > 500 MB/s per core.
-\fBzstd\fR command line is generally similar to gzip, but features the following differences :
- - Source files are preserved by default
- It's possible to remove them automatically by using \fB--rm\fR command
- - By default, when compressing a single file, \fBzstd\fR displays progress notifications and result summary.
- Use \fB-q\fR to turn them off
+\fBzstd\fR command line syntax is generally similar to gzip,
+but features the following differences :
+ - Source files are preserved by default.
+ It's possible to remove them automatically by using \fB--rm\fR command.
+ - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default.
+ Use \fB-q\fR to turn them off
+
+.PP
+.B zstd
+compresses or decompresses each
+.I file
+according to the selected operation mode.
+If no
+.I files
+are given or
+.I file
+is
+.BR \- ,
+.B zstd
+reads from standard input and writes the processed data
+to standard output.
+.B zstd
+will refuse (display an error and skip the
+.IR file )
+to write compressed data to standard output if it is a terminal.
+Similarly,
+.B zstd
+will refuse to read compressed data
+from standard input if it is a terminal.
+
+.PP
+Unless
+.B \-\-stdout
+is specified,
+.I files
+are written to a new file whose name is derived from the source
+.I file
+name:
+.IP \(bu 3
+When compressing, the suffix
+.B .zst
+is appended to the source filename to get the target filename.
+.IP \(bu 3
+When decompressing, the
+.B .zst
+suffix is removed from the filename to get the target filename.
+
+.SS "Concatenation with .zst files"
+It is possible to concatenate
+.B .zst
+files as is.
+.B zstd
+will decompress such files as if they were a single
+.B .zst
+file.
.SH OPTIONS
+
+.
+.SS "Integer suffixes and special values"
+In most places where an integer argument is expected,
+an optional suffix is supported to easily indicate large integers.
+There must be no space between the integer and the suffix.
+.TP
+.B KiB
+Multiply the integer by 1,024 (2^10).
+.BR Ki ,
+.BR K ,
+and
+.B KB
+are accepted as synonyms for
+.BR KiB .
+.TP
+.B MiB
+Multiply the integer by 1,048,576 (2^20).
+.BR Mi ,
+.BR M ,
+and
+.B MB
+are accepted as synonyms for
+.BR MiB .
+
+.
+.SS "Operation mode"
+If multiple operation mode options are given,
+the last one takes effect.
+.TP
+.BR \-z ", " \-\-compress
+Compress.
+This is the default operation mode when no operation mode option
+is specified and no other operation mode is implied from
+the command name (for example,
+.B unzstd
+implies
+.BR \-\-decompress ).
+.TP
+.BR \-d ", " \-\-decompress ", " \-\-uncompress
+Decompress.
+.TP
+.BR \-t ", " \-\-test
+Test the integrity of compressed
+.IR files .
+This option is equivalent to
+.B "\-\-decompress \-\-stdout"
+except that the decompressed data is discarded instead of being
+written to standard output.
+No files are created or removed.
+.TP
+.B \-b#
+ benchmark file(s) using compression level #
+.TP
+.B \--train FILEs
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100).
+
+.
+.SS "Operation modifiers"
.TP
.B \-#
- # compression level [1-22] (default:3)
+ # compression level [1-19] (default:3)
.TP
-.BR \-d ", " --decompress
- decompression
+.BR \--ultra
+ unlocks high compression levels 20+ (maximum 22), using a lot more memory
.TP
.B \-D file
use `file` as Dictionary to compress or decompress FILE(s)
.TP
+.BR \--no-dictID
+ do not store dictionary ID within frame header (dictionary compression).
+ The decoder will have to rely on implicit knowledge about which dictionary to use,
+it won't be able to check if it's correct.
+.TP
.B \-o file
save result into `file` (only possible with a single INPUT-FILE)
.TP
@@ -60,6 +178,13 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
.BR \-c ", " --stdout
force write to standard output, even if it is the console
.TP
+.BR \--[no-]sparse
+ enable / disable sparse FS support, to make files with many zeroes smaller on disk.
+ Creating sparse files may save disk space and speed up the decompression
+by reducing the amount of disk I/O.
+ default : enabled when output is into a file, and disabled when output is stdout.
+ This setting overrides default and can force sparse mode over stdout.
+.TP
.BR \--rm
remove source file(s) after successful compression or decompression
.TP
@@ -83,8 +208,8 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
suppress warnings, interactivity and notifications.
specify twice to suppress errors too.
.TP
-.BR \-C ", " --check
- add integrity check computed from uncompressed data
+.BR \-C ", " --[no-]check
+ add integrity check computed from uncompressed data (default : enabled)
.TP
.BR \-t ", " --test
Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
@@ -94,7 +219,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
All arguments after -- are treated as files
-.SH DICTIONARY
+.SH DICTIONARY BUILDER
.PP
\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages.
It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`.
@@ -103,10 +228,8 @@ It will improve compression ratio of small files.
Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
.TP
.B \--train FILEs
- use FILEs as training set to create a dictionary.
- The training set should contain a lot of small files (> 100).
- and weight typically 100x the target dictionary size
- (for example, 10 MB for a 100 KB dictionary)
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100),
+and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)
.TP
.B \-o file
dictionary saved into `file` (default: dictionary)
@@ -131,6 +254,9 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
.B \-b#
benchmark file(s) using compression level #
.TP
+.B \-e#
+ benchmark file(s) using multiple compression levels, from -b# to -e# (included).
+.TP
.B \-i#
minimum evaluation time, in seconds (default : 3s), benchmark mode only
.TP
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 6d1d9f649..db4d6acc6 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -8,13 +8,6 @@
*/
-/*
- Note : this is a user program, not part of libzstd.
- The license of libzstd is BSD.
- The license of this command line program is GPLv2.
-*/
-
-
/*-************************************
* Tuning parameters
**************************************/
@@ -32,7 +25,6 @@
**************************************/
#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
#include /* strcmp, strlen */
-#include /* toupper */
#include /* errno */
#include "fileio.h"
#ifndef ZSTD_NOBENCH
@@ -142,6 +134,7 @@ static int usage_advanced(const char* programName)
DISPLAY( "--test : test compressed file integrity \n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
#endif
+ DISPLAY( " -M# : Set a memory usage limit for decompression \n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT
DISPLAY( "\n");
@@ -179,40 +172,55 @@ static void waitEnter(void)
}
/*! readU32FromChar() :
- @return : unsigned integer value reach from input in `char` format
+ @return : unsigned integer value read from input in `char` format
+ allows and interprets K, KB, KiB, M, MB and MiB suffix.
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
- Note : this function can overflow if digit string > MAX_UINT */
+ Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+ if ((**stringPtr=='K') || (**stringPtr=='M')) {
+ result <<= 10;
+ if (**stringPtr=='M') result <<= 10;
+ (*stringPtr)++ ;
+ if (**stringPtr=='i') (*stringPtr)++;
+ if (**stringPtr=='B') (*stringPtr)++;
+ }
return result;
}
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+ size_t const comSize = strlen(longCommand);
+ unsigned const result = !strncmp(*stringPtr, longCommand, comSize);
+ if (result) *stringPtr += comSize;
+ return result;
+}
+
+typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
int main(int argCount, const char* argv[])
{
int argNb,
- bench=0,
- decode=0,
- testmode=0,
forceStdout=0,
main_pause=0,
nextEntryIsDictionary=0,
operationResult=0,
- dictBuild=0,
nextArgumentIsOutFileName=0,
nextArgumentIsMaxDict=0,
nextArgumentIsDictID=0,
nextArgumentIsFile=0,
ultra=0,
lastCommand = 0;
+ zstd_operation_mode operation = zom_compress;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
+ unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0;
const char* programName = argv[0];
@@ -231,7 +239,8 @@ int main(int argCount, const char* argv[])
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */
- (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
+ (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
+ (void)ultra; (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
displayOut = stderr;
@@ -242,20 +251,22 @@ int main(int argCount, const char* argv[])
}
/* preset behaviors */
- if (!strcmp(programName, ZSTD_UNZSTD)) decode=1;
- if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
+ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
+ if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
/* command switches */
- for(argNb=1; argNb maxCLevel) {
DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
cLevel = maxCLevel;
} }
+#endif
/* No warning message in pipe mode (stdin + stdout) or multi-files mode */
if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
@@ -503,7 +525,7 @@ int main(int argCount, const char* argv[])
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
- if (!decode) {
+ if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
@@ -512,9 +534,10 @@ int main(int argCount, const char* argv[])
#else
DISPLAY("Compression not supported\n");
#endif
- } else { /* decompression */
+ } else { /* decompression or test */
#ifndef ZSTD_NODECOMPRESS
- if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
+ if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
+ FIO_setMemLimit(memLimit);
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
else
diff --git a/tests/Makefile b/tests/Makefile
index 3ce9f317e..ecff18290 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,25 +1,10 @@
# ##########################################################################
-# ZSTD tests - Makefile
-# Copyright (C) Yann Collet 2015-2016
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
#
-# GPL v2 License
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# You can contact the author at :
-# - zstd homepage : http://www.zstd.net/
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
# ##########################################################################
# datagen : Synthetic and parametrable data generator, for tests
# fullbench : Precisely measure speed for each zstd inner functions
@@ -94,10 +79,10 @@ zstd32:
zstd_nolegacy:
$(MAKE) -C $(PRGDIR) $@
-fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c
+fullbench : $(ZSTD_FILES) $(PRGDIR)/datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
-fullbench32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c
+fullbench32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 670b51681..ffc32f9bb 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -22,7 +22,6 @@
#include "zstd.h" /* ZSTD_VERSION_STRING */
#define FSE_STATIC_LINKING_ONLY /* FSE_DTABLE_SIZE_U32 */
#include "fse.h"
-#include "zbuff.h"
#include "datagen.h"
@@ -130,29 +129,39 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
}
-static ZBUFF_CCtx* g_zbcc = NULL;
-size_t local_ZBUFF_compress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static ZSTD_CStream* g_cstream= NULL;
+size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
- size_t compressedSize;
- size_t srcRead = srcSize, dstWritten = dstCapacity;
+ ZSTD_outBuffer buffOut;
+ ZSTD_inBuffer buffIn;
(void)buff2;
- ZBUFF_compressInit(g_zbcc, 1);
- ZBUFF_compressContinue(g_zbcc, dst, &dstWritten, src, &srcRead);
- compressedSize = dstWritten;
- dstWritten = dstCapacity-compressedSize;
- ZBUFF_compressEnd(g_zbcc, ((char*)dst)+compressedSize, &dstWritten);
- compressedSize += dstWritten;
- return compressedSize;
+ ZSTD_initCStream(g_cstream, 1);
+ buffOut.dst = dst;
+ buffOut.size = dstCapacity;
+ buffOut.pos = 0;
+ buffIn.src = src;
+ buffIn.size = srcSize;
+ buffIn.pos = 0;
+ ZSTD_compressStream(g_cstream, &buffOut, &buffIn);
+ ZSTD_endStream(g_cstream, &buffOut);
+ return buffOut.pos;
}
-static ZBUFF_DCtx* g_zbdc = NULL;
-static size_t local_ZBUFF_decompress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static ZSTD_DStream* g_dstream= NULL;
+static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
- size_t srcRead = g_cSize, dstWritten = dstCapacity;
+ ZSTD_outBuffer buffOut;
+ ZSTD_inBuffer buffIn;
(void)src; (void)srcSize;
- ZBUFF_decompressInit(g_zbdc);
- ZBUFF_decompressContinue(g_zbdc, dst, &dstWritten, buff2, &srcRead);
- return dstWritten;
+ ZSTD_initDStream(g_dstream);
+ buffOut.dst = dst;
+ buffOut.size = dstCapacity;
+ buffOut.pos = 0;
+ buffIn.src = buff2;
+ buffIn.size = g_cSize;
+ buffIn.pos = 0;
+ ZSTD_decompressStream(g_dstream, &buffOut, &buffIn);
+ return buffOut.pos;
}
static ZSTD_CCtx* g_zcc = NULL;
@@ -220,10 +229,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
break;
case 41:
- benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue";
+ benchFunction = local_ZSTD_compressStream; benchName = "ZSTD_compressStream";
break;
case 42:
- benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue";
+ benchFunction = local_ZSTD_decompressStream; benchName = "ZSTD_decompressStream";
break;
default :
return 0;
@@ -296,10 +305,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
break;
}
case 41 :
- if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx();
+ if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
break;
case 42 :
- if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx();
+ if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
@@ -311,27 +320,27 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
{ size_t i; for (i=0; i %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); }
- }
- averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
- if (averageTime < bestTime) bestTime = averageTime;
- DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
- }}
+ clockStart = clock();
+ while (clock() == clockStart);
+ clockStart = clock();
+ for (nbRounds=0; BMK_clockSpan(clockStart) < timeLoop; nbRounds++) {
+ benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
+ if (ZSTD_isError(benchResult)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); }
+ }
+ averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
+ if (averageTime < bestTime) bestTime = averageTime;
+ DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
+ } }
DISPLAY("%2u\n", benchNb);
_cleanOut:
@@ -466,7 +475,7 @@ int main(int argc, const char** argv)
switch(argument[0])
{
/* Display help on usage */
- case 'h' :
+ case 'h':
case 'H': return usage_advanced(exename);
/* Pause at the end (hidden option) */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ae8450e40..f8110498d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -27,7 +27,7 @@
#include /* clock_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
-#include "error_public.h" /* ZSTD_getErrorCode */
+#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
#include "mem.h"
@@ -523,7 +523,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
/* notification */
if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
- else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
+ else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed);
{ U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index f253fa39c..5eabcba2b 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -302,7 +302,8 @@ const char* g_stratName[] = { "ZSTD_fast ",
"ZSTD_lazy ",
"ZSTD_lazy2 ",
"ZSTD_btlazy2",
- "ZSTD_btopt " };
+ "ZSTD_btopt ",
+ "ZSTD_btopt2 "};
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
{
@@ -442,7 +443,7 @@ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters par
g_params.chainLog = 0, g_params.searchLog = 0;
if (params.strategy == ZSTD_dfast)
g_params.searchLog = 0;
- if (params.strategy != ZSTD_btopt )
+ if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btopt2)
g_params.targetLength = 0;
return &g_params;
}
@@ -548,7 +549,7 @@ static ZSTD_compressionParameters randomParams(void)
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
- p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
+ p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt2 +1));
validated = !ZSTD_isError(ZSTD_checkCParams(p));
}
return p;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index d94d8fab9..ad70538a3 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -81,6 +81,11 @@ $ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
$ZSTD -dc - < tmp.zst > $INTOVOID
$ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
$ZSTD -d - < tmp.zst > $INTOVOID
+$ECHO "test : impose memory limitation (must fail)"
+$ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed"
+$ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
+$ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
+$ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
$ECHO "test : overwrite protection"
$ZSTD -q tmp && die "overwrite check failed!"
$ECHO "test : force overwrite"
@@ -253,6 +258,17 @@ $ZSTD -t --rm tmp1.zst
ls -ls tmp1.zst # check file is still present
+$ECHO "\n**** benchmark mode tests **** "
+
+$ECHO "bench one file"
+./datagen > tmp1
+$ZSTD -bi1 tmp1
+$ECHO "bench multiple levels"
+$ZSTD -i1b1e3 tmp1
+$ECHO "with recursive and quiet modes"
+$ZSTD -rqi1b1e3 tmp1
+
+
$ECHO "\n**** zstd round-trip tests **** "
roundTripTest
diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py
index 56b4d46b9..23d4f477c 100755
--- a/tests/test-zstd-speed.py
+++ b/tests/test-zstd-speed.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
#
# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
@@ -21,7 +21,7 @@ import time
import traceback
import hashlib
-script_version = 'v1.0.1 (2016-09-15)'
+script_version = 'v1.1.1 (2016-10-28)'
default_repo_url = 'https://github.com/facebook/zstd.git'
working_dir_name = 'speedTest'
working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest
@@ -31,7 +31,7 @@ pid = str(os.getpid())
verbose = False
clang_version = "unknown"
gcc_version = "unknown"
-
+args = None
def hashfile(hasher, fname, blocksize=65536):
@@ -48,17 +48,20 @@ def log(text):
def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
if print_command:
log("> " + command)
- popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
- shell=param_shell, cwd=execute.cwd)
- stdout = popen.communicate()[0]
- stdout_lines = stdout.splitlines()
+ popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd)
+ stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout)
+ stderr_lines = stderr_lines.decode("utf-8")
+ stdout_lines = stdout_lines.decode("utf-8")
if print_output:
- print('\n'.join(stdout_lines))
+ if stdout_lines:
+ print(stdout_lines)
+ if stderr_lines:
+ print(stderr_lines)
if popen.returncode is not None and popen.returncode != 0:
- if not print_output and print_error:
- print('\n'.join(stdout_lines))
- raise RuntimeError('\n'.join(stdout_lines))
- return stdout_lines
+ if stderr_lines and not print_output and print_error:
+ print(stderr_lines)
+ raise RuntimeError(stdout_lines + stderr_lines)
+ return (stdout_lines + stderr_lines).splitlines()
execute.cwd = None
@@ -183,8 +186,10 @@ def update_config_file(branch, commit):
last_commit = None
commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
if os.path.isfile(commitFileName):
- last_commit = file(commitFileName, 'r').read()
- file(commitFileName, 'w').write(commit)
+ with open(commitFileName, 'r') as infile:
+ last_commit = infile.read()
+ with open(commitFileName, 'w') as outfile:
+ outfile.write(commit)
return last_commit
@@ -199,7 +204,7 @@ def double_check(branch, commit, args, executableName, md5sum, compilerVersion,
def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
- local_branch = string.split(branch, '/')[1]
+ local_branch = branch.split('/')[1]
version = local_branch.rpartition('-')[2] + '_' + commit
if not args.dry_run:
execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version +
@@ -255,6 +260,7 @@ if __name__ == '__main__':
parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300)
+ parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800)
parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False)
args = parser.parse_args()
@@ -301,6 +307,7 @@ if __name__ == '__main__':
print("ratioLimit=%s" % args.ratioLimit)
print("lastCLevel=%s" % args.lastCLevel)
print("sleepTime=%s" % args.sleepTime)
+ print("timeout=%s" % args.timeout)
print("dry_run=%s" % args.dry_run)
print("verbose=%s" % args.verbose)
print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))
@@ -323,10 +330,18 @@ if __name__ == '__main__':
exit(1)
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
- file(pidfile, 'w').write(pid)
+ with open(pidfile, 'w') as the_file:
+ the_file.write(pid)
+ branch = ""
+ commit = ""
+ first_time = True
while True:
try:
+ if first_time:
+ first_time = False
+ else:
+ time.sleep(args.sleepTime)
loadavg = os.getloadavg()[0]
if (loadavg <= args.maxLoadAvg):
branches = git_get_branches()
@@ -344,13 +359,11 @@ if __name__ == '__main__':
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
if verbose:
log("sleep for %s seconds" % args.sleepTime)
- time.sleep(args.sleepTime)
except Exception as e:
stack = traceback.format_exc()
email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
send_email(args.emails, email_topic, stack, have_mutt, have_mail)
print(stack)
- time.sleep(args.sleepTime)
except KeyboardInterrupt:
os.unlink(pidfile)
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8486013c2..aa119e636 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -77,7 +77,7 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart)
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
-# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *seedPtr;
@@ -281,12 +281,53 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
}
DISPLAYLEVEL(4, "OK \n");
+ /* CDict scenario */
+ DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++);
+ { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1);
+ size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
+ if (ZSTD_isError(initError)) goto _output_error;
+ cSize = 0;
+ outBuff.dst = compressedBuffer;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = CNBufferSize;
+ inBuff.pos = 0;
+ { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+ if (ZSTD_isError(r)) goto _output_error; }
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ if (r != 0) goto _output_error; } /* error, or some data not flushed */
+ cSize = outBuff.pos;
+ ZSTD_freeCDict(cdict);
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+ }
+
DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++);
{ size_t const s = ZSTD_sizeof_CStream(zc);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
}
+ /* DDict scenario */
+ DISPLAYLEVEL(4, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize);
+ { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, 128 KB);
+ size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
+ if (ZSTD_isError(initError)) goto _output_error;
+ inBuff.src = compressedBuffer;
+ inBuff.size = cSize;
+ inBuff.pos = 0;
+ outBuff.dst = decodedBuffer;
+ outBuff.size = CNBufferSize;
+ outBuff.pos = 0;
+ { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+ if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
+ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
+ if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
+ ZSTD_freeDDict(ddict);
+ DISPLAYLEVEL(4, "OK \n");
+ }
+
/* test ZSTD_setDStreamParameter() resilience */
DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
{ size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */
@@ -411,8 +452,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
size_t maxTestSize;
/* init */
- DISPLAYUPDATE(2, "\r%6u", testNb);
- if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
+ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
+ else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
@@ -511,7 +552,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
/* multi - fragments decompression test */
if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
- CHECK( ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+ CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
} else
ZSTD_initDStream_usingDict(zd, dict, dictSize);
{ size_t decompressionResult = 1;
diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore
index f197c8cc6..1fd8f416c 100644
--- a/zlibWrapper/.gitignore
+++ b/zlibWrapper/.gitignore
@@ -1,26 +1,10 @@
-# Object files
-*.o
-*.ko
-
-# Libraries
-*.lib
-*.a
-
-# Shared objects (inc. Windows DLLs)
-*.dll
-*.so
-*.so.*
-*.dylib
-
-# Executables
-*.exe
-*.out
-*.app
-
# Default result files
_*
-example
-example_zstd
+example.*
+example_zstd.*
+fitblk.*
+fitblk_zstd.*
+zwrapbench.*
foo.gz
# Misc files
diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index 69c976fa5..0e4ca9e6f 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -14,6 +14,7 @@ ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a
ZLIBWRAPPER_PATH = .
EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs
+TEST_FILE = ../doc/zstd_compression_format.md
CC ?= gcc
CFLAGS ?= -O3
CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99
@@ -27,11 +28,11 @@ all: clean fitblk example zwrapbench
test: example fitblk example_zstd fitblk_zstd zwrapbench
./example
./example_zstd
- ./fitblk 10240 <../zstd_compression_format.md
- ./fitblk 40960 <../zstd_compression_format.md
- ./fitblk_zstd 10240 <../zstd_compression_format.md
- ./fitblk_zstd 40960 <../zstd_compression_format.md
- ./zwrapbench -qb3B1K ../zstd_compression_format.md
+ ./fitblk 10240 <$(TEST_FILE)
+ ./fitblk 40960 <$(TEST_FILE)
+ ./fitblk_zstd 10240 <$(TEST_FILE)
+ ./fitblk_zstd 40960 <$(TEST_FILE)
+ ./zwrapbench -qb3B1K $(TEST_FILE)
./zwrapbench -rqb1e5 ../lib ../programs ../tests
#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
@@ -40,11 +41,11 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
@echo "\n ---- valgrind tests ----"
$(VALGRIND) ./example
$(VALGRIND) ./example_zstd
- $(VALGRIND) ./fitblk 10240 <../zstd_compression_format.md
- $(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md
- $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md
- $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md
- $(VALGRIND) ./zwrapbench -qb3B1K ../zstd_compression_format.md
+ $(VALGRIND) ./fitblk 10240 <$(TEST_FILE)
+ $(VALGRIND) ./fitblk 40960 <$(TEST_FILE)
+ $(VALGRIND) ./fitblk_zstd 10240 <$(TEST_FILE)
+ $(VALGRIND) ./fitblk_zstd 40960 <$(TEST_FILE)
+ $(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE)
$(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests
.c.o:
diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md
index 427cdbe09..5fefac1d3 100644
--- a/zlibWrapper/README.md
+++ b/zlibWrapper/README.md
@@ -71,7 +71,7 @@ The script used for compilation can be found at [zlibWrapper/Makefile](Makefile)
The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper.
The benchmark is conducted using given filenames or synthetic data if filenames are not provided.
-The files are read into memory and joined together.
+The files are read into memory and processed independently.
It makes benchmark more precise as it eliminates I/O overhead.
Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the -r option.
One can select compression levels starting from `-b` and ending with `-e`. The `-i` parameter selects minimal time used for each of tested levels.
@@ -119,7 +119,7 @@ In our example (the last 2 lines) it gives 4% better compression speed and 5% be
#### Compatibility issues
-After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into strm->msg and return Z_STREAM_ERROR.
+After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into `strm->msg` and return Z_STREAM_ERROR.
Supported methods:
- deflateInit