ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski
2016-11-18 11:47:01 +01:00
parent 2558b4cdbc 0d761dbe95
commit 5a17223691
19 changed files with 154 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2 -1
View File
@@ -12,12 +12,13 @@ matrix:
os: linux os: linux
sudo: false sudo: false
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean" - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
os: linux os: linux
sudo: false sudo: false
language: cpp language: cpp
install: install:
- export CXX="g++-4.8" CC="gcc-4.8" - export CXX="g++-4.8" CC="gcc-4.8"
- export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
addons: addons:
apt: apt:
sources: sources:
NEWS
+3 -1
View File
@@ -1,5 +1,7 @@
v1.1.2 v1.1.2
New : cli : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd) cli : new : preserve file attributes, by Przemyslaw Skibinski
cli : fixed : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd)
API : changed : zbuff prototypes now generate deprecation warnings
v1.1.1 v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
appveyor.yml
+3 -2
View File
@@ -25,7 +25,7 @@ install:
- MKDIR bin - MKDIR bin
- if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
- if [%COMPILER%]==[gcc] ( - if [%COMPILER%]==[gcc] (
SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe &&
@@ -51,7 +51,8 @@ build_script:
ECHO *** Building pzstd for %PLATFORM% && ECHO *** Building pzstd for %PLATFORM% &&
ECHO *** && ECHO *** &&
make -C contrib\pzstd googletest-mingw64 && make -C contrib\pzstd googletest-mingw64 &&
make -C contrib\pzstd all && make -C contrib\pzstd pzstd.exe &&
make -C contrib\pzstd tests &&
make -C contrib\pzstd check && make -C contrib\pzstd check &&
make -C contrib\pzstd clean make -C contrib\pzstd clean
) )
contrib/pzstd/Makefile
+11 -7
View File
@@ -34,7 +34,7 @@ LDFLAGS ?=
PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
GTEST_INC = -isystem googletest/googletest/include GTEST_INC = -isystem googletest/googletest/include
PZSTD_CPPFLAGS = $(PZSTD_INC) $(GTEST_INC) PZSTD_CPPFLAGS = $(PZSTD_INC)
PZSTD_CCXXFLAGS = PZSTD_CCXXFLAGS =
PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS) PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS)
PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS)
@@ -47,10 +47,10 @@ ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
# gtest libraries need to go before "-lpthread" because they depend on it. # gtest libraries need to go before "-lpthread" because they depend on it.
GTEST_LIB = -L googletest/build/googlemock/gtest GTEST_LIB = -L googletest/build/googlemock/gtest
LIBS = $(GTEST_LIB) -lpthread LIBS =
# Compilation commands # Compilation commands
LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -o $@ LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -lpthread -o $@
CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@ CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@
CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@ CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@
@@ -109,7 +109,7 @@ uninstall:
# Targets for many different builds # Targets for many different builds
.PHONY: all .PHONY: all
all: PZSTD_CPPFLAGS += -DNDEBUG all: PZSTD_CPPFLAGS += -DNDEBUG
all: pzstd$(EXT) tests roundtrip all: pzstd$(EXT)
.PHONY: debug .PHONY: debug
debug: EXTRA_FLAGS += -g debug: EXTRA_FLAGS += -g
@@ -130,7 +130,7 @@ ubsan: debug
.PHONY: all32 .PHONY: all32
all32: EXTRA_FLAGS += -m32 all32: EXTRA_FLAGS += -m32
all32: all all32: all tests roundtrip
.PHONY: debug32 .PHONY: debug32
debug32: EXTRA_FLAGS += -m32 debug32: EXTRA_FLAGS += -m32
@@ -177,12 +177,14 @@ test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \
Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
$(LD_COMMAND) $(LD_COMMAND)
test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
test/%Test$(EXT): LIBS += -lgtest -lgtest_main
test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \ test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \
SkippableFrame.o $(ZSTDDIR)/libzstd.a SkippableFrame.o $(ZSTDDIR)/libzstd.a
$(LD_COMMAND) $(LD_COMMAND)
utils/test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main utils/test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
utils/test/%Test$(EXT): LIBS += -lgtest -lgtest_main
utils/test/%Test$(EXT): utils/test/%Test.o utils/test/%Test$(EXT): utils/test/%Test.o
$(LD_COMMAND) $(LD_COMMAND)
@@ -233,10 +235,12 @@ $(PROGDIR)/%.o: $(PROGDIR)/%.c
$(CXX_COMMAND) $(CXX_COMMAND)
$(POSTCOMPILE) $(POSTCOMPILE)
test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
test/%.o: test/%.cpp test/%.o: test/%.cpp
$(CXX_COMMAND) $(CXX_COMMAND)
$(POSTCOMPILE) $(POSTCOMPILE)
utils/test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
utils/test/%.o: utils/test/%.cpp utils/test/%.o: utils/test/%.cpp
$(CXX_COMMAND) $(CXX_COMMAND)
$(POSTCOMPILE) $(POSTCOMPILE)
contrib/pzstd/Pzstd.cpp
+8 -9
View File
@@ -341,15 +341,9 @@ static size_t calculateStep(
std::uintmax_t size, std::uintmax_t size,
size_t numThreads, size_t numThreads,
const ZSTD_parameters &params) { const ZSTD_parameters &params) {
size_t step = size_t{1} << (params.cParams.windowLog + 2); (void)size;
// If file size is known, see if a smaller step will spread work more evenly (void)numThreads;
if (size != 0) { return size_t{1} << (params.cParams.windowLog + 2);
const std::uintmax_t newStep = size / numThreads;
if (newStep != 0 && newStep <= std::numeric_limits<size_t>::max()) {
step = std::min(step, static_cast<size_t>(newStep));
}
}
return step;
} }
namespace { namespace {
@@ -401,6 +395,7 @@ std::uint64_t asyncCompressChunks(
// Break the input up into chunks of size `step` and compress each chunk // Break the input up into chunks of size `step` and compress each chunk
// independently. // independently.
size_t step = calculateStep(size, numThreads, params); size_t step = calculateStep(size, numThreads, params);
state.log(DEBUG, "Chosen frame size: %zu\n", step);
auto status = FileStatus::Continue; auto status = FileStatus::Continue;
while (status == FileStatus::Continue && !state.errorHolder.hasError()) { while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
// Make a new input queue that we will put the chunk's input data into. // Make a new input queue that we will put the chunk's input data into.
@@ -415,6 +410,7 @@ std::uint64_t asyncCompressChunks(
}); });
// Pass the output queue to the writer thread. // Pass the output queue to the writer thread.
chunks.push(std::move(out)); chunks.push(std::move(out));
state.log(VERBOSE, "Starting a new frame\n");
// Fill the input queue for the compression job we just started // Fill the input queue for the compression job we just started
status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead); status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead);
} }
@@ -551,11 +547,14 @@ std::uint64_t asyncDecompressFrames(
if (frameSize == 0) { if (frameSize == 0) {
// We hit a non SkippableFrame ==> not compressed by pzstd or corrupted // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted
// Pass the rest of the source to this decompression task // Pass the rest of the source to this decompression task
state.log(VERBOSE,
"Input not in pzstd format, falling back to serial decompression\n");
while (status == FileStatus::Continue && !state.errorHolder.hasError()) { while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead); status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead);
} }
break; break;
} }
state.log(VERBOSE, "Decompressing a frame of size %zu", frameSize);
// Fill the input queue for the decompression job we just started // Fill the input queue for the decompression job we just started
status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead); status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead);
} }
contrib/pzstd/Pzstd.h
+10 -2
View File
@@ -40,7 +40,8 @@ class SharedState {
if (!options.decompress) { if (!options.decompress) {
auto parameters = options.determineParameters(); auto parameters = options.determineParameters();
cStreamPool.reset(new ResourcePool<ZSTD_CStream>{ cStreamPool.reset(new ResourcePool<ZSTD_CStream>{
[parameters]() -> ZSTD_CStream* { [this, parameters]() -> ZSTD_CStream* {
this->log(VERBOSE, "Creating new ZSTD_CStream\n");
auto zcs = ZSTD_createCStream(); auto zcs = ZSTD_createCStream();
if (zcs) { if (zcs) {
auto err = ZSTD_initCStream_advanced( auto err = ZSTD_initCStream_advanced(
@@ -57,7 +58,8 @@ class SharedState {
}}); }});
} else { } else {
dStreamPool.reset(new ResourcePool<ZSTD_DStream>{ dStreamPool.reset(new ResourcePool<ZSTD_DStream>{
[]() -> ZSTD_DStream* { [this]() -> ZSTD_DStream* {
this->log(VERBOSE, "Creating new ZSTD_DStream\n");
auto zds = ZSTD_createDStream(); auto zds = ZSTD_createDStream();
if (zds) { if (zds) {
auto err = ZSTD_initDStream(zds); auto err = ZSTD_initDStream(zds);
@@ -74,6 +76,12 @@ class SharedState {
} }
} }
~SharedState() {
// The resource pools have references to this, so destroy them first.
cStreamPool.reset();
dStreamPool.reset();
}
Logger log; Logger log;
ErrorHolder errorHolder; ErrorHolder errorHolder;
std::unique_ptr<ResourcePool<ZSTD_CStream>> cStreamPool; std::unique_ptr<ResourcePool<ZSTD_CStream>> cStreamPool;
contrib/pzstd/README.md
+3 -2
View File
@@ -51,5 +51,6 @@ Pigz cannot do parallel decompression, it simply does each of reading, decompres
## Tests ## Tests
Tests require that you have [gtest](https://github.com/google/googletest) installed. Tests require that you have [gtest](https://github.com/google/googletest) installed.
Modify `GTEST_INC` and `GTEST_LIB` in `test/Makefile` and `utils/test/Makefile` to work for your install of gtest. Set `GTEST_INC` and `GTEST_LIB` in `Makefile` to specify the location of the gtest headers and libraries.
Then run `make test` in the `contrib/pzstd` directory. Alternatively, run `make googletest`, which will clone googletest and build it.
Run `make tests && make check` to run tests.
lib/Makefile
+2 -1
View File
@@ -23,7 +23,7 @@ PREFIX ?= /usr/local
LIBDIR ?= $(PREFIX)/lib LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include INCLUDEDIR=$(PREFIX)/include
CPPFLAGS= -I. -I./common CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=XXH_
CFLAGS ?= -O3 CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
@@ -110,6 +110,7 @@ install: libzstd.a libzstd libzstd.pc
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed @echo zstd static and shared library installed
lib/README.md
+3 -3
View File
@@ -46,9 +46,9 @@ Other optional functionalities provided are :
#### Obsolete streaming API #### Obsolete streaming API
Streaming is now provided within `zstd.h`. Streaming is now provided within `zstd.h`.
Older streaming API is still provided within `common/zbuff.h`. Older streaming API is still available within `common/zbuff.h`.
It is considered obsolete, and will be removed in a future version. It is now deprecated, and will be removed in a future version.
Consider migrating towards newer streaming API. Consider migrating towards newer streaming API in `zstd.h`.
#### Miscellaneous #### Miscellaneous
lib/common/zbuff.h
+47 -26
View File
@@ -9,11 +9,11 @@
/* *************************************************************** /* ***************************************************************
* NOTES/WARNINGS * NOTES/WARNINGS
*****************************************************************/ ******************************************************************/
/* The streaming API defined here will soon be deprecated by the /* The streaming API defined here is deprecated.
* new one in 'zstd.h'; consider migrating towards newer streaming * Consider migrating towards ZSTD_compressStream() API in `zstd.h`
* API. See 'lib/README.md'. * See 'lib/README.md'.
*****************************************************************/ *****************************************************************/
#ifndef ZSTD_BUFFERED_H_23987 #ifndef ZSTD_BUFFERED_H_23987
#define ZSTD_BUFFERED_H_23987 #define ZSTD_BUFFERED_H_23987
@@ -39,6 +39,27 @@ extern "C" {
# define ZSTDLIB_API # define ZSTDLIB_API
#endif #endif
/* Deprecation warnings */
/* Should these warnings be a problem,
it is generally possible to disable them,
typically with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual.
Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
# define ZBUFF_DEPRECATED(message) /* disable deprecation warnings */
#else
# if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
# define ZBUFF_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZBUFF_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define ZBUFF_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler")
# define ZBUFF_DEPRECATED(message)
# endif
#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */
/* ************************************* /* *************************************
* Streaming functions * Streaming functions
@@ -50,15 +71,15 @@ extern "C" {
* frames created by one can be decoded by the other one */ * frames created by one can be decoded by the other one */
typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void);
ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
/*-************************************************* /*-*************************************************
* Streaming compression - howto * Streaming compression - howto
@@ -102,13 +123,13 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap
typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void);
ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
void* dst, size_t* dstCapacityPtr, void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr); const void* src, size_t* srcSizePtr);
@@ -141,15 +162,15 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
/* ************************************* /* *************************************
* Tool functions * Tool functions
***************************************/ ***************************************/
ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode);
ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations. /** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */ * These sizes are just hints, they tend to offer better latency */
ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void);
#ifdef ZBUFF_STATIC_LINKING_ONLY #ifdef ZBUFF_STATIC_LINKING_ONLY
@@ -169,15 +190,15 @@ ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
/*--- Custom memory allocator ---*/ /*--- Custom memory allocator ---*/
/*! ZBUFF_createCCtx_advanced() : /*! ZBUFF_createCCtx_advanced() :
* Create a ZBUFF compression context using external alloc and free functions */ * Create a ZBUFF compression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZBUFF_createDCtx_advanced() : /*! ZBUFF_createDCtx_advanced() :
* Create a ZBUFF decompression context using external alloc and free functions */ * Create a ZBUFF decompression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
/*--- Advanced Streaming Initialization ---*/ /*--- Advanced Streaming Initialization ---*/
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTD_parameters params, unsigned long long pledgedSrcSize);
lib/decompress/zstd_decompress.c
+9 -13
View File
@@ -104,7 +104,6 @@ struct ZSTD_DCtx_s
U32 dictID; U32 dictID;
const BYTE* litPtr; const BYTE* litPtr;
ZSTD_customMem customMem; ZSTD_customMem customMem;
size_t litBufSize;
size_t litSize; size_t litSize;
size_t rleSize; size_t rleSize;
BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
@@ -429,10 +428,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litEntropy = 1; dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
@@ -459,13 +458,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected); if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize); memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize; return lhSize+litSize;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+lhSize; dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+litSize; return lhSize+litSize;
} }
@@ -490,9 +488,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
break; break;
} }
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+1; return lhSize+1;
} }
@@ -867,7 +864,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
FORCE_NOINLINE FORCE_NOINLINE
size_t ZSTD_execSequenceLast7(BYTE* op, size_t ZSTD_execSequenceLast7(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_w, const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; BYTE* const oLitEnd = op + sequence.litLength;
@@ -879,7 +876,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */
/* copy literals */ /* copy literals */
@@ -914,7 +911,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
FORCE_INLINE FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op, size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_w, const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; BYTE* const oLitEnd = op + sequence.litLength;
@@ -926,8 +923,8 @@ size_t ZSTD_execSequence(BYTE* op,
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit_w, base, vBase, dictEnd); if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
/* copy Literals */ /* copy Literals */
ZSTD_copy8(op, *litPtr); ZSTD_copy8(op, *litPtr);
@@ -1002,7 +999,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart; BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const vBase = (const BYTE*) (dctx->vBase);
@@ -1028,7 +1024,7 @@ static size_t ZSTD_decompressSequences(
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
nbSeq--; nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState); { seq_t const sequence = ZSTD_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} } } }
lib/legacy/zstd_v02.c
+5 -9
View File
@@ -2868,7 +2868,6 @@ struct ZSTD_DCtx_s
blockType_t bType; blockType_t bType;
U32 phase; U32 phase;
const BYTE* litPtr; const BYTE* litPtr;
size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */ }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2940,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
size_t litSize = BLOCKSIZE; size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize); const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */ return readSize; /* works if it's an error too */
} }
case IS_RAW: case IS_RAW:
@@ -2952,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
if (litSize > srcSize-3) return ERROR(corruption_detected); if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize); memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3; return litSize+3;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+3; dctx->litPtr = istart+3;
dctx->litBufSize = srcSize-3;
dctx->litSize = litSize; dctx->litSize = litSize;
return litSize+3; return litSize+3;
} }
@@ -2966,9 +2964,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
{ {
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected); if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[3], litSize); memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize; dctx->litSize = litSize;
return 4; return 4;
} }
@@ -3175,7 +3172,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* checks */ /* checks */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
/* copy Literals */ /* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3241,7 +3238,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength; size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litMax = litPtr + dctx->litBufSize;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq; int nbSeq;
const BYTE* dumps; const BYTE* dumps;
@@ -3277,7 +3273,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize; size_t oneSeqSize;
nbSeq--; nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState); ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend); oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} }
lib/legacy/zstd_v03.c
+5 -9
View File
@@ -2509,7 +2509,6 @@ struct ZSTD_DCtx_s
blockType_t bType; blockType_t bType;
U32 phase; U32 phase;
const BYTE* litPtr; const BYTE* litPtr;
size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */ }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2581,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
size_t litSize = BLOCKSIZE; size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize); const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */ return readSize; /* works if it's an error too */
} }
case IS_RAW: case IS_RAW:
@@ -2593,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
if (litSize > srcSize-3) return ERROR(corruption_detected); if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize); memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3; return litSize+3;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+3; dctx->litPtr = istart+3;
dctx->litBufSize = srcSize-3;
dctx->litSize = litSize; dctx->litSize = litSize;
return litSize+3; return litSize+3;
} }
@@ -2607,9 +2605,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
{ {
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected); if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[3], litSize); memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize; dctx->litSize = litSize;
return 4; return 4;
} }
@@ -2816,7 +2813,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* checks */ /* checks */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
/* copy Literals */ /* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -2882,7 +2879,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength; size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litMax = litPtr + dctx->litBufSize;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq; int nbSeq;
const BYTE* dumps; const BYTE* dumps;
@@ -2918,7 +2914,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize; size_t oneSeqSize;
nbSeq--; nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState); ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend); oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} }
lib/legacy/zstd_v04.c
+6 -10
View File
@@ -2706,7 +2706,6 @@ struct ZSTDv04_Dctx_s
blockType_t bType; blockType_t bType;
ZSTD_dStage stage; ZSTD_dStage stage;
const BYTE* litPtr; const BYTE* litPtr;
size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
BYTE headerBuffer[ZSTD_frameHeaderSize_max]; BYTE headerBuffer[ZSTD_frameHeaderSize_max];
@@ -2847,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
size_t litSize = BLOCKSIZE; size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize); const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */ return readSize; /* works if it's an error too */
} }
case IS_RAW: case IS_RAW:
@@ -2859,22 +2858,20 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize > srcSize-3) return ERROR(corruption_detected); if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize); memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3; return litSize+3;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+3; dctx->litPtr = istart+3;
dctx->litBufSize = srcSize-3;
dctx->litSize = litSize; dctx->litSize = litSize;
return litSize+3; } return litSize+3; }
case IS_RLE: case IS_RLE:
{ {
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected); if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[3], litSize); memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize; dctx->litSize = litSize;
return 4; return 4;
} }
@@ -3069,7 +3066,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTD_execSequence(BYTE* op, static size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8, const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@@ -3084,7 +3081,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* check */ /* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
/* copy Literals */ /* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3167,7 +3164,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength; size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq; int nbSeq;
const BYTE* dumps; const BYTE* dumps;
@@ -3206,7 +3202,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize; size_t oneSeqSize;
nbSeq--; nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState); ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} }
lib/legacy/zstd_v05.c
+7 -11
View File
@@ -2731,7 +2731,6 @@ struct ZSTDv05_DCtx_s
ZSTDv05_dStage stage; ZSTDv05_dStage stage;
U32 flagStaticTables; U32 flagStaticTables;
const BYTE* litPtr; const BYTE* litPtr;
size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH]; BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv05_frameHeaderSize_max]; BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
@@ -2978,8 +2977,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case IS_PCH: case IS_PCH:
@@ -3002,8 +3001,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
if (HUFv05_isError(errorCode)) return ERROR(corruption_detected); if (HUFv05_isError(errorCode)) return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case IS_RAW: case IS_RAW:
@@ -3028,13 +3027,12 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected); if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize); memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize; return lhSize+litSize;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+lhSize; dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+litSize; return lhSize+litSize;
} }
@@ -3057,9 +3055,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break; break;
} }
if (litSize > BLOCKSIZE) return ERROR(corruption_detected); if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+1; return lhSize+1;
} }
@@ -3289,7 +3286,7 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTDv05_execSequence(BYTE* op, static size_t ZSTDv05_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8, const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@@ -3304,7 +3301,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
/* check */ /* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
/* copy Literals */ /* copy Literals */
ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3378,7 +3375,6 @@ static size_t ZSTDv05_decompressSequences(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength; size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq; int nbSeq;
const BYTE* dumps; const BYTE* dumps;
@@ -3416,7 +3412,7 @@ static size_t ZSTDv05_decompressSequences(
size_t oneSeqSize; size_t oneSeqSize;
nbSeq--; nbSeq--;
ZSTDv05_decodeSequence(&sequence, &seqState); ZSTDv05_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv05_isError(oneSeqSize)) return oneSeqSize; if (ZSTDv05_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} }
lib/legacy/zstd_v06.c
+7 -11
View File
@@ -2893,7 +2893,6 @@ struct ZSTDv06_DCtx_s
ZSTDv06_dStage stage; ZSTDv06_dStage stage;
U32 flagRepeatTable; U32 flagRepeatTable;
const BYTE* litPtr; const BYTE* litPtr;
size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[ZSTDv06_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; BYTE litBuffer[ZSTDv06_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
@@ -3170,8 +3169,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case IS_PCH: case IS_PCH:
@@ -3192,8 +3191,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
if (HUFv06_isError(errorCode)) return ERROR(corruption_detected); if (HUFv06_isError(errorCode)) return ERROR(corruption_detected);
} }
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case IS_RAW: case IS_RAW:
@@ -3217,13 +3216,12 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected); if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize); memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize; return lhSize+litSize;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+lhSize; dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+litSize; return lhSize+litSize;
} }
@@ -3245,9 +3243,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
break; break;
} }
if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected); if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+1; return lhSize+1;
} }
@@ -3438,7 +3435,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
size_t ZSTDv06_execSequence(BYTE* op, size_t ZSTDv06_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8, const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; BYTE* const oLitEnd = op + sequence.litLength;
@@ -3451,7 +3448,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
/* check */ /* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
/* copy Literals */ /* copy Literals */
ZSTDv06_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ ZSTDv06_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3523,7 +3520,6 @@ static size_t ZSTDv06_decompressSequences(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart; BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
FSEv06_DTable* DTableLL = dctx->LLTable; FSEv06_DTable* DTableLL = dctx->LLTable;
FSEv06_DTable* DTableML = dctx->MLTable; FSEv06_DTable* DTableML = dctx->MLTable;
@@ -3567,7 +3563,7 @@ static size_t ZSTDv06_decompressSequences(
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset); pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
#endif #endif
{ size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); { size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv06_isError(oneSeqSize)) return oneSeqSize; if (ZSTDv06_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} } } }
lib/legacy/zstd_v07.c
+7 -11
View File
@@ -3021,7 +3021,6 @@ struct ZSTDv07_DCtx_s
U32 dictID; U32 dictID;
const BYTE* litPtr; const BYTE* litPtr;
ZSTDv07_customMem customMem; ZSTDv07_customMem customMem;
size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX];
@@ -3395,9 +3394,9 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litEntropy = 1; dctx->litEntropy = 1;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case lbt_repeat: case lbt_repeat:
@@ -3418,8 +3417,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
if (HUFv07_isError(errorCode)) return ERROR(corruption_detected); if (HUFv07_isError(errorCode)) return ERROR(corruption_detected);
} }
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case lbt_raw: case lbt_raw:
@@ -3443,13 +3442,12 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected); if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize); memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize; return lhSize+litSize;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+lhSize; dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+litSize; return lhSize+litSize;
} }
@@ -3471,9 +3469,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
break; break;
} }
if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+1; return lhSize+1;
} }
@@ -3662,7 +3659,7 @@ static seq_t ZSTDv07_decodeSequence(seqState_t* seqState)
static static
size_t ZSTDv07_execSequence(BYTE* op, size_t ZSTDv07_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_w, const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; BYTE* const oLitEnd = op + sequence.litLength;
@@ -3674,7 +3671,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
/* check */ /* check */
if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
/* copy Literals */ /* copy Literals */
ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
@@ -3746,7 +3743,6 @@ static size_t ZSTDv07_decompressSequences(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart; BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
FSEv07_DTable* DTableLL = dctx->LLTable; FSEv07_DTable* DTableLL = dctx->LLTable;
FSEv07_DTable* DTableML = dctx->MLTable; FSEv07_DTable* DTableML = dctx->MLTable;
@@ -3776,7 +3772,7 @@ static size_t ZSTDv07_decompressSequences(
for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) { for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) {
nbSeq--; nbSeq--;
{ seq_t const sequence = ZSTDv07_decodeSequence(&seqState); { seq_t const sequence = ZSTDv07_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize; if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} } } }
programs/zstdcli.c
+1 -1
View File
@@ -21,7 +21,7 @@
/*-************************************ /*-************************************
* Includes * Dependencies
**************************************/ **************************************/
#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ #include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
#include <string.h> /* strcmp, strlen */ #include <string.h> /* strcmp, strlen */
tests/Makefile
+6 -9
View File
@@ -14,10 +14,8 @@
# paramgrill : parameter tester for zstd # paramgrill : parameter tester for zstd
# test-zstd-speed.py : script for testing zstd speed difference between commits # test-zstd-speed.py : script for testing zstd speed difference between commits
# versionsTest : compatibility test between zstd versions stored on Github (v0.1+) # versionsTest : compatibility test between zstd versions stored on Github (v0.1+)
# zbufftest : Test tool, to check ZBUFF integrity on target platform
# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode
# zstreamtest : Fuzzer test tool for zstd streaming API # zstreamtest : Fuzzer test tool for zstd streaming API
# zbufftest32: Same as zstreamtest, but forced to compile in 32-bits mode # zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
# ########################################################################## # ##########################################################################
DESTDIR?= DESTDIR?=
@@ -63,9 +61,9 @@ ZSTDRTTEST= --test-large-data
default: fullbench default: fullbench
all: fullbench fuzzer zbufftest zstreamtest paramgrill datagen all: fullbench fuzzer zstreamtest paramgrill datagen
all32: fullbench32 fuzzer32 zbufftest32 zstreamtest32 all32: fullbench32 fuzzer32 zstreamtest32
@@ -139,7 +137,7 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
HOST_OS = POSIX HOST_OS = POSIX
valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
valgrindTest: zstd datagen fuzzer fullbench zbufftest valgrindTest: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----" @echo "\n ---- valgrind tests : memory analyzer ----"
$(VALGRIND) ./datagen -g50M > $(VOID) $(VALGRIND) ./datagen -g50M > $(VOID)
$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi $(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
@@ -151,7 +149,6 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest
@rm tmp @rm tmp
$(VALGRIND) ./fuzzer -T1mn -t1 $(VALGRIND) ./fuzzer -T1mn -t1
$(VALGRIND) ./fullbench -i1 $(VALGRIND) ./fullbench -i1
$(VALGRIND) ./zbufftest -T1mn
endif endif
@@ -169,9 +166,9 @@ zstd-playTests: datagen
file $(ZSTD) file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream test: test-zstd test-fullbench test-fuzzer test-zstream
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
test-all: test test32 valgrindTest test-all: test test32 valgrindTest