From 0dd8071e2e96c0ce4a536a4e97cef3ddca6d0a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Thu, 6 Oct 2016 15:13:10 +0100 Subject: [PATCH 01/26] pzstd: fix test failure on headless build Remove this test as pass/fail status is dependent on whether I/O is connected to a tty. It currrently passes on a tty but fails on a package build system for example. --- contrib/pzstd/test/OptionsTest.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp index e7d4b2b3e..b3efe2b7e 100644 --- a/contrib/pzstd/test/OptionsTest.cpp +++ b/contrib/pzstd/test/OptionsTest.cpp @@ -180,12 +180,6 @@ TEST(Options, GetOutputFile) { EXPECT_SUCCESS(options.parse(args.size(), args.data())); EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0])); } - { - Options options; - auto args = makeArray("-o-"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0])); - } { Options options; auto args = makeArray("x", "y", "-o", nullOutput); From 18b51b99c07f5474cb431a934ac2585551ea6e12 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 08:21:09 -0700 Subject: [PATCH 02/26] sync fse --- lib/common/fse.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/common/fse.h b/lib/common/fse.h index 720d54b11..cecb1aef7 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt BIT_flushBits(bitC); } + /* ====== Decompression ====== */ typedef struct { @@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSE_MAX_MEMORY_USAGE 14 -#define FSE_DEFAULT_MEMORY_USAGE 13 +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif /*!FSE_MAX_SYMBOL_VALUE : * Maximum symbol value authorized. * Required for proper stack allocation */ -#define FSE_MAX_SYMBOL_VALUE 255 - +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif /* ************************************************************** * template functions type & suffix From 3e82f2b61add5954301fa819047856d6a12e0dac Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 11 Oct 2016 15:27:44 -0700 Subject: [PATCH 03/26] Remove references to GPLv2 license License headers added to Makefiles were taken from `zstd/Makefile`. --- examples/Makefile | 29 +++++++---------------------- programs/fileio.c | 7 ------- programs/zstdcli.c | 7 ------- tests/Makefile | 25 +++++-------------------- 4 files changed, 12 insertions(+), 56 deletions(-) diff --git a/examples/Makefile b/examples/Makefile index 54602dfe0..8ce6a2589 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,26 +1,11 @@ -# ########################################################################## -# ZSTD educational examples - Makefile -# Copyright (C) Yann Collet 2016 +# ################################################################ +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. # -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ -# ########################################################################## +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ################################################################ # This Makefile presumes libzstd is installed, using `sudo make install` diff --git a/programs/fileio.c b/programs/fileio.c index 56f22fe75..d57ffedd2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -8,13 +8,6 @@ */ -/* - Note : this file is part of zstd command line, which is not library. - The license of ZSTD library is BSD. - The license of this file is GPLv2. -*/ - - /* ************************************* * Tuning options ***************************************/ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fe97f965c..f64909d98 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -8,13 +8,6 @@ */ -/* - Note : this is a user program, not part of libzstd. - The license of libzstd is BSD. - The license of this command line program is GPLv2. -*/ - - /*-************************************ * Tuning parameters **************************************/ diff --git a/tests/Makefile b/tests/Makefile index 3ce9f317e..309b5d402 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,25 +1,10 @@ # ########################################################################## -# ZSTD tests - Makefile -# Copyright (C) Yann Collet 2015-2016 +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. # -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. # ########################################################################## # datagen : Synthetic and parametrable data generator, for tests # fullbench : Precisely measure speed for each zstd inner functions From a17fd7312af6517b0bc63d339c106761b258f628 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 16:41:09 -0700 Subject: [PATCH 04/26] changed error_public.h into zstd_errors.h --- lib/common/error_private.h | 2 +- lib/common/{error_public.h => zstd_errors.h} | 6 +++--- tests/fuzzer.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename lib/common/{error_public.h => zstd_errors.h} (94%) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index d27e15af8..3e99b87e7 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -21,7 +21,7 @@ extern "C" { * Dependencies ******************************************/ #include /* size_t */ -#include "error_public.h" /* enum list */ +#include "zstd_errors.h" /* enum list */ /* **************************************** diff --git a/lib/common/error_public.h b/lib/common/zstd_errors.h similarity index 94% rename from lib/common/error_public.h rename to lib/common/zstd_errors.h index d46abd2c7..9335eaab0 100644 --- a/lib/common/error_public.h +++ b/lib/common/zstd_errors.h @@ -7,8 +7,8 @@ * of patent rights can be found in the PATENTS file in the same directory. */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 #if defined (__cplusplus) extern "C" { @@ -56,4 +56,4 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code); } #endif -#endif /* ERROR_PUBLIC_H_MODULE */ +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ae8450e40..22f034d13 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,7 +27,7 @@ #include /* clock_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "zstd.h" /* ZSTD_VERSION_STRING */ -#include "error_public.h" /* ZSTD_getErrorCode */ +#include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" From 14efab827bd41e574848792a8ace8236a4fa807f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 16:51:29 -0700 Subject: [PATCH 05/26] added zstd_errors.h to include installation --- lib/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 4fb8ed9d2..c3c993e3c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -97,7 +97,8 @@ install: libzstd libzstd.pc @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @install -m 644 zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed @@ -108,7 +109,8 @@ uninstall: $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled From ef2357d0d3edc936dc04b38e3f5d85285907bd14 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 17:24:50 -0700 Subject: [PATCH 06/26] created error_private.c, so that a single list of error strings get included --- lib/common/error_private.c | 42 ++++++++++++++++++++++++++++++++++++++ lib/common/error_private.h | 30 +-------------------------- 2 files changed, 43 insertions(+), 29 deletions(-) create mode 100644 lib/common/error_private.c diff --git a/lib/common/error_private.c b/lib/common/error_private.c new file mode 100644 index 000000000..839f77434 --- /dev/null +++ b/lib/common/error_private.c @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 3e99b87e7..1bc2e4954 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu * Error Strings ******************************************/ -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(version_unsupported): return "Version not supported"; - case PREFIX(parameter_unknown): return "Unknown parameter type"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ ERR_STATIC const char* ERR_getErrorName(size_t code) { From f52cd03e736dba2acfde57e2d94650a4ff7fba73 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 17:29:27 -0700 Subject: [PATCH 07/26] bumped version number --- lib/zstd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd.h b/lib/zstd.h index dd3f5df4c..c2f5f5d4c 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -33,7 +33,7 @@ extern "C" { /*======= Version =======*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str From 71585843998459e636f639165ff8a1ca9d161b62 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 10 Oct 2016 16:19:21 -0700 Subject: [PATCH 08/26] Fix ZSTD_execSequence() edge case --- lib/decompress/zstd_decompress.c | 5 +++++ lib/legacy/zstd_v04.c | 5 +++++ lib/legacy/zstd_v05.c | 5 +++++ lib/legacy/zstd_v06.c | 5 +++++ lib/legacy/zstd_v07.c | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 47b5f42c7..88869d82a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -878,7 +878,12 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index c9dcb94e0..05e40aac5 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 5027e2b8b..96ffceb97 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3312,7 +3312,12 @@ static size_t ZSTDv05_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index d9e89f806..96a84d3e8 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index f4c8073f9..62285238a 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { From 8b70d012f0b1043f966aa132a45c0b327a053fba Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 10:23:53 -0700 Subject: [PATCH 09/26] fix cmake --- build/cmake/lib/CMakeLists.txt | 3 ++- lib/Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index c984145ba..f970fe7e9 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") SET(Sources ${LIBRARY_DIR}/common/entropy_common.c ${LIBRARY_DIR}/common/zstd_common.c + ${LIBRARY_DIR}/common/error_private.c ${LIBRARY_DIR}/common/xxhash.c ${LIBRARY_DIR}/common/fse_decompress.c ${LIBRARY_DIR}/compress/fse_compress.c @@ -74,7 +75,7 @@ SET(Sources SET(Headers ${LIBRARY_DIR}/common/bitstream.h ${LIBRARY_DIR}/common/error_private.h - ${LIBRARY_DIR}/common/error_public.h + ${LIBRARY_DIR}/common/zstd_errors.h ${LIBRARY_DIR}/common/fse.h ${LIBRARY_DIR}/common/huf.h ${LIBRARY_DIR}/common/mem.h diff --git a/lib/Makefile b/lib/Makefile index c3c993e3c..5df8c3cb4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -97,7 +97,7 @@ install: libzstd libzstd.pc @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed From e19111c42f19f4d98b230faad79bd7763f5e1e49 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 11:09:36 -0700 Subject: [PATCH 10/26] make creates libzstd binaries (#415) --- Makefile | 15 +++++++++------ lib/Makefile | 18 +++++++++--------- programs/Makefile | 12 ++++++------ 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index ac0c583f4..ae7189993 100644 --- a/Makefile +++ b/Makefile @@ -22,15 +22,18 @@ endif .PHONY: default all zlibwrapper zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan -default: zstd +default: libzstd zstd all: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ zstd32 $(MAKE) -C $(TESTDIR) $@ all32 +libzstd: + @$(MAKE) -C $(ZSTDDIR) + zstd: - $(MAKE) -C $(PRGDIR) + @$(MAKE) -C $(PRGDIR) cp $(PRGDIR)/zstd . zlibwrapper: @@ -54,12 +57,12 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) HOST_OS = POSIX install: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ uninstall: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ travis-install: $(MAKE) install PREFIX=~/install_test_dir diff --git a/lib/Makefile b/lib/Makefile index 5df8c3cb4..1117b491c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -103,15 +103,15 @@ install: libzstd libzstd.pc @echo zstd static and shared library installed uninstall: - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) - $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a - $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions - $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif diff --git a/programs/Makefile b/programs/Makefile index ed26f0700..06da7c44b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -153,11 +153,11 @@ install: zstd @echo zstd installation completed uninstall: - $(RM) $(DESTDIR)$(BINDIR)/zstdcat - $(RM) $(DESTDIR)$(BINDIR)/unzstd - $(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) - $(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 - $(RM) $(DESTDIR)$(MANDIR)/unzstd.1 - $(RM) $(DESTDIR)$(MANDIR)/zstd.1 + @$(RM) $(DESTDIR)$(BINDIR)/zstdcat + @$(RM) $(DESTDIR)$(BINDIR)/unzstd + @$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) + @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 + @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1 + @$(RM) $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled endif From e405c7a126af23fccd5660ecccf65e46b1160bff Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 15:29:22 -0700 Subject: [PATCH 11/26] updated visual projects --- Makefile | 6 +++--- NEWS | 2 ++ build/VS2005/fullbench/fullbench.vcproj | 6 +++++- build/VS2005/fuzzer/fuzzer.vcproj | 6 +++++- build/VS2005/zstd/zstd.vcproj | 6 +++++- build/VS2005/zstdlib/zstdlib.vcproj | 6 +++++- build/VS2008/fullbench/fullbench.vcproj | 6 +++++- build/VS2008/fuzzer/fuzzer.vcproj | 6 +++++- build/VS2008/zstd/zstd.vcproj | 6 +++++- build/VS2008/zstdlib/zstdlib.vcproj | 6 +++++- build/VS2010/fullbench/fullbench.vcxproj | 2 ++ build/VS2010/fuzzer/fuzzer.vcxproj | 2 ++ build/VS2010/zstd/zstd.vcxproj | 4 +++- build/VS2010/zstdlib/zstdlib.vcxproj | 5 ++++- 14 files changed, 56 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index ae7189993..20ae31e44 100644 --- a/Makefile +++ b/Makefile @@ -51,9 +51,9 @@ clean: @echo Cleaning completed -#---------------------------------------------------------------------------------- -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets -#---------------------------------------------------------------------------------- +#------------------------------------------------------------------------------ +# make install is validated only for Linux, OSX, Hurd and some BSD targets +#------------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) HOST_OS = POSIX install: diff --git a/NEWS b/NEWS index ad56d17a8..ab6b26b42 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,5 @@ +v1.1.1 + v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell added : NetBSD install target (#338) diff --git a/build/VS2005/fullbench/fullbench.vcproj b/build/VS2005/fullbench/fullbench.vcproj index 17af9c819..c28d1f696 100644 --- a/build/VS2005/fullbench/fullbench.vcproj +++ b/build/VS2005/fullbench/fullbench.vcproj @@ -335,6 +335,10 @@ RelativePath="..\..\..\lib\common\entropy_common.c" > + + @@ -394,7 +398,7 @@ > + + @@ -398,7 +402,7 @@ > + + @@ -450,7 +454,7 @@ > + + @@ -426,7 +430,7 @@ > + + @@ -395,7 +399,7 @@ > + + @@ -399,7 +403,7 @@ > + + @@ -451,7 +455,7 @@ > + + @@ -427,7 +431,7 @@ > + @@ -175,6 +176,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index a436b0389..020e521aa 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -157,6 +157,7 @@ + @@ -176,6 +177,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index eb7e7b504..181bbe6df 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -20,6 +20,7 @@ + @@ -54,6 +55,7 @@ + @@ -220,4 +222,4 @@ - \ No newline at end of file + diff --git a/build/VS2010/zstdlib/zstdlib.vcxproj b/build/VS2010/zstdlib/zstdlib.vcxproj index b97808dd0..d32b4861f 100644 --- a/build/VS2010/zstdlib/zstdlib.vcxproj +++ b/build/VS2010/zstdlib/zstdlib.vcxproj @@ -20,6 +20,7 @@ + @@ -39,9 +40,11 @@ + + - + From 5d919e7ac349b9efaaf340f54a22cffa496884b8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 17:28:59 -0700 Subject: [PATCH 12/26] added ZSTD_error_frameParameter_windowTooLarge (#403) --- lib/common/error_private.c | 1 + lib/common/zstd_errors.h | 1 + lib/decompress/zstd_decompress.c | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/common/error_private.c b/lib/common/error_private.c index 839f77434..a0fa1724a 100644 --- a/lib/common/error_private.c +++ b/lib/common/error_private.c @@ -23,6 +23,7 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(parameter_unknown): return "Unknown parameter type"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h index 9335eaab0..50dc4f720 100644 --- a/lib/common/zstd_errors.h +++ b/lib/common/zstd_errors.h @@ -29,6 +29,7 @@ typedef enum { ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, + ZSTD_error_frameParameter_windowTooLarge, ZSTD_error_compressionParameter_unsupported, ZSTD_error_init_missing, ZSTD_error_memory_allocation, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 88869d82a..d157e0051 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (!singleSegment) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ windowSize = (1U << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t case 3 : frameContentSize = MEM_readLE64(ip+pos); break; } if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->windowSize = windowSize; fparamsPtr->dictID = dictID; From 3b9cdf922072a0fe4996fb3db1c4ed828605c693 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 12 Oct 2016 20:54:42 -0700 Subject: [PATCH 13/26] Fix ubsan failures (pass NULL to memcpy) --- lib/compress/zstd_compress.c | 6 ++++-- lib/decompress/zstd_decompress.c | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 94f4b5a25..e53ecc5fd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* op = ostart; U32 const maxDist = 1 << cctx->params.cParams.windowLog; - if (cctx->params.fParams.checksumFlag) + if (cctx->params.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { @@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d157e0051..bb4336ff2 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1402,7 +1402,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); From d4cda27b63ec07f1b9f93a63581868c65204eb92 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 13:13:13 -0700 Subject: [PATCH 14/26] new command -M#, to limit memory usage during decompression (#403) --- NEWS | 1 + lib/decompress/zstd_decompress.c | 4 ++-- programs/fileio.c | 4 ++++ programs/fileio.h | 2 +- programs/zstdcli.c | 29 +++++++++++++++++++---------- tests/playTests.sh | 2 ++ 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/NEWS b/NEWS index ab6b26b42..1b65b508c 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v1.1.1 +New : command -M, to limit allowed memory consumption v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d157e0051..e6c409990 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1573,7 +1573,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, switch(paramType) { default : return ERROR(parameter_unknown); - case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break; + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break; } return 0; } @@ -1654,7 +1654,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); diff --git a/programs/fileio.c b/programs/fileio.c index d57ffedd2..c4c308e03 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -120,6 +120,9 @@ static U32 g_checksumFlag = 1; void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } static U32 g_removeSrcFile = 0; void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } +static U32 g_memLimit = 0; +void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } + /*-************************************* @@ -480,6 +483,7 @@ static dRess_t FIO_createDResources(const char* dictFileName) /* Allocation */ ress.dctx = ZSTD_createDStream(); if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream"); + ZSTD_setDStreamParameter(ress.dctx, ZSTDdsp_maxWindowSize, g_memLimit); ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZSTD_DStreamOutSize(); diff --git a/programs/fileio.h b/programs/fileio.h index 1e89aec27..60a7e0de8 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -15,7 +15,6 @@ extern "C" { #endif - /* ************************************* * Special i/o constants **************************************/ @@ -37,6 +36,7 @@ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdo void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setChecksumFlag(unsigned checksumFlag); void FIO_setRemoveSrcFile(unsigned flag); +void FIO_setMemLimit(unsigned memLimit); /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f64909d98..3a284c31d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -135,6 +135,7 @@ static int usage_advanced(const char* programName) DISPLAY( "--test : test compressed file integrity \n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #endif + DISPLAY( " -M# : Set a memory usage limit for decompression \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); @@ -172,14 +173,19 @@ static void waitEnter(void) } /*! readU32FromChar() : - @return : unsigned integer value reach from input in `char` format + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : this function can overflow if digit string > MAX_UINT */ + Note : function result can overflow if digit string > MAX_UINT */ static unsigned readU32FromChar(const char** stringPtr) { unsigned result = 0; while ((**stringPtr >='0') && (**stringPtr <='9')) result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if (toupper(**stringPtr)=='K') result <<= 10, (*stringPtr)++ ; + else if (toupper(**stringPtr)=='M') result <<= 20, (*stringPtr)++ ; + if (toupper(**stringPtr)=='i') (*stringPtr)++; + if (toupper(**stringPtr)=='B') (*stringPtr)++; return result; } @@ -206,6 +212,7 @@ int main(int argCount, const char* argv[]) int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; + unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; @@ -224,8 +231,8 @@ int main(int argCount, const char* argv[]) /* init */ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ (void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */ - (void)decode; (void)cLevel; (void)testmode;/* not used when ZSTD_NOCOMPRESS set */ - (void)ultra; /* not used when ZSTD_NODECOMPRESS set */ + (void)decode; (void)cLevel; (void)testmode; /* not used when ZSTD_NOCOMPRESS set */ + (void)ultra; (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; displayOut = stderr; @@ -331,6 +338,12 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + /* limit decompression memory */ + case 'M': + argument++; + memLimit = readU32FromChar(&argument); + break; + #ifdef UTIL_HAS_CREATEFILELIST /* recursive */ case 'r': recursive=1; argument++; break; @@ -359,10 +372,7 @@ int main(int argCount, const char* argv[]) /* cut input into blocks (benchmark only) */ case 'B': argument++; - { size_t bSize = readU32FromChar(&argument); - if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ - if (toupper(*argument)=='M') bSize<<=20, argument++; - if (toupper(*argument)=='B') argument++; + { size_t const bSize = readU32FromChar(&argument); BMK_setNotificationLevel(displayLevel); BMK_SetBlockSize(bSize); } @@ -395,8 +405,6 @@ int main(int argCount, const char* argv[]) nextArgumentIsMaxDict = 0; lastCommand = 0; maxDictSize = readU32FromChar(&argument); - if (toupper(*argument)=='K') maxDictSize <<= 10; - if (toupper(*argument)=='M') maxDictSize <<= 20; continue; } @@ -511,6 +519,7 @@ int main(int argCount, const char* argv[]) } else { /* decompression */ #ifndef ZSTD_NODECOMPRESS if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ + FIO_setMemLimit(memLimit); if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); else diff --git a/tests/playTests.sh b/tests/playTests.sh index d94d8fab9..efb0f0cbd 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -81,6 +81,8 @@ $ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout $ZSTD -dc - < tmp.zst > $INTOVOID $ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used $ZSTD -d - < tmp.zst > $INTOVOID +$ECHO "test : impose memory limitation (must fail)" +$ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" $ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite" From 1122349ac2d8d022031f4a4b1ec002317603916d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 14:07:11 -0700 Subject: [PATCH 15/26] added long comment --memlimit= --- programs/zstdcli.c | 13 ++++++++++--- tests/playTests.sh | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3a284c31d..9a38ede7a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -247,16 +247,16 @@ int main(int argCount, const char* argv[]) if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } /* command switches */ - for(argNb=1; argNb $INTOVOID # implicit stdout when stdin is used $ZSTD -d - < tmp.zst > $INTOVOID $ECHO "test : impose memory limitation (must fail)" $ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" +$ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command $ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite" From d7b120ab5cb34224c978ea2674b7c2b7517e3435 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 14:22:32 -0700 Subject: [PATCH 16/26] added long commands --memory= and --memlimit-decompress= --- programs/zstdcli.c | 10 ++++++++++ tests/playTests.sh | 2 ++ 2 files changed, 12 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9a38ede7a..da3786d60 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -281,6 +281,16 @@ int main(int argCount, const char* argv[]) memLimit = readU32FromChar(&argument); continue; } + if (!strncmp(argument, "--memory=", strlen("--memory="))) { + argument += strlen("--memory="); + memLimit = readU32FromChar(&argument); + continue; + } + if (!strncmp(argument, "--memlimit-decompress=", strlen("--memlimit-decompress="))) { + argument += strlen("--memlimit-decompress="); + memLimit = readU32FromChar(&argument); + continue; + } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){ diff --git a/tests/playTests.sh b/tests/playTests.sh index df9b167f5..c5a58d628 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -84,6 +84,8 @@ $ZSTD -d - < tmp.zst > $INTOVOID $ECHO "test : impose memory limitation (must fail)" $ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" $ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +$ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +$ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command $ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite" From 70077bc9bb7dcc835fd3bb57ce7ca4bc64009056 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 14:41:17 -0700 Subject: [PATCH 17/26] refactor for long commands --- programs/zstdcli.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index da3786d60..2df410431 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -189,6 +189,14 @@ static unsigned readU32FromChar(const char** stringPtr) return result; } +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + unsigned const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -276,21 +284,9 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } /* long commands with arguments */ - if (!strncmp(argument, "--memlimit=", strlen("--memlimit="))) { - argument += strlen("--memlimit="); - memLimit = readU32FromChar(&argument); - continue; - } - if (!strncmp(argument, "--memory=", strlen("--memory="))) { - argument += strlen("--memory="); - memLimit = readU32FromChar(&argument); - continue; - } - if (!strncmp(argument, "--memlimit-decompress=", strlen("--memlimit-decompress="))) { - argument += strlen("--memlimit-decompress="); - memLimit = readU32FromChar(&argument); - continue; - } + if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){ From 2b361cf2f1b5b461e93357eb4a1cafed6e2bcfeb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 16:03:34 -0700 Subject: [PATCH 18/26] minor opt --- NEWS | 2 +- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_opt.h | 6 +++--- lib/dictBuilder/zdict.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 1b65b508c..a9454b40b 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,5 @@ v1.1.1 -New : command -M, to limit allowed memory consumption +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e53ecc5fd..e71873b03 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co const U32 dictLimit = zc->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - const BYTE* match = base + matchIndex; + const BYTE* match; const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index cb5872908..cea67056a 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); { U32 i; for (i=0; irep[i]; } - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, U32 current = (U32)(ip-base); memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; - inr = ip; + //inr = ip; opt[0].litlen = (U32)(ip - anchor); /* check repCode */ diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 47a82af14..a88e70520 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -811,7 +811,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); #endif - dstPtr += 12; + //dstPtr += 12; eSize += 12; _cleanup: From 06573e17be39c6f308fc09a40efab58773902a3e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 17 Oct 2016 17:28:28 -0700 Subject: [PATCH 19/26] fixed minor corner case --- lib/decompress/zstd_decompress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 990a64910..00b069e9f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -807,7 +807,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (ofCode <= 1) { offset += (llCode==0); if (offset) { - size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset = temp; From 33fdd099bbaef46d4a0c94783cca21649824b2fc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 17 Oct 2016 17:48:48 -0700 Subject: [PATCH 20/26] fix command line interpretation --- programs/zstdcli.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 2df410431..9aa97cdd2 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -25,7 +25,6 @@ **************************************/ #include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ #include /* strcmp, strlen */ -#include /* toupper */ #include /* errno */ #include "fileio.h" #ifndef ZSTD_NOBENCH @@ -182,10 +181,13 @@ static unsigned readU32FromChar(const char** stringPtr) unsigned result = 0; while ((**stringPtr >='0') && (**stringPtr <='9')) result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - if (toupper(**stringPtr)=='K') result <<= 10, (*stringPtr)++ ; - else if (toupper(**stringPtr)=='M') result <<= 20, (*stringPtr)++ ; - if (toupper(**stringPtr)=='i') (*stringPtr)++; - if (toupper(**stringPtr)=='B') (*stringPtr)++; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } return result; } From 197a55ee7b508c0e92bb521de78ad32fef0b60bb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 11:27:52 -0700 Subject: [PATCH 21/26] fix edge case --- lib/compress/huf_compress.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index b7d3d77a2..78784aa36 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si } } /* fill val */ - { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; { U32 n; for (n=0; n0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ + U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ min += nbPerRank[n]; min >>= 1; } } From 83d7bdee4b4695e6036b9c7889572df0a96dbd56 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 12:25:43 -0700 Subject: [PATCH 22/26] fixed invalid invocation of dictionary in legacy decoder v0.5 --- lib/legacy/zstd_v05.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 96ffceb97..7e0690174 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3063,7 +3063,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb, - const void* src, size_t srcSize) + const void* src, size_t srcSize, U32 flagStaticTable) { const BYTE* const istart = (const BYTE* const)src; const BYTE* ip = istart; @@ -3118,6 +3118,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableLL, LLbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3141,6 +3142,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableOffb, Offbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3164,6 +3166,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableML, MLbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3376,7 +3379,7 @@ static size_t ZSTDv05_decompressSequences( /* Build Decoding Tables */ errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, DTableLL, DTableML, DTableOffb, - ip, seqSize); + ip, seqSize, dctx->flagStaticTables); if (ZSTDv05_isError(errorCode)) return errorCode; ip += errorCode; @@ -3385,6 +3388,7 @@ static size_t ZSTDv05_decompressSequences( seq_t sequence; seqState_t seqState; + dctx->flagStaticTables = 1; memset(&sequence, 0, sizeof(sequence)); sequence.offset = REPCODE_STARTVALUE; seqState.dumps = dumps; From 9313c8d95302a3c0c0fe3252b47aa7d3c4b6a045 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 13:36:15 -0700 Subject: [PATCH 23/26] no repeat mode in legacy v0.5 --- lib/legacy/zstd_v05.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 7e0690174..4c1c8c5e4 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3388,7 +3388,6 @@ static size_t ZSTDv05_decompressSequences( seq_t sequence; seqState_t seqState; - dctx->flagStaticTables = 1; memset(&sequence, 0, sizeof(sequence)); sequence.offset = REPCODE_STARTVALUE; seqState.dumps = dumps; From f7906d595500c04951643eb98c8f26405b01973f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 13:48:32 -0700 Subject: [PATCH 24/26] detect early impossible decompression scenario in legacy decoder v0.5 --- NEWS | 1 + lib/legacy/zstd_v05.c | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index a9454b40b..570d66799 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +Changed : zstd_errors.h is now part of include installation v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 4c1c8c5e4..3cb5bb4af 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2032,13 +2032,14 @@ size_t HUFv05_decompress1X2_usingDTable( { BYTE* op = (BYTE*)dst; BYTE* const oend = op + dstSize; - size_t errorCode; const U32 dtLog = DTable[0]; const void* dtPtr = DTable; const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1; BITv05_DStream_t bitD; - errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize); - if (HUFv05_isError(errorCode)) return errorCode; + + if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall); + { size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize); + if (HUFv05_isError(errorCode)) return errorCode; } HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog); From a7a4690b0a097dbb29b1f91823c86c0f9e0f21c9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 16:01:03 -0700 Subject: [PATCH 25/26] use implicit rules to compile zstd_decompress.c --- programs/Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 06da7c44b..0ed7f3d07 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -76,17 +76,18 @@ default: zstd all: zstd - +$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ - -$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT) + +$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c + $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ + zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT) From 52c1bf93fe677dc45f243ba05bf7e3a20aa9196c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 16:34:58 -0700 Subject: [PATCH 26/26] improved dicitonary segment merge --- lib/dictBuilder/zdict.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index a88e70520..b3f20b12b 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -371,21 +371,22 @@ static dictItem ZDICT_analyzePos( static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) { const U32 tableSize = table->pos; - const U32 max = elt.pos + (elt.length-1); + const U32 eltEnd = elt.pos + elt.length; /* tail overlap */ U32 u; for (u=1; u elt.pos) && (table[u].pos < max)) { /* overlap */ + if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ /* append */ U32 addedLength = table[u].pos - elt.pos; table[u].length += addedLength; table[u].pos = elt.pos; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ - table[u].savings += elt.length / 8; /* rough approx */ + table[u].savings += elt.length / 8; /* rough approx bonus */ elt = table[u]; + /* sort : improve rank */ while ((u>1) && (table[u-1].savings < elt.savings)) - table[u] = table[u-1], u--; + table[u] = table[u-1], u--; table[u] = elt; return u; } } @@ -393,14 +394,15 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) /* front overlap */ for (u=1; u elt.pos) && (table[u].pos < elt.pos)) { /* overlap */ + if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int addedLength = (elt.pos + elt.length) - (table[u].pos + table[u].length); - table[u].savings += elt.length / 8; /* rough approx */ - if (addedLength > 0) { /* otherwise, already included */ + int addedLength = (int)eltEnd - (table[u].pos + table[u].length); + table[u].savings += elt.length / 8; /* rough approx bonus */ + if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ } + /* sort : improve rank */ elt = table[u]; while ((u>1) && (table[u-1].savings < elt.savings)) table[u] = table[u-1], u--;