From 9597b438e931dcdd0cc8497bc330e9bcc495992a Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Tue, 17 Jul 2018 18:52:57 +0200 Subject: [PATCH 1/7] fix #1241 Ensure that first input position is valid for a match even during first usage of context by starting reference at 1 (avoiding the problematic 0). --- lib/compress/zstd_compress.c | 3 +++ tests/fuzzer.c | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c66862524..8f63eb953 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1032,6 +1032,9 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->hashLog3 = hashLog3; memset(&ms->window, 0, sizeof(ms->window)); + ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ + ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ ZSTD_invalidateMatchState(ms); /* opt parser space */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f7bacd5ca..b9d319ea2 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -411,6 +411,26 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); + { int i; + for (i=0; i<20; i++) + ((char*)CNBuffer)[i] = i; /* ensure no match during initial section */ + memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ + for (i=1; i<=19; i++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t size1, size2; + DISPLAYLEVEL(5, "l%i ", i); + size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + CHECK_Z(size1); + size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + ZSTD_freeCCtx(cctx); + } + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_outBuffer out = {NULL, 0, 0}; From 7d1bc9cc8c79f7412c44835b910945792b2238c7 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Wed, 18 Jul 2018 16:10:23 +0200 Subject: [PATCH 2/7] fix minor conversion warning --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b9d319ea2..c411d4177 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -414,7 +414,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); { int i; for (i=0; i<20; i++) - ((char*)CNBuffer)[i] = i; /* ensure no match during initial section */ + ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ for (i=1; i<=19; i++) { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); From 3d4b09a5aa7592b86df77a7956ea51b70799d135 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:29:20 +0200 Subject: [PATCH 3/7] support %zu under mingw --- tests/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Makefile b/tests/Makefile index 813380cc2..0955272ef 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -27,6 +27,9 @@ DEBUGLEVEL ?= 1 DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO +endif CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ From c738a2c795380c2a75810c48f320df93fd8bd2ea Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:44:20 +0200 Subject: [PATCH 4/7] ensure appveyor test fails due to formatting error to catch %zu incompatibility --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 742f61206..42eb71d64 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -181,7 +181,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "make allzstd" + SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" From 3f535007e40f1e43ea1513c2ff9356114f1be024 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:56:18 +0200 Subject: [PATCH 5/7] fix %zu support under minGW and relevant test on Appveyor --- appveyor.yml | 2 +- lib/Makefile | 7 +++++-- programs/Makefile | 5 ++++- tests/Makefile | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 42eb71d64..c9f88dce6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -181,7 +181,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd" + SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" diff --git a/lib/Makefile b/lib/Makefile index 9cedd53b7..01689c6d5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -19,6 +19,9 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) VERSION?= $(LIBVER) CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_ +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif CFLAGS ?= -O3 DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ @@ -52,11 +55,11 @@ ifeq ($(ZSTD_LIB_DECOMPRESSION), 0) endif ifneq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_FILES += $(ZSTDCOMP_FILES) + ZSTD_FILES += $(ZSTDCOMP_FILES) endif ifneq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_FILES += $(ZSTDDECOMP_FILES) + ZSTD_FILES += $(ZSTDDECOMP_FILES) endif ifneq ($(ZSTD_LIB_DEPRECATED), 0) diff --git a/programs/Makefile b/programs/Makefile index 4202764c2..912f9eff0 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -38,6 +38,9 @@ endif CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder \ -DXXH_NAMESPACE=ZSTD_ +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ @@ -158,7 +161,7 @@ zstd-release: DEBUGFLAGS := zstd-release: zstd zstd32 : CPPFLAGS += $(THREAD_CPP) -zstd32 : LDFLAGS += $(THREAD_LD) +zstd32 : LDFLAGS += $(THREAD_LD) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c ifneq (,$(filter Windows%,$(OS))) diff --git a/tests/Makefile b/tests/Makefile index 0955272ef..81e685780 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,7 +28,7 @@ DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ From e85b4c56b288a3480e10117b9e37d6744b665fe9 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 17:08:40 +0200 Subject: [PATCH 6/7] speed up appveyor tests --- appveyor.yml | 4 ++-- build/cmake/lib/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c9f88dce6..2b674ce3c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -185,11 +185,11 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" - SCRIPT: "make allzstd" + SCRIPT: "CFLAGS=-Werror make -j allzstd" - COMPILER: "clang" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd" + SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd" - COMPILER: "visual" HOST: "visual" diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index c4c2f81e6..e84e06301 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -14,7 +14,7 @@ OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON) OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON) IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC) - MESSAGE(SEND_ERROR "You need to build at least one flavor of libstd") + MESSAGE(SEND_ERROR "You need to build at least one flavor of libzstd") ENDIF() # Define library directory, where sources and header files are located From 2ca7c69167e17006f867ab550408b6e0487a81e7 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 7 Aug 2018 17:05:05 -0700 Subject: [PATCH 7/7] Fix CDict Attachment to Handle CDicts with Non-Zero Starts CDicts were previously guaranteed to be generated with `lowLimit=dictLimit=0`. This is no longer true, and so the old length and index calculations are no longer valid. This diff fixes them to handle non-zero start indices in CDicts. --- lib/compress/zstd_compress.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3ebfd019f..ed3aab871 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1284,8 +1284,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, } if (attachDict) { - const U32 cdictLen = (U32)( cdict->matchState.window.nextSrc + const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; if (cdictLen == 0) { /* don't even attach dictionaries with no contents */ DEBUGLOG(4, "skipping attaching empty dictionary"); @@ -1295,9 +1296,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, /* prep working match state so dict matches never have negative indices * when they are translated to the working context's index space. */ - if (cctx->blockState.matchState.window.dictLimit < cdictLen) { + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { cctx->blockState.matchState.window.nextSrc = - cctx->blockState.matchState.window.base + cdictLen; + cctx->blockState.matchState.window.base + cdictEnd; ZSTD_window_clear(&cctx->blockState.matchState.window); } cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit;