From 09c8e5390dbaaaca84b27e99d55e1052f5e8dcef Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 12:45:53 +0100 Subject: [PATCH 1/8] __builtin_bswap requires gcc 4.3+ --- lib/common/mem.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index aff044de1..1c223fe5e 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -182,7 +182,7 @@ MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ return _byteswap_ulong(in); -#elif defined (__GNUC__) +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) return __builtin_bswap32(in); #else return ((in << 24) & 0xff000000 ) | @@ -196,7 +196,7 @@ MEM_STATIC U64 MEM_swap64(U64 in) { #if defined(_MSC_VER) /* Visual Studio */ return _byteswap_uint64(in); -#elif defined (__GNUC__) +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) return __builtin_bswap64(in); #else return ((in << 56) & 0xff00000000000000ULL) | From 35bf23c08675edfab1e95aef83bb06e92110f1c9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 13:57:29 +0100 Subject: [PATCH 2/8] MinGW-w64 requires _FILE_OFFSET_BITS 64 --- programs/platform.h | 4 ++-- programs/util.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index 1b53e1f85..89a9f6cd4 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -51,8 +51,8 @@ extern "C" { /* ********************************************************* * Turn on Large Files support (>4GB) for 32-bit Linux/Unix ***********************************************************/ -#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ -# if !defined(_FILE_OFFSET_BITS) +#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ +# if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif # if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ diff --git a/programs/util.h b/programs/util.h index 16bd3bdfc..fe132d38d 100644 --- a/programs/util.h +++ b/programs/util.h @@ -187,7 +187,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; #if defined(_MSC_VER) - struct _stat64 statbuf; + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #elif defined(__MINGW32__) && defined (__MSVCRT__) From ecf90ca24b7031444133bc236a51683bd059dfc7 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 13 Feb 2017 18:27:34 -0800 Subject: [PATCH 3/8] [zstdmt] Fix MSAN failure with ZSTD_p_forceWindow Reproduction steps: ``` make zstreamtest CC=clang CFLAGS="-O3 -g -fsanitize=memory -fsanitize-memory-track-origins" ./zstreamtest -vv -t4178 -i4178 -s4531 ``` How to get to the error in gdb (may be a more efficient way): * 2 breaks at zstd_compress.c:2418 -- in ZSTD_compressContinue_internal() * 2 breaks at zstd_compress.c:2276 -- in ZSTD_compressBlock_internal() * 1 break at zstd_compress.c:1547 Why the error occurred: When `zc->forceWindow == 1`, after calling `ZSTD_loadDictionaryContent()` we have `zc->loadedDictEnd == zc->nextToUpdate == 0`. But, we've really loaded up to `iend` into the dictionary. Then in `ZSTD_compressBlock_internal()` we see that `current > zc->nextToUpdate + 384`, so we load the last 192 bytes a second time. In this case the bytes we are loading are a block of all 0s, starting in the previous block. So when we are loading the last 192 bytes, we find a `match` in the future, 183 bytes beyond `ip`. Since the block is all 0s, the match extends to the end of the block. But in `ZSTD_count()` we only check that `pIn < pInLoopLimit`, but since `pMatch > pIn`, `pMatch` eventually points past the end of the buffer, causing the MSAN failure. The fix: The line changed sets sets `zc->nextToUpdate` to the end of the dictionary. This is the behavior that existed before `ZSTD_p_forceWindow` was introduced. This fixes the exposing test case. Since the code doesn't fail without `zc->forceWindow`, it makes sense that this works. I've run the command `./zstreamtest -T2mn` 64 times without failures. CI should also verify nothing obvious broke. --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 765c8e34d..91e81d9c2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2512,7 +2512,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t return ERROR(GENERIC); /* strategy doesn't exist; impossible */ } - zc->nextToUpdate = zc->loadedDictEnd; + zc->nextToUpdate = (U32)(iend - zc->base); return 0; } From 442c75f13214bc476ee96b865023f3d2b9845e1e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:38:51 +0100 Subject: [PATCH 4/8] removed UTIL_doesFileExists (replaced with UTIL_isRegFile) --- programs/fileio.c | 2 +- programs/util.h | 48 +++++++++++++++++------------------------------ 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b2998ea12..b8d4ef02d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -191,7 +191,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName) f = stdin; SET_BINARY_MODE(stdin); } else { - if (!UTIL_doesFileExists(srcFileName)) { + if (!UTIL_isRegFile(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); return NULL; } diff --git a/programs/util.h b/programs/util.h index 7f710c686..364aa650f 100644 --- a/programs/util.h +++ b/programs/util.h @@ -182,12 +182,29 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) return 1; } + UTIL_STATIC int UTIL_isRegFile(const char* infilename) { stat_t statbuf; return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ } + +UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) +{ + int r; + stat_t statbuf; +#if defined(_MSC_VER) + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; @@ -218,37 +235,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi } -UTIL_STATIC int UTIL_doesFileExists(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ -#endif - return 1; -} - - -UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; -} - /* * A modified version of realloc(). * If UTIL_realloc() fails the original block is freed. From abd6302423d262ef04c299a9585813ed0bc4b01a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:39:09 +0100 Subject: [PATCH 5/8] Windows resources updated to v1.1.4 --- programs/windres/zstd32.res | Bin 1044 -> 1044 bytes programs/windres/zstd64.res | Bin 1044 -> 1044 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res index 748192a988648a0be3b17ce47353f28015003815..b5dd78db717ce9c7ddb2f69a9bc81804e51aedf6 100644 GIT binary patch delta 33 pcmbQjF@`OkT;H4FH692nqlI delta 33 pcmbQjF@`OkT;H4FH692nqlI delta 33 pcmbQjF@ Date: Tue, 14 Feb 2017 09:45:33 +0100 Subject: [PATCH 6/8] Avoid fseek()'s 2GiB barrier with MacOS and *BSD --- programs/fileio.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index b8d4ef02d..c152fdcbe 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -91,8 +91,13 @@ static clock_t g_time = 0; #define MIN(a,b) ((a) < (b) ? (a) : (b)) +/* ************************************************************ +* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW +***************************************************************/ #if defined(_MSC_VER) && _MSC_VER >= 1400 # define LONG_SEEK _fseeki64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define fseek fseeko #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) # define LONG_SEEK fseeko64 #elif defined(_WIN32) && !defined(__DJGPP__) From ce13d087d9af867a1540b6351fd98b212d91dbe4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:52:52 +0100 Subject: [PATCH 7/8] fix LONG_SEEK --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index c152fdcbe..b384f3d1f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -97,7 +97,7 @@ static clock_t g_time = 0; #if defined(_MSC_VER) && _MSC_VER >= 1400 # define LONG_SEEK _fseeki64 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define fseek fseeko +# define LONG_SEEK fseeko #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) # define LONG_SEEK fseeko64 #elif defined(_WIN32) && !defined(__DJGPP__) From 74b81ada256f45b0ea69f50c3b9b3918faacc91a Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 14 Feb 2017 10:08:14 -0800 Subject: [PATCH 8/8] Don't run test-pool with QEMU > make test -n ... ./pool > make test -n QEMU_SYS=valgrind ... ./legacy # ./pool not run --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index f64be1695..2b58c949c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -231,7 +231,7 @@ zstd-playTests: datagen ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy -ifneq ($(QEMU_SYS),qemu-ppc64-static) +ifeq ($(QEMU_SYS),) test: test-pool endif