From eb70d219fdbd0834f83bf2c66551a4b3fa145de0 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 11 Apr 2017 17:15:13 -0700 Subject: [PATCH 1/9] Add test of file > 4GB to playTests --- tests/playTests.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index 266bbd91b..6ef5914a9 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -11,6 +11,7 @@ roundTripTest() { local_p="$2" else local_c="$2" + local_p="" fi rm -f tmp1 tmp2 @@ -20,6 +21,23 @@ roundTripTest() { $DIFF -q tmp1 tmp2 } +fileRoundTripTest() { + if [ -n "$3" ]; then + local_c="$3" + local_p="$2" + else + local_c="$2" + local_p="" + fi + + rm -f tmp.zstd tmp.md5.1 tmp.md5.2 + $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp1 && $ZSTD -v$local_c -c | $ZSTD -d | $MD5SUM > tmp.md5.2" + ./datagen $1 $local_p > tmp + cat tmp | $MD5SUM > tmp.md5.1 + $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2 + $DIFF -q tmp.md5.1 tmp.md5.2 +} + isTerminal=false if [ -t 0 ] && [ -t 1 ] then @@ -441,6 +459,8 @@ roundTripTest -g519K 6 # greedy, hash chain roundTripTest -g517K 16 # btlazy2 roundTripTest -g516K 19 # btopt +fileRoundTripTest -g500K + rm tmp* if [ "$1" != "--test-large-data" ]; then @@ -476,4 +496,6 @@ roundTripTest -g50000000 -P94 19 roundTripTest -g99000000 -P99 20 roundTripTest -g6000000000 -P99 1 +fileRoundTripTest -g4193M -P99 1 + rm tmp* From d37e1df2ab26b9cbb22e6b602ea5da9b3662af89 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 11 Apr 2017 17:33:26 -0700 Subject: [PATCH 2/9] Fix message --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 6ef5914a9..3675bb168 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -31,7 +31,7 @@ fileRoundTripTest() { fi rm -f tmp.zstd tmp.md5.1 tmp.md5.2 - $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp1 && $ZSTD -v$local_c -c | $ZSTD -d | $MD5SUM > tmp.md5.2" + $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c | $ZSTD -d" ./datagen $1 $local_p > tmp cat tmp | $MD5SUM > tmp.md5.1 $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2 From afa48518e2fbd5dfce3982e4ed003f4c707d1801 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 13 Apr 2017 12:28:28 -0700 Subject: [PATCH 3/9] -T0 detects number of physical cores --- programs/util.h | 172 +++++++++++++++++++++++++++++++++++++++++++++ programs/zstdcli.c | 5 ++ 2 files changed, 177 insertions(+) diff --git a/programs/util.h b/programs/util.h index 9992d79da..0b90cda19 100644 --- a/programs/util.h +++ b/programs/util.h @@ -25,6 +25,7 @@ extern "C" { #include /* malloc */ #include /* size_t, ptrdiff_t */ #include /* fprintf */ +#include /* strncmp */ #include /* stat, utime */ #include /* stat */ #if defined(_MSC_VER) @@ -488,6 +489,177 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu if (filenameTable) free((void*)filenameTable); } +/* count the number of physical cores */ +#if defined(_WIN32) || defined(WIN32) + +#include + +typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); + +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores; + if (numPhysicalCores != 0) return numPhysicalCores; + + { LPFN_GLPI glpi; + BOOL done = FALSE; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; + DWORD returnLength = 0; + size_t byteOffset = 0; + + glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), + "GetLogicalProcessorInformation"); + + if (glpi == NULL) { + goto failed; + } + + while(!done) { + DWORD rc = glpi(buffer, &returnLength); + if (FALSE == rc) { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + if (buffer) + free(buffer); + buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); + + if (buffer == NULL) { + perror("zstd"); + exit(1); + } + } else { + /* some other error */ + goto failed; + } + } else { + done = TRUE; + } + } + + while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { + ptr = buffer; + + if (ptr->RelationShip == RelationProcessorCore) { + numPhysicalCores++; + } + } + + free(buffer); + + return numPhysicalCores; + } + +failed: + /* try to fall back on GetSystemInfo */ + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + numPhysicalCores = sysinfo.dwNumberOfProcessors; + if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */ + return numPhysicalCores; +} + +#elif defined(__APPLE__) + +#include + +/* Use apple-provided syscall + * see: man 3 sysctl */ +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static S32 numPhysicalCores; /* apple specifies int32_t */ + if (numPhysicalCores != 0) return numPhysicalCores; + + { int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, sizeof(int), NULL, 0); + if (ret != 0) { + if (errno == ENOENT) { + /* entry not present, fall back on 1 */ + numPhysicalCores = 1; + } else { + perror("zstd: can't get number of physical cpus"); + exit(1); + } + } + + return numPhysicalCores; + } +} + +#elif defined(__linux__) + +/* parse /proc/cpuinfo + * siblings / cpu cores should give hyperthreading ratio + * otherwise fall back on sysconf */ +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores; + + if (numPhysicalCores != 0) return numPhysicalCores; + + numPhysicalCores = sysconf(_SC_NPROCESSORS_ONLN); + if (numPhysicalCores == -1) { + /* value not queryable, fall back on 1 */ + return numPhysicalCores = 1; + } + + /* try to determine if there's hyperthreading */ + { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); + size_t const BUF_SIZE = 80; + char buff[BUF_SIZE]; + + int siblings = 0; + int cpu_cores = 0; + int ratio = 1; + + if (cpuinfo == NULL) { + /* fall back on the sysconf value */ + return numPhysicalCores; + } + + /* assume the cpu cores/siblings values will be constant across all + * present processors */ + while (!feof(cpuinfo)) { + if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { + if (strncmp(buff, "siblings", 8) == 0) { + const char* const sep = strchr(buff, ':'); + if (*sep == '\0') { + /* formatting was broken? */ + goto failed; + } + + siblings = atoi(sep + 1); + } + if (strncmp(buff, "cpu cores", 9) == 0) { + const char* const sep = strchr(buff, ':'); + if (*sep == '\0') { + /* formatting was broken? */ + goto failed; + } + + cpu_cores = atoi(sep + 1); + } + } else if (ferror(cpuinfo)) { + /* fall back on the sysconf value */ + goto failed; + } + } + if (siblings && cpu_cores) { + ratio = siblings / cpu_cores; + } +failed: + fclose(cpuinfo); + return numPhysicalCores = numPhysicalCores / ratio; + } +} + +#else + +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + /* assume 1 */ + return 1; +} + +#endif #if defined (__cplusplus) } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 18e259c74..73b4eab9d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -602,6 +602,11 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); #endif + if (nbThreads == 0) { + /* try to guess */ + nbThreads = UTIL_countPhysicalCores(); + DISPLAYLEVEL(3, "Note: %d physical core(s) detected\n", nbThreads); + } g_utilDisplayLevel = g_displayLevel; if (!followLinks) { From f876f1200cab5d616bdecfb30144a23ea76bf9fd Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 13 Apr 2017 12:33:45 -0700 Subject: [PATCH 4/9] Fix compilation on macOS --- programs/bench.c | 9 +++++++-- programs/dibio.c | 4 +++- programs/fileio.c | 4 +++- programs/util.h | 3 ++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index d25ff8f04..8d8ed4ffa 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -146,8 +146,13 @@ typedef struct { } blockParam_t; -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +#ifndef MIN +# define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif +#ifndef MAX +# define MAX(a,b) ((a) > (b) ? (a) : (b)) +#endif static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, diff --git a/programs/dibio.c b/programs/dibio.c index b7ed280ea..5e685a325 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -89,7 +89,9 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#ifndef MIN +# define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif /* ******************************************************** diff --git a/programs/fileio.c b/programs/fileio.c index f1f5a5577..1dc222b65 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -96,7 +96,9 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; -#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#ifndef MIN +# define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif /* ************************************************************ * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW diff --git a/programs/util.h b/programs/util.h index 0b90cda19..104f8dad2 100644 --- a/programs/util.h +++ b/programs/util.h @@ -569,7 +569,8 @@ UTIL_STATIC int UTIL_countPhysicalCores(void) static S32 numPhysicalCores; /* apple specifies int32_t */ if (numPhysicalCores != 0) return numPhysicalCores; - { int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, sizeof(int), NULL, 0); + { size_t size = sizeof(S32); + int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0); if (ret != 0) { if (errno == ENOENT) { /* entry not present, fall back on 1 */ From 3b6207d4bd9143204745f90a38278891dbaa4474 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 13 Apr 2017 14:03:56 -0700 Subject: [PATCH 5/9] Fix compilation on windows --- programs/util.h | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/util.h b/programs/util.h index 104f8dad2..63cfa9647 100644 --- a/programs/util.h +++ b/programs/util.h @@ -536,12 +536,16 @@ UTIL_STATIC int UTIL_countPhysicalCores(void) } } - while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { - ptr = buffer; + ptr = buffer; - if (ptr->RelationShip == RelationProcessorCore) { + while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { + + if (ptr->Relationship == RelationProcessorCore) { numPhysicalCores++; } + + ptr++; + byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); } free(buffer); @@ -551,10 +555,11 @@ UTIL_STATIC int UTIL_countPhysicalCores(void) failed: /* try to fall back on GetSystemInfo */ - SYSTEM_INFO sysinfo; - GetSystemInfo(&sysinfo); - numPhysicalCores = sysinfo.dwNumberOfProcessors; - if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */ + { SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + numPhysicalCores = sysinfo.dwNumberOfProcessors; + if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */ + } return numPhysicalCores; } From 9227aae0013b44fc7628911d735731e01d1f9480 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 13 Apr 2017 14:06:40 -0700 Subject: [PATCH 6/9] Fix clang linux compilation --- programs/util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/util.h b/programs/util.h index 63cfa9647..a50b5e55b 100644 --- a/programs/util.h +++ b/programs/util.h @@ -601,7 +601,7 @@ UTIL_STATIC int UTIL_countPhysicalCores(void) if (numPhysicalCores != 0) return numPhysicalCores; - numPhysicalCores = sysconf(_SC_NPROCESSORS_ONLN); + numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); if (numPhysicalCores == -1) { /* value not queryable, fall back on 1 */ return numPhysicalCores = 1; From ad8da8855bd3a14c84789eda116d7f33920f454b Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 13 Apr 2017 14:40:06 -0700 Subject: [PATCH 7/9] Make appveyor small tests use new mingw as well --- appveyor.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 27994853a..5887d52e6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -218,10 +218,10 @@ - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - SET PATH_ORIGINAL=%PATH% - if [%HOST%]==[mingw] ( - SET "PATH_MINGW32=C:\MinGW\bin;C:\MinGW\usr\bin" && - SET "PATH_MINGW64=C:\msys64\mingw64\bin;C:\msys64\usr\bin" && - COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe && - COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe + SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" && + SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe ) - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] ( SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" From 42bac7fa84e5ce26c582ea517e4c51befd19cd88 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 13 Apr 2017 15:35:05 -0700 Subject: [PATCH 8/9] Change ifndef's to undef's --- programs/bench.c | 10 ++++------ programs/dibio.c | 5 ++--- programs/fileio.c | 5 ++--- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 8d8ed4ffa..c3681eb05 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -147,12 +147,10 @@ typedef struct { -#ifndef MIN -# define MIN(a,b) ((a) < (b) ? (a) : (b)) -#endif -#ifndef MAX -# define MAX(a,b) ((a) > (b) ? (a) : (b)) -#endif +#undef MIN +#undef MAX +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, diff --git a/programs/dibio.c b/programs/dibio.c index 5e685a325..aac36425c 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -89,9 +89,8 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } -#ifndef MIN -# define MIN(a,b) ((a) < (b) ? (a) : (b)) -#endif +#undef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) /* ******************************************************** diff --git a/programs/fileio.c b/programs/fileio.c index 1dc222b65..9bca20664 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -96,9 +96,8 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; -#ifndef MIN -# define MIN(a,b) ((a) < (b) ? (a) : (b)) -#endif +#undef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) /* ************************************************************ * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW From e4f3235c85303cee20a2f93732b93a346f273c48 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 13 Apr 2017 16:34:28 -0700 Subject: [PATCH 9/9] Add 0 initializers to static variables --- programs/util.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/util.h b/programs/util.h index a50b5e55b..b989e8232 100644 --- a/programs/util.h +++ b/programs/util.h @@ -498,7 +498,7 @@ typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); UTIL_STATIC int UTIL_countPhysicalCores(void) { - static int numPhysicalCores; + static int numPhysicalCores = 0; if (numPhysicalCores != 0) return numPhysicalCores; { LPFN_GLPI glpi; @@ -571,7 +571,7 @@ failed: * see: man 3 sysctl */ UTIL_STATIC int UTIL_countPhysicalCores(void) { - static S32 numPhysicalCores; /* apple specifies int32_t */ + static S32 numPhysicalCores = 0; /* apple specifies int32_t */ if (numPhysicalCores != 0) return numPhysicalCores; { size_t size = sizeof(S32); @@ -597,7 +597,7 @@ UTIL_STATIC int UTIL_countPhysicalCores(void) * otherwise fall back on sysconf */ UTIL_STATIC int UTIL_countPhysicalCores(void) { - static int numPhysicalCores; + static int numPhysicalCores = 0; if (numPhysicalCores != 0) return numPhysicalCores;