From 4d9e557f44348a5caaa9a7991d7e8ecfcc68b3d9 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 13 May 2016 13:04:27 +0200 Subject: [PATCH 01/26] Visual Studio: remove MSVC*.DLL dependency from Release executables --- visual/2013/fullbench/fullbench.vcxproj | 2 ++ visual/2013/fuzzer/fuzzer.vcxproj | 2 ++ visual/2013/zstdlib/zstdlib.vcxproj | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/visual/2013/fullbench/fullbench.vcxproj b/visual/2013/fullbench/fullbench.vcxproj index c24105ee3..40935bbf7 100644 --- a/visual/2013/fullbench/fullbench.vcxproj +++ b/visual/2013/fullbench/fullbench.vcxproj @@ -128,6 +128,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false true + MultiThreaded Console @@ -147,6 +148,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true false + MultiThreaded Console diff --git a/visual/2013/fuzzer/fuzzer.vcxproj b/visual/2013/fuzzer/fuzzer.vcxproj index 5fd5f5b48..bd3e849f4 100644 --- a/visual/2013/fuzzer/fuzzer.vcxproj +++ b/visual/2013/fuzzer/fuzzer.vcxproj @@ -128,6 +128,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false true + MultiThreaded Console @@ -147,6 +148,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true false + MultiThreaded Console diff --git a/visual/2013/zstdlib/zstdlib.vcxproj b/visual/2013/zstdlib/zstdlib.vcxproj index 94da3e5d3..c13ff0af0 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj +++ b/visual/2013/zstdlib/zstdlib.vcxproj @@ -176,7 +176,7 @@ true ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false - MultiThreadedDLL + MultiThreaded ProgramDatabase All @@ -199,7 +199,7 @@ ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false false - MultiThreadedDLL + MultiThreaded ProgramDatabase true true From 7c4d041d3cb94d96c6a4aa3bff88f8b99103b879 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 16 May 2016 14:48:04 +0200 Subject: [PATCH 02/26] added Visual Studio 2008 and 2010 projects --- appveyor.yml | 55 +- projects/README.md | 8 + projects/VS2008/fullbench/fullbench.vcproj | 441 +++++++++++++++ projects/VS2008/fuzzer/fuzzer.vcproj | 437 ++++++++++++++ projects/VS2008/zstd.sln | 56 ++ projects/VS2008/zstd/zstd.vcproj | 533 ++++++++++++++++++ projects/VS2008/zstdlib/zstdlib.vcproj | 487 ++++++++++++++++ .../VS2010}/fullbench/fullbench.vcxproj | 18 +- .../fullbench/fullbench.vcxproj.filters | 0 .../VS2010}/fuzzer/fuzzer.vcxproj | 18 +- .../VS2010}/fuzzer/fuzzer.vcxproj.filters | 0 {visual/2013 => projects/VS2010}/zstd.sln | 0 .../VS2010}/zstd/zstd.vcxproj | 30 +- .../VS2010}/zstd/zstd.vcxproj.filters | 0 .../VS2010}/zstdlib/zstdlib.vcxproj | 17 +- .../VS2010}/zstdlib/zstdlib.vcxproj.filters | 9 - visual/2013/zstdlib/resource.h | Bin 812 -> 0 bytes visual/2013/zstdlib/zstdlib.rc | Bin 5196 -> 0 bytes 18 files changed, 2040 insertions(+), 69 deletions(-) create mode 100644 projects/README.md create mode 100644 projects/VS2008/fullbench/fullbench.vcproj create mode 100644 projects/VS2008/fuzzer/fuzzer.vcproj create mode 100644 projects/VS2008/zstd.sln create mode 100644 projects/VS2008/zstd/zstd.vcproj create mode 100644 projects/VS2008/zstdlib/zstdlib.vcproj rename {visual/2013 => projects/VS2010}/fullbench/fullbench.vcxproj (88%) rename {visual/2013 => projects/VS2010}/fullbench/fullbench.vcxproj.filters (100%) rename {visual/2013 => projects/VS2010}/fuzzer/fuzzer.vcxproj (88%) rename {visual/2013 => projects/VS2010}/fuzzer/fuzzer.vcxproj.filters (100%) rename {visual/2013 => projects/VS2010}/zstd.sln (100%) rename {visual/2013 => projects/VS2010}/zstd/zstd.vcxproj (85%) mode change 100755 => 100644 rename {visual/2013 => projects/VS2010}/zstd/zstd.vcxproj.filters (100%) mode change 100755 => 100644 rename {visual/2013 => projects/VS2010}/zstdlib/zstdlib.vcxproj (88%) rename {visual/2013 => projects/VS2010}/zstdlib/zstdlib.vcxproj.filters (90%) delete mode 100644 visual/2013/zstdlib/resource.h delete mode 100644 visual/2013/zstdlib/zstdlib.rc diff --git a/appveyor.yml b/appveyor.yml index df8d15572..848db3836 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,21 +5,46 @@ configuration: platform: - Win32 - x64 -environment: - matrix: -# - PlatformToolset: v100 - - PlatformToolset: v110 - - PlatformToolset: v120 - - PlatformToolset: v140 + build_script: -- cmd: >- - ECHO PlatformToolset=%PlatformToolset% +- cmd: | + ECHO *** + ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% + ECHO *** + REM msbuild "projects\VS2010\zstd.sln" /m /verbosity:detailed /property:PlatformToolset=Windows7.1SDK /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + REM msbuild "projects\VS2010\zstd.sln" "/p:AdditionalLibPaths=C:\Program Files\Microsoft SDKs\Windows\v7.1\Lib\X64" /m /verbosity:detailed /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + msbuild "projects\VS2010\zstd.sln" /p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe - msbuild "visual\2013\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=%PlatformToolset% /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" test_script: -- cmd: >- - ECHO %APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST% - - SET FUZZERTEST=-T2mn - - IF %CONFIGURATION%==Release (%APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST%) \ No newline at end of file +- cmd: SET FUZZERTEST=-T1mn +- cmd: | + IF %CONFIGURATION%==Release (projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%) diff --git a/projects/README.md b/projects/README.md new file mode 100644 index 000000000..d69c71b14 --- /dev/null +++ b/projects/README.md @@ -0,0 +1,8 @@ +projects for various integrated development environments (IDE) +================================ + +#### Included projects + +The following projects are included with the zstd distribution: +- VS2008 - Visual Studio 2008 project +- VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/projects/VS2008/fullbench/fullbench.vcproj new file mode 100644 index 000000000..6ab878cc9 --- /dev/null +++ b/projects/VS2008/fullbench/fullbench.vcproj @@ -0,0 +1,441 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2008/fuzzer/fuzzer.vcproj b/projects/VS2008/fuzzer/fuzzer.vcproj new file mode 100644 index 000000000..9e572da26 --- /dev/null +++ b/projects/VS2008/fuzzer/fuzzer.vcproj @@ -0,0 +1,437 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2008/zstd.sln b/projects/VS2008/zstd.sln new file mode 100644 index 000000000..97b88c8d0 --- /dev/null +++ b/projects/VS2008/zstd.sln @@ -0,0 +1,56 @@ + +Microsoft Visual Studio Solution File, Format Version 10.00 +# Visual Studio 2008 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcproj", "{1A2AB08E-5CE7-4C5B-BE55-458157C14051}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcproj", "{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcproj", "{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcproj", "{99DE2A79-7298-4004-A0ED-030D7A3796CA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.ActiveCfg = Debug|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.Build.0 = Debug|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.ActiveCfg = Debug|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.Build.0 = Debug|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.ActiveCfg = Release|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.Build.0 = Release|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.ActiveCfg = Release|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.Build.0 = Release|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.ActiveCfg = Debug|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.Build.0 = Debug|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.ActiveCfg = Debug|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.Build.0 = Debug|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.ActiveCfg = Release|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.Build.0 = Release|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.ActiveCfg = Release|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.Build.0 = Release|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.ActiveCfg = Debug|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.Build.0 = Debug|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.ActiveCfg = Debug|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.Build.0 = Debug|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.ActiveCfg = Release|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.Build.0 = Release|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.ActiveCfg = Release|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.Build.0 = Release|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.ActiveCfg = Debug|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.Build.0 = Debug|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.ActiveCfg = Debug|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.Build.0 = Debug|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.ActiveCfg = Release|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.Build.0 = Release|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.ActiveCfg = Release|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj new file mode 100644 index 000000000..9e4626500 --- /dev/null +++ b/projects/VS2008/zstd/zstd.vcproj @@ -0,0 +1,533 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2008/zstdlib/zstdlib.vcproj b/projects/VS2008/zstdlib/zstdlib.vcproj new file mode 100644 index 000000000..2a4779aac --- /dev/null +++ b/projects/VS2008/zstdlib/zstdlib.vcproj @@ -0,0 +1,487 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/visual/2013/fullbench/fullbench.vcxproj b/projects/VS2010/fullbench/fullbench.vcxproj similarity index 88% rename from visual/2013/fullbench/fullbench.vcxproj rename to projects/VS2010/fullbench/fullbench.vcxproj index 40935bbf7..2d9b78ce1 100644 --- a/visual/2013/fullbench/fullbench.vcxproj +++ b/projects/VS2010/fullbench/fullbench.vcxproj @@ -29,25 +29,21 @@ Application true MultiByte - v110 Application true - v110 MultiByte Application false - v110 true MultiByte Application false - v110 true MultiByte @@ -69,22 +65,24 @@ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(Platform)\$(Configuration)\ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(Platform)\$(Configuration)\ false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false @@ -127,7 +125,7 @@ true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false - true + false MultiThreaded @@ -146,7 +144,7 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true + false false MultiThreaded diff --git a/visual/2013/fullbench/fullbench.vcxproj.filters b/projects/VS2010/fullbench/fullbench.vcxproj.filters similarity index 100% rename from visual/2013/fullbench/fullbench.vcxproj.filters rename to projects/VS2010/fullbench/fullbench.vcxproj.filters diff --git a/visual/2013/fuzzer/fuzzer.vcxproj b/projects/VS2010/fuzzer/fuzzer.vcxproj similarity index 88% rename from visual/2013/fuzzer/fuzzer.vcxproj rename to projects/VS2010/fuzzer/fuzzer.vcxproj index bd3e849f4..047d5fd55 100644 --- a/visual/2013/fuzzer/fuzzer.vcxproj +++ b/projects/VS2010/fuzzer/fuzzer.vcxproj @@ -28,26 +28,22 @@ Application true - v110 MultiByte Application true - v110 MultiByte Application false - v110 true MultiByte Application false - v110 true MultiByte @@ -69,23 +65,25 @@ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); @@ -127,7 +125,7 @@ true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false - true + false MultiThreaded @@ -146,7 +144,7 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true + false false MultiThreaded diff --git a/visual/2013/fuzzer/fuzzer.vcxproj.filters b/projects/VS2010/fuzzer/fuzzer.vcxproj.filters similarity index 100% rename from visual/2013/fuzzer/fuzzer.vcxproj.filters rename to projects/VS2010/fuzzer/fuzzer.vcxproj.filters diff --git a/visual/2013/zstd.sln b/projects/VS2010/zstd.sln similarity index 100% rename from visual/2013/zstd.sln rename to projects/VS2010/zstd.sln diff --git a/visual/2013/zstd/zstd.vcxproj b/projects/VS2010/zstd/zstd.vcxproj old mode 100755 new mode 100644 similarity index 85% rename from visual/2013/zstd/zstd.vcxproj rename to projects/VS2010/zstd/zstd.vcxproj index d6f7edd66..8fff6320d --- a/visual/2013/zstd/zstd.vcxproj +++ b/projects/VS2010/zstd/zstd.vcxproj @@ -82,27 +82,23 @@ Application true - v110 MultiByte Application true - v110 MultiByte Application false true - v110 MultiByte Application false true - v110 MultiByte @@ -123,23 +119,29 @@ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath) + $(Platform)\$(Configuration)\ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath); false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath) + $(Platform)\$(Configuration)\ false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath); @@ -154,6 +156,7 @@ Console true + setargv.obj;%(AdditionalDependencies) @@ -163,12 +166,13 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - false + true false Console true + setargv.obj;%(AdditionalDependencies) @@ -176,12 +180,12 @@ Level4 - Full + MaxSpeed true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false - true + false MultiThreaded @@ -189,7 +193,7 @@ true true true - setargv.obj; kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + setargv.obj;%(AdditionalDependencies) @@ -210,7 +214,7 @@ true true true - setargv.obj;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + setargv.obj;%(AdditionalDependencies) diff --git a/visual/2013/zstd/zstd.vcxproj.filters b/projects/VS2010/zstd/zstd.vcxproj.filters old mode 100755 new mode 100644 similarity index 100% rename from visual/2013/zstd/zstd.vcxproj.filters rename to projects/VS2010/zstd/zstd.vcxproj.filters diff --git a/visual/2013/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj similarity index 88% rename from visual/2013/zstdlib/zstdlib.vcxproj rename to projects/VS2010/zstdlib/zstdlib.vcxproj index c13ff0af0..e8625a501 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -48,9 +48,6 @@ - - - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} Win32Proj @@ -61,27 +58,23 @@ DynamicLibrary true - v110 MultiByte DynamicLibrary true - v110 MultiByte DynamicLibrary false true - v110 MultiByte DynamicLibrary false true - v110 MultiByte @@ -104,28 +97,28 @@ true zstdlib_x86 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false true zstdlib_x64 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false zstdlib_x86 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false zstdlib_x64 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false @@ -155,7 +148,7 @@ Level4 Disabled ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - false + true EnableFastChecks MultiThreadedDebugDLL ProgramDatabase diff --git a/visual/2013/zstdlib/zstdlib.vcxproj.filters b/projects/VS2010/zstdlib/zstdlib.vcxproj.filters similarity index 90% rename from visual/2013/zstdlib/zstdlib.vcxproj.filters rename to projects/VS2010/zstdlib/zstdlib.vcxproj.filters index 327f3a26e..e4bebb546 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj.filters +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj.filters @@ -9,10 +9,6 @@ {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hh;hpp;hxx;hm;inl;inc;xsd - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - @@ -102,9 +98,4 @@ Header Files - - - Resource Files - - \ No newline at end of file diff --git a/visual/2013/zstdlib/resource.h b/visual/2013/zstdlib/resource.h deleted file mode 100644 index db984ef4d6c17963c0691479aa0061d0c8000ffe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 812 zcmezWPoF`bp_-wZ!H>b8A)dj7!IdF^!Ii-e%<^M!X7FTiWe8@dWvFG~W#D1}sa9a{ zWyoYmW+-CFXDDXKXGmixVNhTQW5{GEW+-JyWXNGqU~p#8X3%C(U`S_3WyoVlWhi1u zWGG=sWk_LAV8~?1V@PJmVJKxtVMt|AU`S)gWXNGiWza*iA(WvQsyd0Gl0ku?ilLaH zgdv3?har<8i9wH{h#?uS8sshzhPcg-A(f#5>>`lqX$*-Br3^V>ei_(pr3|SI#b8_W z8HyMbz^*H2P+-VsNMguhNM%T7C}Ajuo2>w5DKlg;q=8+V#-PC9$PmB~%n-s5%HYD_ z2@W++20w-{27d-`23H0bBprEp^aL@uGB`50F!(e0G59bf!r;l^&k)ZL#Nf){$l$`@&)~=4!w|_3&k)QI$>7W2#Nf~1!w?L%59G?h zi9vn@xeP`tFt}rhi8OE=<}xTSlrUs~LpOxMl_7#5gu#=+kHMM22OJ)*3Q($ld=O<7e%V1Do z&;auj!Feu~K@&@?7lTtiNIxjIfx<@-8d9K~3d-Rc42}$L3=z;g6wKhy5Xun5;LPC4 z5YOPk;KSepHn)bMhCu+a|Neb4+b3u1%_aTPzEOk?D8PjgD}JgkdgpfJcDvU7=u59H-jsK3)lsqup68h zZT7dxqv6sRz3a_O(cfLW?2F~l$wF~l(BL2ZY)QGo#@j@?DZWcr#^dz8VYZVE#t z11P6J+y#my;!+zZ>_8Z#V{l@a>tHmf9RRBBiy1N*@)`0NVB#Pegq2ZpyaGcMxMYZD z@L}*|aAJsO2xD+%2x15Z*TH@aAbC)&?1CW^&*018$l%4`&k)3*!w?0oyRoYRb^LHrQmb+W`wJ?4b#2!GtgbfkPEk9>;_0di<_|xdFS0kWv}smN153EO7)1dp8Du zaO)5`USZ-OouC@XlK~X^m~jtrC3d?ZdIK1O82lMr7(yAG8A7n69>ONLF+gm01J@Pq z48h=(70lqxpuk|jP{Cl#kjCHxPRF3qC=Hr^6~J|wD}xh5D1$p#7U2(sNe~kZ;rde< zau|vkQVFOxKvEBCg@DQqh;LzT^=Al%y3T~b01Qo_dJwJ%VTfc10N0TqUl?J?L+VFV zRWO^em-(RD)Stl_95PA_!3-q~Md0>bI)fWSCPNNGDuX9O9zz;KK3XY^-84v9r~s~` z5vCe2fP9q90HTxNwFyiujK*&wOjQ_zBe(?%Y9}Z$I5XrkXHkz+)50`4m)QLdqdfDl!D8 z4+Cha4RJ5RwFnj{Z{Tk4k=M@}oUJ~Pt3bX*#*o?(R9}HgG~(K~Aae#M2Kfx+G8j$i r_=z!=o;2$C3CJc82H83|G01he({~<2K6o4ubF>81+C%L{gG>Sd7At^m From 0ba773878fcbd3cefede2720420748c1cf908e0f Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 16 May 2016 14:54:48 +0200 Subject: [PATCH 03/26] updated .gitignore for Visual Studio --- .gitignore | 6 +++++- appveyor.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b6d5d1325..2d3b6917a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,11 @@ *.app # Visual C++ -visual/ +projects/VS2008 +projects/VS2010 +projects/VS2012 +projects/VS2013 +projects/VS2015 # IDEA solution files *.idea diff --git a/appveyor.yml b/appveyor.yml index 848db3836..064baf9f9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,7 +32,7 @@ build_script: ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% ECHO *** msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe ECHO *** ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% From e30bac4c4b66730fa83981f4927c920b1d76b414 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 16 May 2016 15:10:33 +0200 Subject: [PATCH 04/26] fixed Win32 in AppVeyor --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 064baf9f9..d37f48a60 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,7 @@ version: 1.0.{build} configuration: -- Release - Debug +- Release platform: - Win32 - x64 @@ -19,7 +19,8 @@ build_script: ECHO *** REM msbuild "projects\VS2010\zstd.sln" /m /verbosity:detailed /property:PlatformToolset=Windows7.1SDK /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" REM msbuild "projects\VS2010\zstd.sln" "/p:AdditionalLibPaths=C:\Program Files\Microsoft SDKs\Windows\v7.1\Lib\X64" /m /verbosity:detailed /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - msbuild "projects\VS2010\zstd.sln" /p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + If %PLATFORM%==x64 SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe ECHO *** From 5e334be4573dd3ff64f79b8b795a11721d6852ff Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 09:07:28 +0200 Subject: [PATCH 05/26] AppVeyor tests for mingw32, mingw64, clang --- appveyor.yml | 141 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 46 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d37f48a60..b5ffb4a25 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,51 +1,100 @@ version: 1.0.{build} -configuration: -- Debug -- Release -platform: -- Win32 -- x64 +environment: + matrix: + - ZSTD_APPVEYOR: "gcc" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + +install: + - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% + - if [%ZSTD_APPVEYOR%]==[gcc] ( + SET "MINGW_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen" && + SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && + SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && + SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && + COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe + ) else ( + IF [%PLATFORM%]==[x64] SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + ) build_script: -- cmd: | - ECHO *** - ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% - ECHO *** - REM msbuild "projects\VS2010\zstd.sln" /m /verbosity:detailed /property:PlatformToolset=Windows7.1SDK /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - REM msbuild "projects\VS2010\zstd.sln" "/p:AdditionalLibPaths=C:\Program Files\Microsoft SDKs\Windows\v7.1\Lib\X64" /m /verbosity:detailed /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - If %PLATFORM%==x64 SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + - ECHO Building %ZSTD_APPVEYOR% + - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building mingw32 && + ECHO *** && + ECHO make %MINGW_PARAMS% && + make %MINGW_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_mingw32.exe && + make clean + ) + - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building mingw64 && + ECHO *** && + ECHO make %MINGW_PARAMS% && + make %MINGW_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_mingw64.exe && + make clean && + ECHO *** && + ECHO *** Building clang && + ECHO *** && + ECHO make %CLANG_PARAMS% && + make %CLANG_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_clang.exe && + make clean + ) + - if [%ZSTD_APPVEYOR%]==[visual] ( + ECHO *** && + ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + ) test_script: -- cmd: SET FUZZERTEST=-T1mn -- cmd: | - IF %CONFIGURATION%==Release (projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%) + - SET FUZZERTEST=-T1mn + - if [%ZSTD_APPVEYOR%]==[gcc] ( + projects\fuzzer_mingw32.exe %FUZZERTEST% && + projects\fuzzer_mingw64.exe %FUZZERTEST% && + projects\fuzzer_clang.exe %FUZZERTEST% + ) + - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( + projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% + ) From f33021552b5036db1c143928f701d2b71355a4ec Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 16:38:36 +0200 Subject: [PATCH 06/26] added playTestsMSYS.sh --- appveyor.yml | 42 ++++---- programs/Makefile | 52 ++++++---- programs/{ => tests}/playTests.sh | 0 programs/tests/playTestsMSYS.sh | 160 ++++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 41 deletions(-) rename programs/{ => tests}/playTests.sh (100%) mode change 100755 => 100644 create mode 100644 programs/tests/playTestsMSYS.sh diff --git a/appveyor.yml b/appveyor.yml index b5ffb4a25..9a493a616 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,11 @@ version: 1.0.{build} environment: matrix: - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw32" + - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw64" - ZSTD_APPVEYOR: "visual" CONFIGURATION: "Debug" PLATFORM: "Win32" @@ -18,7 +23,6 @@ environment: install: - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%ZSTD_APPVEYOR%]==[gcc] ( - SET "MINGW_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen" && SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && @@ -29,24 +33,9 @@ install: build_script: - ECHO Building %ZSTD_APPVEYOR% - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw32 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw32.exe && - make clean - ) - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw64 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw64.exe && + - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] ( make clean && ECHO *** && ECHO *** Building clang && @@ -56,6 +45,15 @@ build_script: COPY programs\fuzzer.exe projects\fuzzer_clang.exe && make clean ) + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building %PLATFORM% && + ECHO *** && + ECHO make %MAKE_PARAMS% && + make %MAKE_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_%PLATFORM%.exe && + make clean + ) - if [%ZSTD_APPVEYOR%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && @@ -82,14 +80,14 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: - SET FUZZERTEST=-T1mn - if [%ZSTD_APPVEYOR%]==[gcc] ( - projects\fuzzer_mingw32.exe %FUZZERTEST% && - projects\fuzzer_mingw64.exe %FUZZERTEST% && - projects\fuzzer_clang.exe %FUZZERTEST% + projects\fuzzer_%PLATFORM%.exe %FUZZERTEST% && + if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && diff --git a/programs/Makefile b/programs/Makefile index 70434cedc..339427e2d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -163,7 +163,11 @@ clean: #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) +HOST_OS = POSIX +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) install: zstd @echo Installing binaries @@ -186,15 +190,41 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled +valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 +valgrindTest: zstd datagen fuzzer fullbench zbufftest + @echo "\n ---- valgrind tests : memory analyzer ----" + $(VALGRIND) ./datagen -g50M > $(VOID) + $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi + ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) + ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) + ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp + $(VALGRIND) ./zstd -vdf tmp -o $(VOID) + ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) + @rm tmp + $(VALGRIND) ./fuzzer -T1mn -t1 + $(VALGRIND) ./fullbench -i1 + $(VALGRIND) ./zbufftest -T1mn + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTestsMSYS.sh $(ZSTDRTTEST) +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) test: test-zstd test-fullbench test-fuzzer test-zbuff test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest -zstd-playTests: datagen - ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) - test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests @@ -223,20 +253,4 @@ test-zbuff: zbufftest test-zbuff32: zbufftest32 ./zbufftest32 $(ZBUFFTEST) - -valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: zstd datagen fuzzer fullbench zbufftest - @echo "\n ---- valgrind tests : memory analyzer ----" - $(VALGRIND) ./datagen -g50M > $(VOID) - $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi - ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) - ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp - $(VALGRIND) ./zstd -vdf tmp -o $(VOID) - ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) - @rm tmp - $(VALGRIND) ./fuzzer -T1mn -t1 - $(VALGRIND) ./fullbench -i1 - $(VALGRIND) ./zbufftest -T1mn - endif diff --git a/programs/playTests.sh b/programs/tests/playTests.sh old mode 100755 new mode 100644 similarity index 100% rename from programs/playTests.sh rename to programs/tests/playTests.sh diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh new file mode 100644 index 000000000..82f2357bd --- /dev/null +++ b/programs/tests/playTestsMSYS.sh @@ -0,0 +1,160 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 + fc tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + + +echo "\n**** simple tests **** " +./datagen > tmp +$ZSTD -f tmp # trivial compression case, creates tmp.zst +$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) +echo "test : too large compression level (must fail)" +$ZSTD -99 tmp && die "too large compression level undetected" +echo "test : compress to stdout" +$ZSTD tmp -c > tmpCompressed +$ZSTD tmp --stdout > tmpCompressed # long command format +echo "test : null-length file roundtrip" +echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout +echo "test : decompress file with wrong suffix (must fail)" +$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" +$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout +$ZSTD --decompress tmpCompressed -c > tmpResult +$ZSTD --decompress tmpCompressed --stdout > tmpResult +#$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout +#$ZSTD -d - < tmp.zst > /dev/null +$ZSTD -dc < tmp.zst > /dev/null +$ZSTD -dc - < tmp.zst > /dev/null +$ZSTD -q tmp && die "overwrite check failed!" +$ZSTD -q -f tmp +$ZSTD -q --force tmp +$ZSTD -df tmp && die "should have refused : wrong extension" +cp tmp tmp2.zst +$ZSTD -df tmp2.zst && die "should have failed : wrong format" +rm tmp2.zst + +echo "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD -c hello.tmp > hello.zstd +$ZSTD -c world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -dc helloworld.zstd > result.tmp +cat result.tmp +fc helloworld.tmp result.tmp +rm ./*.tmp ./*.zstd + +echo frame concatenation test completed + + +echo "\n**** dictionary tests **** " + +./datagen > tmpDict +./datagen -g1M | md5sum > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 +fc tmp1 tmp2 +$ZSTD --train *.c *.h -o tmpDict +$ZSTD xxhash.c -D tmpDict -of tmp +$ZSTD -d tmp -D tmpDict -of result +fc xxhash.c result + + +echo "\n**** multiple files tests **** " + +./datagen -s1 > tmp1 2> /dev/null +./datagen -s2 -g100K > tmp2 2> /dev/null +./datagen -s3 -g1M > tmp3 2> /dev/null +$ZSTD -f tmp* +echo "compress tmp* : " +ls -ls tmp* +rm tmp1 tmp2 tmp3 +echo "decompress tmp* : " +$ZSTD -df *.zst +ls -ls tmp* +echo "compress tmp* into stdout > tmpall : " +$ZSTD -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* +echo "decompress tmpall* into stdout > tmpdec : " +cp tmpall tmpall2 +# $ZSTD -dc tmpall* > tmpdec +ls -ls tmp* +echo "compress multiple files including a missing one (notHere) : " +$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" + +echo "\n**** integrity tests **** " +echo "test one file (tmp1.zst) " +$ZSTD -t tmp1.zst +$ZSTD --test tmp1.zst +echo "test multiple files (*.zst) " +$ZSTD -t *.zst +echo "test good and bad files (*) " +$ZSTD -t * && die "bad files not detected !" + +echo "\n**** zstd round-trip tests **** " + +roundTripTest +roundTripTest -g15K # TableID==3 +roundTripTest -g127K # TableID==2 +roundTripTest -g255K # TableID==1 +roundTripTest -g513K # TableID==0 +roundTripTest -g512K 6 # greedy, hash chain +roundTripTest -g512K 16 # btlazy2 +roundTripTest -g512K 19 # btopt + +rm tmp* + +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 + +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 1 + +rm tmp* + From 877cc84f859cd910355682d65e502349afae5ca8 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:03:02 +0200 Subject: [PATCH 07/26] changed appveyor.yml --- .gitattributes | 3 +++ appveyor.yml | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index 3959727b8..f314af9ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -14,3 +14,6 @@ *.vcxproj* text eol=crlf *.suo binary *.rc binary + +# Windows +*.bat text eol=crlf diff --git a/appveyor.yml b/appveyor.yml index 9a493a616..b79ef2a59 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,9 @@ install: SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe ) else ( - IF [%PLATFORM%]==[x64] SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") && + COPY programs\tests\playTests.bat playTests.bat && + COPY programs\tests\roundTripTest.bat roundTripTest.bat ) build_script: @@ -51,7 +53,6 @@ build_script: ECHO *** && ECHO make %MAKE_PARAMS% && make %MAKE_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_%PLATFORM%.exe && make clean ) - if [%ZSTD_APPVEYOR%]==[visual] ( @@ -79,14 +80,13 @@ build_script: ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: - SET FUZZERTEST=-T1mn - if [%ZSTD_APPVEYOR%]==[gcc] ( - projects\fuzzer_%PLATFORM%.exe %FUZZERTEST% && if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( @@ -94,5 +94,6 @@ test_script: projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && + zstd_VS2015_%PLATFORM%_Release.exe ) From 3aa4b2b598686973525a540e00247405d37a3d92 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:04:10 +0200 Subject: [PATCH 08/26] update file permission --- programs/tests/playTests.sh | 0 programs/tests/playTestsMSYS.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 programs/tests/playTests.sh mode change 100644 => 100755 programs/tests/playTestsMSYS.sh diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh old mode 100644 new mode 100755 diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh old mode 100644 new mode 100755 From 2bd6b13e5337ed10af0292963e51b1ac38ab3258 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:05:56 +0200 Subject: [PATCH 09/26] added playTests.bat --- programs/tests/playTests.bat | 149 +++++++++++++++++++++++++++++++ programs/tests/roundTripTest.bat | 8 ++ 2 files changed, 157 insertions(+) create mode 100644 programs/tests/playTests.bat create mode 100644 programs/tests/roundTripTest.bat diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat new file mode 100644 index 000000000..19c57d5fc --- /dev/null +++ b/programs/tests/playTests.bat @@ -0,0 +1,149 @@ +@echo off +if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 + +echo. && echo **** simple tests **** +datagen > tmp +%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst +%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) +echo test : too large compression level (must fail) +%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1) +echo test : compress to stdout +%ZSTD% tmp -c > tmpCompressed +%ZSTD% tmp --stdout > tmpCompressed && REM long command format +echo test : null-length file roundtrip +echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1) +echo test : decompress file with wrong suffix (must fail) +%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1) +%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout +%ZSTD% --decompress tmpCompressed -c > tmpResult +%ZSTD% --decompress tmpCompressed --stdout > tmpResult +REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout +REM %ZSTD% -d - < tmp.zst > NUL +%ZSTD% -dc < tmp.zst > NUL +%ZSTD% -dc - < tmp.zst > NUL +%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1) +%ZSTD% -q -f tmp +%ZSTD% -q --force tmp +%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) +cp tmp tmp2.zst +%ZSTD% -df tmp2.zst && (echo should have failed : wrong format && exit /b 1) +rm tmp2.zst + +echo. && echo **** frame concatenation **** + +echo hello > hello.tmp +echo world! > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +%ZSTD% -c hello.tmp > hello.zstd +%ZSTD% -c world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +%ZSTD% -dc helloworld.zstd > result.tmp +cat result.tmp +fc /b helloworld.tmp result.tmp +rm *.tmp *.zstd + +echo frame concatenation test completed + + +REM echo. && echo **** flush write error test **** + +REM echo echo foo ^| %ZSTD% ^> v:\full +REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1) +REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full" +REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1) + + +echo. && echo **** dictionary tests **** + +datagen > tmpDict +datagen -g1M | md5sum > tmp1 +datagen -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 +fc tmp1 tmp2 +%ZSTD% --train *.c *.h -o tmpDict +%ZSTD% xxhash.c -D tmpDict -of tmp +%ZSTD% -d tmp -D tmpDict -of result +fc xxhash.c result + + +echo. && echo **** multiple files tests **** + +datagen -s1 > tmp1 2> NUL +datagen -s2 -g100K > tmp2 2> NUL +datagen -s3 -g1M > tmp3 2> NUL +%ZSTD% -f tmp* +echo compress tmp* : +ls -ls tmp* +rm tmp1 tmp2 tmp3 +echo decompress tmp* : +%ZSTD% -df *.zst +ls -ls tmp* + +echo compress tmp* into stdout ^> tmpall : +%ZSTD% -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* +echo decompress tmpall* into stdout ^> tmpdec : +cp tmpall tmpall2 +%ZSTD% -dc tmpall* > tmpdec +ls -ls tmp* +echo compress multiple files including a missing one (notHere) : +%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1) + + +echo. && echo **** integrity tests **** +echo test one file (tmp1.zst) +%ZSTD% -t tmp1.zst +%ZSTD% --test tmp1.zst +echo test multiple files (*.zst) +%ZSTD% -t *.zst +echo test good and bad files (*) +%ZSTD% -t * && (echo bad files not detected! && exit /b 1) + + +echo. && echo **** zstd round-trip tests **** + +CALL roundTripTest.bat +CALL roundTripTest.bat -g15K && REM TableID==3 +CALL roundTripTest.bat -g127K && REM TableID==2 +CALL roundTripTest.bat -g255K && REM TableID==1 +CALL roundTripTest.bat -g513K && REM TableID==0 +CALL roundTripTest.bat -g512K 6 && REM greedy, hash chain +CALL roundTripTest.bat -g512K 16 && REM btlazy2 +CALL roundTripTest.bat -g512K 19 && REM btopt + +rm tmp* +echo Param = %1 +if NOT "%1"=="--test-large-data" ( + echo skipping large data tests + exit /b 0 +) + +CALL roundTripTest.bat -g270000000 1 +CALL roundTripTest.bat -g270000000 2 +CALL roundTripTest.bat -g270000000 3 + +CALL roundTripTest.bat -g140000000 -P60 4 +CALL roundTripTest.bat -g140000000 -P60 5 +CALL roundTripTest.bat -g140000000 -P60 6 + +CALL roundTripTest.bat -g70000000 -P70 7 +CALL roundTripTest.bat -g70000000 -P70 8 +CALL roundTripTest.bat -g70000000 -P70 9 + +CALL roundTripTest.bat -g35000000 -P75 10 +CALL roundTripTest.bat -g35000000 -P75 11 +CALL roundTripTest.bat -g35000000 -P75 12 + +CALL roundTripTest.bat -g18000000 -P80 13 +CALL roundTripTest.bat -g18000000 -P80 14 +CALL roundTripTest.bat -g18000000 -P80 15 +CALL roundTripTest.bat -g18000000 -P80 16 +CALL roundTripTest.bat -g18000000 -P80 17 + +CALL roundTripTest.bat -g50000000 -P94 18 +CALL roundTripTest.bat -g50000000 -P94 19 + +CALL roundTripTest.bat -g99000000 -P99 20 +CALL roundTripTest.bat -g6000000000 -P99 1 + +rm tmp* +exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat new file mode 100644 index 000000000..b1541bd14 --- /dev/null +++ b/programs/tests/roundTripTest.bat @@ -0,0 +1,8 @@ +@echo off +if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) +rm -f tmp1 tmp2 +echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d +datagen %1 %P% | md5sum > tmp1 +datagen %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 +fc tmp1 tmp2 +EXIT /B %ERRORLEVEL% From 60af95d9718a54c165937fb8f325e1d278b61565 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 10:29:49 +0200 Subject: [PATCH 10/26] fixed compression with -c with multiple files on Windows --- appveyor.yml | 2 +- programs/fileio.c | 3 ++- programs/tests/playTestsMSYS.sh | 14 +++++++------- programs/zstdcli.c | 5 +---- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b79ef2a59..e002d89ef 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -95,5 +95,5 @@ test_script: projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - zstd_VS2015_%PLATFORM%_Release.exe + REM zstd_VS2015_%PLATFORM%_Release.exe ) diff --git a/programs/fileio.c b/programs/fileio.c index b3f324ac6..638e63b57 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -71,7 +71,7 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ # include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); } #else # include /* isatty */ # define SET_BINARY_MODE(file) @@ -439,6 +439,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile if (!strcmp(suffix, stdoutmark)) { unsigned u; ress.dstFile = stdout; + SET_BINARY_MODE(stdout); for (u=0; u tmp $ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) @@ -51,7 +51,7 @@ cp tmp tmp2.zst $ZSTD -df tmp2.zst && die "should have failed : wrong format" rm tmp2.zst -echo "\n**** frame concatenation **** " +echo -e "\n**** frame concatenation **** " echo "hello " > hello.tmp echo "world!" > world.tmp @@ -67,7 +67,7 @@ rm ./*.tmp ./*.zstd echo frame concatenation test completed -echo "\n**** dictionary tests **** " +echo -e "\n**** dictionary tests **** " ./datagen > tmpDict ./datagen -g1M | md5sum > tmp1 @@ -79,7 +79,7 @@ $ZSTD -d tmp -D tmpDict -of result fc xxhash.c result -echo "\n**** multiple files tests **** " +echo -e "\n**** multiple files tests **** " ./datagen -s1 > tmp1 2> /dev/null ./datagen -s2 -g100K > tmp2 2> /dev/null @@ -96,12 +96,12 @@ $ZSTD -c tmp1 tmp2 tmp3 > tmpall ls -ls tmp* echo "decompress tmpall* into stdout > tmpdec : " cp tmpall tmpall2 -# $ZSTD -dc tmpall* > tmpdec +$ZSTD -dc tmpall* > tmpdec ls -ls tmp* echo "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" -echo "\n**** integrity tests **** " +echo -e "\n**** integrity tests **** " echo "test one file (tmp1.zst) " $ZSTD -t tmp1.zst $ZSTD --test tmp1.zst @@ -110,7 +110,7 @@ $ZSTD -t *.zst echo "test good and bad files (*) " $ZSTD -t * && die "bad files not detected !" -echo "\n**** zstd round-trip tests **** " +echo -e "\n**** zstd round-trip tests **** " roundTripTest roundTripTest -g15K # TableID==3 diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 310db7f88..8a8af3a2c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -49,14 +49,11 @@ * OS-specific Includes **************************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else extern int fileno(FILE *stream); /* triggers fileno() within on POSIX */ # include /* isatty */ -# define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #endif From 44bb2f6c89161915c8ff46ddc3b3492116b1c377 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 12:16:14 +0200 Subject: [PATCH 11/26] added datagen in Visual Studio project --- appveyor.yml | 12 +- programs/datagencli.c | 2 +- programs/tests/playTests.bat | 75 ++++---- programs/tests/roundTripTest.bat | 4 +- projects/VS2010/datagen/datagen.vcxproj | 168 ++++++++++++++++++ .../VS2010/datagen/datagen.vcxproj.filters | 26 +++ projects/VS2010/zstd.sln | 10 ++ 7 files changed, 253 insertions(+), 44 deletions(-) create mode 100644 projects/VS2010/datagen/datagen.vcxproj create mode 100644 projects/VS2010/datagen/datagen.vcxproj.filters diff --git a/appveyor.yml b/appveyor.yml index e002d89ef..a68e12eab 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,9 +28,7 @@ install: SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe ) else ( - IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") && - COPY programs\tests\playTests.bat playTests.bat && - COPY programs\tests\roundTripTest.bat roundTripTest.bat + IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") ) build_script: @@ -81,7 +79,8 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\datagen.exe projects\datagen_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: @@ -95,5 +94,8 @@ test_script: projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - REM zstd_VS2015_%PLATFORM%_Release.exe + CD programs && + SET ZSTD=..\projects\zstd_VS2015_%PLATFORM%_Release.exe && + SET DATAGEN=..\projects\datagen_VS2015_%PLATFORM%_Release.exe && + CALL tests\playTests.bat --test-large-data ) diff --git a/programs/datagencli.c b/programs/datagencli.c index 6122c29bd..d437d5cb3 100644 --- a/programs/datagencli.c +++ b/programs/datagencli.c @@ -27,8 +27,8 @@ /*-************************************ * Includes **************************************/ +#include "util.h" /* Compiler options */ #include /* fprintf, stderr */ -#include "mem.h" #include "datagen.h" /* RDG_generate */ diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index 19c57d5fc..6e9fd53ff 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -1,8 +1,11 @@ @echo off if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 +if [%DATAGEN%]==[] echo DATAGEN variable must be defined! && exit /b 1 + +SET ROUNDTRIPTEST=tests\roundTripTest.bat echo. && echo **** simple tests **** -datagen > tmp +%DATAGEN% > tmp %ZSTD% -f tmp && REM trivial compression case, creates tmp.zst %ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) echo test : too large compression level (must fail) @@ -55,9 +58,9 @@ REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! echo. && echo **** dictionary tests **** -datagen > tmpDict -datagen -g1M | md5sum > tmp1 -datagen -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 +%DATAGEN% > tmpDict +%DATAGEN% -g1M | md5sum > tmp1 +%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 fc tmp1 tmp2 %ZSTD% --train *.c *.h -o tmpDict %ZSTD% xxhash.c -D tmpDict -of tmp @@ -67,9 +70,9 @@ fc xxhash.c result echo. && echo **** multiple files tests **** -datagen -s1 > tmp1 2> NUL -datagen -s2 -g100K > tmp2 2> NUL -datagen -s3 -g1M > tmp3 2> NUL +%DATAGEN% -s1 > tmp1 2> NUL +%DATAGEN% -s2 -g100K > tmp2 2> NUL +%DATAGEN% -s3 -g1M > tmp3 2> NUL %ZSTD% -f tmp* echo compress tmp* : ls -ls tmp* @@ -101,14 +104,14 @@ echo test good and bad files (*) echo. && echo **** zstd round-trip tests **** -CALL roundTripTest.bat -CALL roundTripTest.bat -g15K && REM TableID==3 -CALL roundTripTest.bat -g127K && REM TableID==2 -CALL roundTripTest.bat -g255K && REM TableID==1 -CALL roundTripTest.bat -g513K && REM TableID==0 -CALL roundTripTest.bat -g512K 6 && REM greedy, hash chain -CALL roundTripTest.bat -g512K 16 && REM btlazy2 -CALL roundTripTest.bat -g512K 19 && REM btopt +CALL %ROUNDTRIPTEST% +CALL %ROUNDTRIPTEST% -g15K && REM TableID==3 +CALL %ROUNDTRIPTEST% -g127K && REM TableID==2 +CALL %ROUNDTRIPTEST% -g255K && REM TableID==1 +CALL %ROUNDTRIPTEST% -g513K && REM TableID==0 +CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain +CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2 +CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt rm tmp* echo Param = %1 @@ -117,33 +120,33 @@ if NOT "%1"=="--test-large-data" ( exit /b 0 ) -CALL roundTripTest.bat -g270000000 1 -CALL roundTripTest.bat -g270000000 2 -CALL roundTripTest.bat -g270000000 3 +CALL %ROUNDTRIPTEST% -g270000000 1 +CALL %ROUNDTRIPTEST% -g270000000 2 +CALL %ROUNDTRIPTEST% -g270000000 3 -CALL roundTripTest.bat -g140000000 -P60 4 -CALL roundTripTest.bat -g140000000 -P60 5 -CALL roundTripTest.bat -g140000000 -P60 6 +CALL %ROUNDTRIPTEST% -g140000000 -P60 4 +CALL %ROUNDTRIPTEST% -g140000000 -P60 5 +CALL %ROUNDTRIPTEST% -g140000000 -P60 6 -CALL roundTripTest.bat -g70000000 -P70 7 -CALL roundTripTest.bat -g70000000 -P70 8 -CALL roundTripTest.bat -g70000000 -P70 9 +CALL %ROUNDTRIPTEST% -g70000000 -P70 7 +CALL %ROUNDTRIPTEST% -g70000000 -P70 8 +CALL %ROUNDTRIPTEST% -g70000000 -P70 9 -CALL roundTripTest.bat -g35000000 -P75 10 -CALL roundTripTest.bat -g35000000 -P75 11 -CALL roundTripTest.bat -g35000000 -P75 12 +CALL %ROUNDTRIPTEST% -g35000000 -P75 10 +CALL %ROUNDTRIPTEST% -g35000000 -P75 11 +CALL %ROUNDTRIPTEST% -g35000000 -P75 12 -CALL roundTripTest.bat -g18000000 -P80 13 -CALL roundTripTest.bat -g18000000 -P80 14 -CALL roundTripTest.bat -g18000000 -P80 15 -CALL roundTripTest.bat -g18000000 -P80 16 -CALL roundTripTest.bat -g18000000 -P80 17 +CALL %ROUNDTRIPTEST% -g18000000 -P80 13 +CALL %ROUNDTRIPTEST% -g18000000 -P80 14 +CALL %ROUNDTRIPTEST% -g18000000 -P80 15 +CALL %ROUNDTRIPTEST% -g18000000 -P80 16 +CALL %ROUNDTRIPTEST% -g18000000 -P80 17 -CALL roundTripTest.bat -g50000000 -P94 18 -CALL roundTripTest.bat -g50000000 -P94 19 +CALL %ROUNDTRIPTEST% -g50000000 -P94 18 +CALL %ROUNDTRIPTEST% -g50000000 -P94 19 -CALL roundTripTest.bat -g99000000 -P99 20 -CALL roundTripTest.bat -g6000000000 -P99 1 +CALL %ROUNDTRIPTEST% -g99000000 -P99 20 +CALL %ROUNDTRIPTEST% -g6000000000 -P99 1 rm tmp* exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat index b1541bd14..759891f0b 100644 --- a/programs/tests/roundTripTest.bat +++ b/programs/tests/roundTripTest.bat @@ -2,7 +2,7 @@ if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) rm -f tmp1 tmp2 echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d -datagen %1 %P% | md5sum > tmp1 -datagen %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 +%DATAGEN% %1 %P% | md5sum > tmp1 +%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 fc tmp1 tmp2 EXIT /B %ERRORLEVEL% diff --git a/projects/VS2010/datagen/datagen.vcxproj b/projects/VS2010/datagen/datagen.vcxproj new file mode 100644 index 000000000..fe2b7eec6 --- /dev/null +++ b/projects/VS2010/datagen/datagen.vcxproj @@ -0,0 +1,168 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {037E781E-81A6-494B-B1B3-438AB1200523} + Win32Proj + datagen + $(SolutionDir)bin\$(Platform)\$(Configuration)\ + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + true + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ + + + true + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + + + false + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ + + + false + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + + + + + + + + + \ No newline at end of file diff --git a/projects/VS2010/datagen/datagen.vcxproj.filters b/projects/VS2010/datagen/datagen.vcxproj.filters new file mode 100644 index 000000000..1ebbd6b03 --- /dev/null +++ b/projects/VS2010/datagen/datagen.vcxproj.filters @@ -0,0 +1,26 @@ + + + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + Source Files + + + + + Header Files + + + \ No newline at end of file diff --git a/projects/VS2010/zstd.sln b/projects/VS2010/zstd.sln index 825d7a130..698b8fe50 100644 --- a/projects/VS2010/zstd.sln +++ b/projects/VS2010/zstd.sln @@ -9,6 +9,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\full EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -49,6 +51,14 @@ Global {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32 {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64 {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.ActiveCfg = Debug|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.Build.0 = Debug|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.ActiveCfg = Debug|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.Build.0 = Debug|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.ActiveCfg = Release|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.Build.0 = Release|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.ActiveCfg = Release|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 96abb5a806941d07ae5e6de91ad62c566c7c40c5 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 15:50:58 +0200 Subject: [PATCH 12/26] AppVeyor tests use playTests.bat --- appveyor.yml | 64 +++++++++++++++++++----------------- programs/tests/playTests.bat | 7 ++-- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a68e12eab..c6f735ea3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,28 +1,29 @@ version: 1.0.{build} environment: matrix: - - ZSTD_APPVEYOR: "gcc" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" + - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw32" - - ZSTD_APPVEYOR: "gcc" + - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw64" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" install: - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% + - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% + - if [%COMPILER%]==[gcc] ( SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && @@ -32,7 +33,7 @@ install: ) build_script: - - ECHO Building %ZSTD_APPVEYOR% + - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] ( @@ -45,7 +46,7 @@ build_script: COPY programs\fuzzer.exe projects\fuzzer_clang.exe && make clean ) - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - if [%COMPILER%]==[gcc] ( ECHO *** && ECHO *** Building %PLATFORM% && ECHO *** && @@ -53,7 +54,7 @@ build_script: make %MAKE_PARAMS% && make clean ) - - if [%ZSTD_APPVEYOR%]==[visual] ( + - if [%COMPILER%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -62,7 +63,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && @@ -77,25 +78,28 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\datagen.exe projects\datagen_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\ ) test_script: + - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - SET FUZZERTEST=-T1mn - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - if [%COMPILER%]==[gcc] ( if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( + - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( + CD programs && + SET ZSTD=..\projects\zstd.exe && + SET DATAGEN=..\projects\datagen.exe && + CALL tests\playTests.bat --test-large-data && + CD .. && + projects\fullbench.exe -i1 && + projects\fullbench.exe -i1 -P0 && projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - CD programs && - SET ZSTD=..\projects\zstd_VS2015_%PLATFORM%_Release.exe && - SET DATAGEN=..\projects\datagen_VS2015_%PLATFORM%_Release.exe && - CALL tests\playTests.bat --test-large-data + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index 6e9fd53ff..edbbcb340 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -1,6 +1,9 @@ @echo off -if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 -if [%DATAGEN%]==[] echo DATAGEN variable must be defined! && exit /b 1 +ECHO Start playTests.bat +ECHO ZSTD^: %ZSTD% +ECHO DATAGEN^: %DATAGEN% +if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1 +if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1 SET ROUNDTRIPTEST=tests\roundTripTest.bat From 35b891c5b02608ddb09b5765104c52bf0819745d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 20 May 2016 19:42:20 +0200 Subject: [PATCH 13/26] zstd_stats.h updated to v0.6 --- lib/common/zstd_internal.h | 14 +--- lib/compress/.debug/zstd_stats.h | 135 ++++++++++++++++--------------- lib/compress/zstd_compress.c | 11 ++- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 4c9a7615a..8d9883d0f 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -186,6 +186,8 @@ MEM_STATIC unsigned ZSTD_highbit(U32 val) /*-******************************************* * Private interfaces *********************************************/ +typedef struct ZSTD_stats_s ZSTD_stats_t; + typedef struct { U32 off; U32 len; @@ -199,16 +201,6 @@ typedef struct { U32 rep[ZSTD_REP_INIT]; } ZSTD_optimal_t; -#if ZSTD_OPT_DEBUG == 3 - #include ".debug/zstd_stats.h" -#else - typedef struct { U32 unused; } ZSTD_stats_t; - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif - typedef struct { void* buffer; U32* offsetStart; @@ -245,7 +237,7 @@ typedef struct { U32 cachedPrice; U32 cachedLitLength; const BYTE* cachedLiterals; - ZSTD_stats_t stats; + ZSTD_stats_t* stats; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index d0189f872..51406a930 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -37,31 +37,21 @@ extern "C" { #endif -/*-************************************* -* Dependencies -***************************************/ -//#include "zstd.h" -//#include "mem.h" - - -/*-************************************* -* Constants -***************************************/ -//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#if ZSTD_OPT_DEBUG == 3 /*-************************************* * Types ***************************************/ -typedef struct { - U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength; +struct ZSTD_stats_s { + U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength; U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum; U32 litSum, matchLengthSum, litLengthSum, offCodeSum; - U32 matchLengthFreq[1<totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); - printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); - printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8); + printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); + printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8); } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1; - stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0; + stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { unsigned u; - stats->litSum = (1<litLengthSum = (1<matchLengthSum = (1<offCodeSum = (1<litSum = (2<litLengthSum = MaxLL+1; + stats->matchLengthSum = MaxML+1; + stats->offCodeSum = (MaxOff+1); for (u=0; u<=MaxLit; u++) stats->litFreq[u] = 1; @@ -99,62 +91,75 @@ MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) stats->offCodeFreq[u] = 1; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { - /* offset */ - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; - stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); - stats->priceOffset += (offCode-1) + (!offCode); - - /* match Length */ - stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); - stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1); - - if (litLength) { - /* literals */ - U32 u; - stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); - for (u=0; u < litLength; u++) - stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); - - /* literal Length */ - stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1); - } else { - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); - } - - - if (offset == 0) stats->totalRepSum++; - stats->totalSeqSum++; - stats->totalMatchSum += matchLength; - stats->totalLitSum += litLength; - U32 u; /* literals */ + stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); + for (u=0; u < litLength; u++) + stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); stats->litSum += litLength; for (u=0; u < litLength; u++) stats->litFreq[literals[u]]++; /* literal Length */ - stats->litLengthSum++; - if (litLength >= MaxLL) - stats->litLengthFreq[MaxLL]++; - else - stats->litLengthFreq[litLength]++; + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + if (litLength) { + stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1); + } else { + stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); + } + stats->litLengthFreq[llCode]++; + stats->litLengthSum++; + } /* match offset */ - stats->offCodeSum++; - stats->offCodeFreq[offCode]++; + { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); + stats->priceOffset += offCode; + stats->offCodeSum++; + stats->offCodeFreq[offCode]++; + } /* match Length */ - stats->matchLengthSum++; - if (matchLength >= MaxML) - stats->matchLengthFreq[MaxML]++; - else - stats->matchLengthFreq[matchLength]++; + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1); + stats->matchLengthFreq[mlCode]++; + stats->matchLengthSum++; + } + + if (offset == 0) stats->totalRepSum++; + stats->totalSeqSum++; + stats->totalMatchSum += matchLength; + stats->totalLitSum += litLength; } +#else + struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } +#endif // #if ZSTD_OPT_DEBUG == 3 #if defined (__cplusplus) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1bb75c68c..988a74fb7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -57,6 +57,7 @@ #include "fse_static.h" #include "huf_static.h" #include "zstd_internal.h" +#include ".debug/zstd_stats.h" /*-************************************* @@ -856,7 +857,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode); + ZSTD_statsUpdatePrices(seqStorePtr->stats, litLength, literals, offsetCode, matchCode); /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); @@ -1748,7 +1749,7 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; - ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); + ZSTD_statsUpdatePrices(seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -2020,8 +2021,9 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = &zc->seqStore.stats; - + ZSTD_stats_t* stats = malloc(sizeof(ZSTD_stats_t)); + if (!stats) return ERROR(memory_allocation); + zc->seqStore.stats = stats; ZSTD_statsInit(stats); while (remaining) { @@ -2059,6 +2061,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); + free(stats); return op-ostart; } From 50e82c015d56c00879bb8d2d551d117b840a9804 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 15:49:09 +0200 Subject: [PATCH 14/26] added ZSTD_createCCtx_advanced --- lib/common/zstd_static.h | 9 ++++++++ lib/compress/zstd_compress.c | 45 +++++++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index ef42b0a97..912eb1335 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -94,10 +94,19 @@ typedef struct { ZSTD_frameParameters fParams; } ZSTD_parameters; +typedef void* (*ZSTD_allocFunction) (size_t size); +typedef void (*ZSTD_freeFunction) (void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem; + /*-************************************* * Advanced functions ***************************************/ +/*! ZSTD_createCCtx_advanced() : + * Create ZSTD context using external alloc and free functions */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); + + ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getCParams() : diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 988a74fb7..00106f5fe 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -104,6 +104,8 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -118,13 +120,32 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - return (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + ZSTD_CCtx* ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + ctx->customAlloc = malloc; + ctx->customFree = free; + return ctx; +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + if (!customMem.customAlloc || !customMem.customFree) + return ZSTD_createCCtx(); + + ZSTD_CCtx* ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + memset(ctx, 0, sizeof(ZSTD_CCtx)); + ctx->customAlloc = customMem.customAlloc; + ctx->customFree = customMem.customFree; + return ctx; } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - free(cctx->workSpace); - free(cctx); + cctx->customFree(cctx->workSpace); + cctx->customFree(cctx); return 0; /* reserved as a potential error code in the future */ } @@ -229,8 +250,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); if (zc->workSpaceSize < neededSpace) { - free(zc->workSpace); - zc->workSpace = malloc(neededSpace); + zc->customFree(zc->workSpace); + zc->workSpace = zc->customAlloc(neededSpace); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; } } @@ -2021,7 +2042,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = malloc(sizeof(ZSTD_stats_t)); + ZSTD_stats_t* stats = (ZSTD_stats_t*) zc->customAlloc(sizeof(ZSTD_stats_t)); if (!stats) return ERROR(memory_allocation); zc->seqStore.stats = stats; ZSTD_statsInit(stats); @@ -2030,7 +2051,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { zc->customFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2041,11 +2062,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; + if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; + if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2061,7 +2082,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - free(stats); + zc->customFree(stats); return op-ostart; } @@ -2427,8 +2448,10 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); + ctxBody.customAlloc = malloc; + ctxBody.customFree = free; result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); - free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ + ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } From 107e2431955a7bb0d78f31d74bddd880f2250f75 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 16:24:52 +0200 Subject: [PATCH 15/26] added ZSTD_createDCtx_advanced --- lib/common/zstd_static.h | 5 ++++- lib/compress/zstd_compress.c | 27 +++++++++++++++++---------- lib/decompress/zstd_decompress.c | 31 ++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index 912eb1335..e0f50cf40 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -103,9 +103,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } * Advanced functions ***************************************/ /*! ZSTD_createCCtx_advanced() : - * Create ZSTD context using external alloc and free functions */ + * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API unsigned ZSTD_maxCLevel (void); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 00106f5fe..966d9acbd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -120,20 +120,25 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - ZSTD_CCtx* ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); - if (!ctx) return NULL; - - ctx->customAlloc = malloc; - ctx->customFree = free; - return ctx; + ZSTD_customMem customMem = { NULL, NULL }; + return ZSTD_createCCtx_advanced(customMem); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { - if (!customMem.customAlloc || !customMem.customFree) - return ZSTD_createCCtx(); + ZSTD_CCtx* ctx; - ZSTD_CCtx* ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + if (!customMem.customAlloc || !customMem.customFree) + { + ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + ctx->customAlloc = malloc; + ctx->customFree = free; + return ctx; + } + + ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); if (!ctx) return NULL; memset(ctx, 0, sizeof(ZSTD_CCtx)); @@ -307,7 +312,9 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { if (srcCCtx->stage!=1) return ERROR(stage_wrong); - dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ + dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ + dstCCtx->customAlloc = srcCCtx->customAlloc; + dstCCtx->customFree = srcCCtx->customFree; ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 177e9c8a2..a3eee709e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -116,6 +116,8 @@ struct ZSTD_DCtx_s size_t expected; size_t headerSize; ZSTD_frameParams fParams; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 flagRepeatTable; @@ -143,15 +145,38 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) ZSTD_DCtx* ZSTD_createDCtx(void) { - ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx)); - if (dctx==NULL) return NULL; + ZSTD_customMem customMem = { NULL, NULL }; + return ZSTD_createDCtx_advanced(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_DCtx* dctx; + + if (!customMem.customAlloc || !customMem.customFree) + { + dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); + if (!dctx) return NULL; + dctx->customAlloc = malloc; + dctx->customFree = free; + + ZSTD_decompressBegin(dctx); + return dctx; + } + + dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); + if (!dctx) return NULL; + dctx->customAlloc = customMem.customAlloc; + dctx->customFree = customMem.customFree; + ZSTD_decompressBegin(dctx); return dctx; } + size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { - free(dctx); + dctx->customFree(dctx); return 0; /* reserved as a potential error code in the future */ } From 13ba880b4974817cca5962c0c94acbd1cc3aec16 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 17:04:23 +0200 Subject: [PATCH 16/26] added ZBUFF_createCCtx_advanced and ZBUFF_createDCtx_advanced --- lib/common/zbuff.h | 5 +++- lib/compress/zbuff_compress.c | 45 +++++++++++++++++++++++------- lib/compress/zstd_compress.c | 5 +++- lib/decompress/zbuff_decompress.c | 46 ++++++++++++++++++++++++------- lib/decompress/zstd_decompress.c | 5 +++- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 3caa39caf..4d7f12a1f 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -38,7 +38,8 @@ extern "C" { /* ************************************* * Dependencies ***************************************/ -#include /* size_t */ +#include /* size_t */ +#include "zstd_static.h" /* ZSTD_customMem */ /* *************************************************************** @@ -60,6 +61,7 @@ extern "C" { ***************************************/ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); @@ -112,6 +114,7 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 260aca087..cabf447a4 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -95,14 +95,39 @@ struct ZBUFF_CCtx_s { size_t outBuffContentSize; size_t outBuffFlushedSize; ZBUFF_cStage stage; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; }; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ ZBUFF_CCtx* ZBUFF_createCCtx(void) { - ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx)); + ZSTD_customMem customMem = { NULL, NULL }; + return ZBUFF_createCCtx_advanced(customMem); +} + +ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_CCtx* zbc; + + if (!customMem.customAlloc && !customMem.customFree) + { + zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx)); + if (zbc==NULL) return NULL; + zbc->customAlloc = malloc; + zbc->customFree = free; + zbc->zc = ZSTD_createCCtx(); + return zbc; + } + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; - memset(zbc, 0, sizeof(*zbc)); - zbc->zc = ZSTD_createCCtx(); + memset(zbc, 0, sizeof(ZBUFF_CCtx)); + zbc->customAlloc = customMem.customAlloc; + zbc->customFree = customMem.customFree; + zbc->zc = ZSTD_createCCtx_advanced(customMem); return zbc; } @@ -110,9 +135,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) { if (zbc==NULL) return 0; /* support free on NULL */ ZSTD_freeCCtx(zbc->zc); - free(zbc->inBuff); - free(zbc->outBuff); - free(zbc); + zbc->customFree(zbc->inBuff); + zbc->customFree(zbc->outBuff); + zbc->customFree(zbc); return 0; } @@ -127,16 +152,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zbc->inBuffSize < neededInBuffSize) { zbc->inBuffSize = neededInBuffSize; - free(zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)malloc(neededInBuffSize); + zbc->customFree(zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); } if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; - free(zbc->outBuff); /* should not be necessary */ - zbc->outBuff = (char*)malloc(zbc->outBuffSize); + zbc->customFree(zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize); if (zbc->outBuff == NULL) return ERROR(memory_allocation); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 966d9acbd..bca725f54 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -128,7 +128,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ZSTD_CCtx* ctx; - if (!customMem.customAlloc || !customMem.customFree) + if (!customMem.customAlloc && !customMem.customFree) { ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); if (!ctx) return NULL; @@ -138,6 +138,9 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) return ctx; } + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); if (!ctx) return NULL; diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 858b442f7..d4ca550a7 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -82,15 +82,41 @@ struct ZBUFF_DCtx_s { size_t blockSize; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ ZBUFF_DCtx* ZBUFF_createDCtx(void) { - ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); + ZSTD_customMem customMem = { NULL, NULL }; + return ZBUFF_createDCtx_advanced(customMem); +} + +ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_DCtx* zbd; + + if (!customMem.customAlloc && !customMem.customFree) + { + zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx)); + if (zbd==NULL) return NULL; + zbd->customAlloc = malloc; + zbd->customFree = free; + zbd->zd = ZSTD_createDCtx(); + zbd->stage = ZBUFFds_init; + return zbd; + } + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx)); if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(*zbd)); - zbd->zd = ZSTD_createDCtx(); + memset(zbd, 0, sizeof(ZBUFF_DCtx)); + zbd->customAlloc = customMem.customAlloc; + zbd->customFree = customMem.customFree; + zbd->zd = ZSTD_createDCtx_advanced(customMem); zbd->stage = ZBUFFds_init; return zbd; } @@ -99,9 +125,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) { if (zbd==NULL) return 0; /* support free on null */ ZSTD_freeDCtx(zbd->zd); - free(zbd->inBuff); - free(zbd->outBuff); - free(zbd); + zbd->customFree(zbd->inBuff); + zbd->customFree(zbd->outBuff); + zbd->customFree(zbd); return 0; } @@ -170,16 +196,16 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; if (zbd->inBuffSize < blockSize) { - free(zbd->inBuff); + zbd->customFree(zbd->inBuff); zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)malloc(blockSize); + zbd->inBuff = (char*)zbd->customAlloc(blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; if (zbd->outBuffSize < neededOutSize) { - free(zbd->outBuff); + zbd->customFree(zbd->outBuff); zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)malloc(neededOutSize); + zbd->outBuff = (char*)zbd->customAlloc(neededOutSize); if (zbd->outBuff == NULL) return ERROR(memory_allocation); } } } zbd->stage = ZBUFFds_read; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index a3eee709e..b670c548c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -153,7 +153,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; - if (!customMem.customAlloc || !customMem.customFree) + if (!customMem.customAlloc && !customMem.customFree) { dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; @@ -164,6 +164,9 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) return dctx; } + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; dctx->customAlloc = customMem.customAlloc; From 71dd67cf57182c31686d1262b785512d2149c8ae Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 10:57:14 +0200 Subject: [PATCH 17/26] fullbench: tests for ZBUFF_createCCtx_advanced and ZBUFF_createDCtx_advanced --- lib/common/mem.h | 7 +++-- lib/compress/.debug/zstd_stats.h | 7 ++++- lib/compress/zstd_compress.c | 11 ++++--- programs/fullbench.c | 51 +++++++++++++++++++++++++++++++- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 5abd83d1a..7f5a2ef7c 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -42,8 +42,11 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ -#include /* size_t, ptrdiff_t */ -#include /* memcpy */ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +#endif /*-**************************************** diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index 51406a930..428514bfe 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -55,8 +55,11 @@ struct ZSTD_stats_s { /*-************************************* -* Advanced functions +* Stats functions ***************************************/ +MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc() { return malloc(sizeof(ZSTD_stats_t)); } +MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { free(stats); } + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); @@ -155,6 +158,8 @@ MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, co #else struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc(void) { return NULL; } + MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { (void)stats; } MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bca725f54..93efa99c9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2052,8 +2052,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = (ZSTD_stats_t*) zc->customAlloc(sizeof(ZSTD_stats_t)); - if (!stats) return ERROR(memory_allocation); + ZSTD_stats_t* stats = ZSTD_statsAlloc(); zc->seqStore.stats = stats; ZSTD_statsInit(stats); @@ -2061,7 +2060,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { zc->customFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { ZSTD_statsFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2072,11 +2071,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } + if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } + if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2092,7 +2091,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - zc->customFree(stats); + ZSTD_statsFree(stats); return op-ostart; } diff --git a/programs/fullbench.c b/programs/fullbench.c index 6f5d6a782..fe1abf1a2 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -213,6 +213,19 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, /*_******************************************************* * Bench functions *********************************************************/ +void* BMK_allocFunction(size_t size) +{ + void* address = malloc(size); + /* printf("alloc %p, %d \n", address, (int)size); */ + return address; +} + +void BMK_freeFunction(void* address) +{ +/* printf("free %p \n", address); */ + free(address); +} + static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { BYTE* dstBuff; @@ -220,6 +233,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) BYTE* buff2; const char* benchName; size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); + ZSTD_customMem customMem = { BMK_allocFunction, BMK_freeFunction }; double bestTime = 100000000.; /* Selection */ @@ -247,8 +261,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue"; break; case 42: + benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_createCCtx_advanced/ZBUFF_compressContinue"; + break; + case 43: benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue"; break; + case 44: + benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_createDCtx_advanced/ZBUFF_decompressContinue"; + break; default : return 0; } @@ -323,9 +343,16 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx(); break; case 42 : + if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx_advanced(customMem); + break; + case 43 : if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx(); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); break; + case 44 : + if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx_advanced(customMem); + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + break; /* test functions */ /* by convention, test functions can be added > 100 */ @@ -355,7 +382,29 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds; if (averageTime < bestTime) bestTime = averageTime; DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult); - }} + } + + /* free allocated structures */ + switch(benchNb) + { + case 11 : + if (g_zcc) { ZSTD_freeCCtx(g_zcc); g_zcc=NULL; } + break; + case 12 : + case 31: + case 32: + if (g_zdc) { ZSTD_freeDCtx(g_zdc); g_zdc=NULL; } + break; + case 41 : + case 42 : + if (g_zbcc) { ZBUFF_freeCCtx(g_zbcc); g_zbcc=NULL; } + break; + case 43 : + case 44 : + if (g_zbdc) { ZBUFF_freeDCtx(g_zbdc); g_zbdc=NULL; } + break; + default : ; + } } DISPLAY("%2u\n", benchNb); _cleanOut: From a1653fbfd05653c2c3f18741da925a5dade2f5b9 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 15:35:48 +0200 Subject: [PATCH 18/26] zbufftest.c: tests for ZBUFF_createCCtx_advanced/ZBUFF_createDCtx_advanced --- programs/zbufftest.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f8d527ef2..d7f7ac00d 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -131,7 +131,20 @@ static unsigned FUZ_highbit32(U32 v32) } */ -static int basicUnitTests(U32 seed, double compressibility) +void* ZBUFF_allocFunction(size_t size) +{ + void* address = malloc(size); + /* DISPLAYLEVEL(4, "alloc %p, %d \n", address, (int)size); */ + return address; +} + +void ZBUFF_freeFunction(void* address) +{ + /* if (address) DISPLAYLEVEL(4, "free %p \n", address); */ + free(address); +} + +static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) { int testResult = 0; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; @@ -143,8 +156,8 @@ static int basicUnitTests(U32 seed, double compressibility) U32 randState = seed; size_t result, cSize, readSize, genSize; U32 testNb=0; - ZBUFF_CCtx* zc = ZBUFF_createCCtx(); - ZBUFF_DCtx* zd = ZBUFF_createDCtx(); + ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); + ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); /* Create compressible test buffer */ if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { @@ -489,6 +502,8 @@ int main(int argc, const char** argv) int result=0; U32 mainPause = 0; const char* programName = argv[0]; + ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction }; + ZSTD_customMem customNULL = { NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Tue, 24 May 2016 15:36:37 +0200 Subject: [PATCH 19/26] zstd_stats.h included only in debug mode --- lib/common/zstd_internal.h | 12 +++++++- lib/compress/.debug/zstd_stats.h | 12 -------- lib/compress/zstd_compress.c | 15 ++++------ programs/fullbench.c | 51 +------------------------------- 4 files changed, 18 insertions(+), 72 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8d9883d0f..2eea5feee 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -201,6 +201,16 @@ typedef struct { U32 rep[ZSTD_REP_INIT]; } ZSTD_optimal_t; +#if ZSTD_OPT_DEBUG == 3 + #include ".debug/zstd_stats.h" +#else + struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } +#endif // #if ZSTD_OPT_DEBUG == 3 + typedef struct { void* buffer; U32* offsetStart; @@ -237,7 +247,7 @@ typedef struct { U32 cachedPrice; U32 cachedLitLength; const BYTE* cachedLiterals; - ZSTD_stats_t* stats; + ZSTD_stats_t stats; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index 428514bfe..5cefa3115 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -38,8 +38,6 @@ extern "C" { #endif -#if ZSTD_OPT_DEBUG == 3 - /*-************************************* * Types ***************************************/ @@ -156,16 +154,6 @@ MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, co stats->totalLitSum += litLength; } -#else - struct ZSTD_stats_s { U32 unused; }; - MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc(void) { return NULL; } - MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif // #if ZSTD_OPT_DEBUG == 3 - #if defined (__cplusplus) } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 93efa99c9..01d276fe3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -57,7 +57,6 @@ #include "fse_static.h" #include "huf_static.h" #include "zstd_internal.h" -#include ".debug/zstd_stats.h" /*-************************************* @@ -888,7 +887,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - ZSTD_statsUpdatePrices(seqStorePtr->stats, litLength, literals, offsetCode, matchCode); + ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode); /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); @@ -1780,7 +1779,7 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; - ZSTD_statsUpdatePrices(seqStorePtr->stats, lastLLSize, anchor, 0, 0); + ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -2052,15 +2051,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = ZSTD_statsAlloc(); - zc->seqStore.stats = stats; + ZSTD_stats_t* stats = &zc->seqStore.stats; ZSTD_statsInit(stats); while (remaining) { size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { ZSTD_statsFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2071,11 +2069,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } + if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } + if (ZSTD_isError(cSize)) return cSize; } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2091,7 +2089,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - ZSTD_statsFree(stats); return op-ostart; } diff --git a/programs/fullbench.c b/programs/fullbench.c index fe1abf1a2..6f5d6a782 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -213,19 +213,6 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, /*_******************************************************* * Bench functions *********************************************************/ -void* BMK_allocFunction(size_t size) -{ - void* address = malloc(size); - /* printf("alloc %p, %d \n", address, (int)size); */ - return address; -} - -void BMK_freeFunction(void* address) -{ -/* printf("free %p \n", address); */ - free(address); -} - static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { BYTE* dstBuff; @@ -233,7 +220,6 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) BYTE* buff2; const char* benchName; size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); - ZSTD_customMem customMem = { BMK_allocFunction, BMK_freeFunction }; double bestTime = 100000000.; /* Selection */ @@ -261,14 +247,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue"; break; case 42: - benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_createCCtx_advanced/ZBUFF_compressContinue"; - break; - case 43: benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue"; break; - case 44: - benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_createDCtx_advanced/ZBUFF_decompressContinue"; - break; default : return 0; } @@ -343,16 +323,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx(); break; case 42 : - if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx_advanced(customMem); - break; - case 43 : if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx(); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); break; - case 44 : - if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx_advanced(customMem); - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); - break; /* test functions */ /* by convention, test functions can be added > 100 */ @@ -382,29 +355,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds; if (averageTime < bestTime) bestTime = averageTime; DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult); - } - - /* free allocated structures */ - switch(benchNb) - { - case 11 : - if (g_zcc) { ZSTD_freeCCtx(g_zcc); g_zcc=NULL; } - break; - case 12 : - case 31: - case 32: - if (g_zdc) { ZSTD_freeDCtx(g_zdc); g_zdc=NULL; } - break; - case 41 : - case 42 : - if (g_zbcc) { ZBUFF_freeCCtx(g_zbcc); g_zbcc=NULL; } - break; - case 43 : - case 44 : - if (g_zbdc) { ZBUFF_freeDCtx(g_zbdc); g_zbdc=NULL; } - break; - default : ; - } } + }} DISPLAY("%2u\n", benchNb); _cleanOut: From 33b8a893edd3dadc960e5060044cc35a141324fd Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 17:44:14 +0200 Subject: [PATCH 20/26] CMake project moved to projects --- Makefile | 2 +- projects/README.md | 1 + {contrib => projects}/cmake/CMakeLists.txt | 0 .../cmake/CMakeModules/AddExtraCompilationFlags.cmake | 0 projects/cmake/build/.keep | 0 {contrib => projects}/cmake/cmake_uninstall.cmake.in | 0 {contrib => projects}/cmake/lib/CMakeLists.txt | 0 {contrib => projects}/cmake/programs/CMakeLists.txt | 0 8 files changed, 2 insertions(+), 1 deletion(-) rename {contrib => projects}/cmake/CMakeLists.txt (100%) rename {contrib => projects}/cmake/CMakeModules/AddExtraCompilationFlags.cmake (100%) create mode 100644 projects/cmake/build/.keep rename {contrib => projects}/cmake/cmake_uninstall.cmake.in (100%) rename {contrib => projects}/cmake/lib/CMakeLists.txt (100%) rename {contrib => projects}/cmake/programs/CMakeLists.txt (100%) diff --git a/Makefile b/Makefile index 95ecc007c..0894f75d7 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ travis-install: $(MAKE) install PREFIX=~/install_test_dir cmaketest: - cd contrib/cmake ; cmake . ; $(MAKE) + cd projects/cmake/build ; cmake .. ; $(MAKE) clangtest: clean clang -v diff --git a/projects/README.md b/projects/README.md index d69c71b14..b6831ce29 100644 --- a/projects/README.md +++ b/projects/README.md @@ -4,5 +4,6 @@ projects for various integrated development environments (IDE) #### Included projects The following projects are included with the zstd distribution: +- cmake - CMake project contributed by Artyom Dymchenko - VS2008 - Visual Studio 2008 project - VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) diff --git a/contrib/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt similarity index 100% rename from contrib/cmake/CMakeLists.txt rename to projects/cmake/CMakeLists.txt diff --git a/contrib/cmake/CMakeModules/AddExtraCompilationFlags.cmake b/projects/cmake/CMakeModules/AddExtraCompilationFlags.cmake similarity index 100% rename from contrib/cmake/CMakeModules/AddExtraCompilationFlags.cmake rename to projects/cmake/CMakeModules/AddExtraCompilationFlags.cmake diff --git a/projects/cmake/build/.keep b/projects/cmake/build/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/cmake/cmake_uninstall.cmake.in b/projects/cmake/cmake_uninstall.cmake.in similarity index 100% rename from contrib/cmake/cmake_uninstall.cmake.in rename to projects/cmake/cmake_uninstall.cmake.in diff --git a/contrib/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt similarity index 100% rename from contrib/cmake/lib/CMakeLists.txt rename to projects/cmake/lib/CMakeLists.txt diff --git a/contrib/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt similarity index 100% rename from contrib/cmake/programs/CMakeLists.txt rename to projects/cmake/programs/CMakeLists.txt From 460a4ee6d4d6659d46c052b7477c085e7646f369 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 18:22:38 +0200 Subject: [PATCH 21/26] playTests updated with Pass-Through mode --- programs/tests/playTests.bat | 10 +++++---- programs/tests/playTestsMSYS.sh | 38 +++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index edbbcb340..5f7ebd685 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -31,9 +31,11 @@ REM %ZSTD% -d - < tmp.zst > NUL %ZSTD% -q -f tmp %ZSTD% -q --force tmp %ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) -cp tmp tmp2.zst -%ZSTD% -df tmp2.zst && (echo should have failed : wrong format && exit /b 1) -rm tmp2.zst + +echo. && echo **** Pass-Through mode **** +echo "Hello world !" | %ZSTD% -df +echo "Hello world !" | %ZSTD% -dcf + echo. && echo **** frame concatenation **** @@ -48,7 +50,7 @@ cat result.tmp fc /b helloworld.tmp result.tmp rm *.tmp *.zstd -echo frame concatenation test completed +echo frame concatenation tests completed REM echo. && echo **** flush write error test **** diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh index 4289977a0..3d5a0e842 100755 --- a/programs/tests/playTestsMSYS.sh +++ b/programs/tests/playTestsMSYS.sh @@ -47,9 +47,12 @@ $ZSTD -q tmp && die "overwrite check failed!" $ZSTD -q -f tmp $ZSTD -q --force tmp $ZSTD -df tmp && die "should have refused : wrong extension" -cp tmp tmp2.zst -$ZSTD -df tmp2.zst && die "should have failed : wrong format" -rm tmp2.zst + + +echo -e "\n**** Pass-Through mode **** " +echo "Hello world !" | $ZSTD -df +echo "Hello world !" | $ZSTD -dcf + echo -e "\n**** frame concatenation **** " @@ -64,7 +67,34 @@ cat result.tmp fc helloworld.tmp result.tmp rm ./*.tmp ./*.zstd -echo frame concatenation test completed +echo frame concatenation tests completed + + +echo -e "\n**** test sparse file support **** " + +./datagen -g5M -P100 > tmpSparse +$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen +fc tmpSparse tmpSparseRegen +$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse +fc tmpSparse tmpOutSparse +$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse +fc tmpSparse tmpOutNoSparse +ls -ls tmpSparse* +./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) +./datagen -s1 -g1200007 -P100 | fc - tmpSparseOdd +ls -ls tmpSparseOdd +echo -e "\n Sparse Compatibility with Console :" +echo "Hello World 1 !" | $ZSTD | $ZSTD -d -c +echo "Hello World 2 !" | $ZSTD | $ZSTD -d | cat +echo -e "\n Sparse Compatibility with Append :" +./datagen -P100 -g1M > tmpSparse1M +cat tmpSparse1M tmpSparse1M > tmpSparse2M +$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed +$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated +$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated +ls -ls tmpSparse* +fc tmpSparse2M tmpSparseRegenerated + echo -e "\n**** dictionary tests **** " From 5d589562fe93d517abf71aa8fc71c5cbb545d0cf Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 10:50:28 +0200 Subject: [PATCH 22/26] playTests.sh joined with playTestsMSYS.sh --- appveyor.yml | 30 ++--- programs/Makefile | 8 +- programs/tests/playTests.sh | 94 +++++++++------- programs/tests/playTestsMSYS.sh | 190 -------------------------------- 4 files changed, 71 insertions(+), 251 deletions(-) delete mode 100755 programs/tests/playTestsMSYS.sh diff --git a/appveyor.yml b/appveyor.yml index c6f735ea3..4721fa951 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,24 +1,24 @@ version: 1.0.{build} environment: matrix: - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - COMPILER: "gcc" - MAKE_PARAMS: "test" - PLATFORM: "mingw32" - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw64" + - COMPILER: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% diff --git a/programs/Makefile b/programs/Makefile index 11709300a..c4e19229a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -155,9 +155,6 @@ clean: #------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) HOST_OS = POSIX -zstd-playTests: datagen - ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) - install: zstd @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @@ -199,8 +196,6 @@ endif ifneq (,$(filter MSYS%,$(shell uname))) HOST_OS = MSYS -zstd-playTests: datagen - ZSTD=$(ZSTD) ./tests/playTestsMSYS.sh $(ZSTDRTTEST) endif @@ -208,6 +203,9 @@ endif #make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets #------------------------------------------------------------------------ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) + test: test-zstd test-fullbench test-fuzzer test-zbuff test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index 58455c0f9..dbeb916d0 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -1,7 +1,7 @@ #!/bin/sh -e die() { - echo "$@" 1>&2 + $ECHO "$@" 1>&2 exit 1 } @@ -14,34 +14,44 @@ roundTripTest() { fi rm -f tmp1 tmp2 - echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" ./datagen $1 $p | md5sum > tmp1 ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 diff -q tmp1 tmp2 } +isWindows=false +ECHO="echo" +if [[ "$OS" == "Windows"* ]]; then + isWindows=true + ECHO="echo -e" +fi + +$ECHO "\nStarting playTests.sh isWindows=$isWindows" + [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" - -echo "\n**** simple tests **** " +$ECHO "\n**** simple tests **** " ./datagen > tmp $ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) -echo "test : too large compression level (must fail)" +$ECHO "test : too large compression level (must fail)" $ZSTD -99 tmp && die "too large compression level undetected" -echo "test : compress to stdout" +$ECHO "test : compress to stdout" $ZSTD tmp -c > tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format -echo "test : null-length file roundtrip" -echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout -echo "test : decompress file with wrong suffix (must fail)" +$ECHO "test : null-length file roundtrip" +$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout +$ECHO "test : decompress file with wrong suffix (must fail)" $ZSTD -d tmpCompressed && die "wrong suffix error not detected!" $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout $ZSTD --decompress tmpCompressed -c > tmpResult $ZSTD --decompress tmpCompressed --stdout > tmpResult -$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout -$ZSTD -d - < tmp.zst > /dev/null +if [ "$isWindows" = false ] ; then + $ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout + $ZSTD -d - < tmp.zst > /dev/null +fi $ZSTD -dc < tmp.zst > /dev/null $ZSTD -dc - < tmp.zst > /dev/null $ZSTD -q tmp && die "overwrite check failed!" @@ -50,15 +60,15 @@ $ZSTD -q --force tmp $ZSTD -df tmp && die "should have refused : wrong extension" -echo "\n**** Pass-Through mode **** " -echo "Hello world !" | $ZSTD -df -echo "Hello world !" | $ZSTD -dcf +$ECHO "\n**** Pass-Through mode **** " +$ECHO "Hello world !" | $ZSTD -df +$ECHO "Hello world !" | $ZSTD -dcf -echo "\n**** frame concatenation **** " +$ECHO "\n**** frame concatenation **** " -echo "hello " > hello.tmp -echo "world!" > world.tmp +$ECHO "hello " > hello.tmp +$ECHO "world!" > world.tmp cat hello.tmp world.tmp > helloworld.tmp $ZSTD -c hello.tmp > hello.zstd $ZSTD -c world.tmp > world.zstd @@ -67,18 +77,20 @@ $ZSTD -dc helloworld.zstd > result.tmp cat result.tmp sdiff helloworld.tmp result.tmp rm ./*.tmp ./*.zstd -echo "frame concatenation tests completed" +$ECHO "frame concatenation tests completed" -echo "\n**** flush write error test **** " +if [ "$isWindows" = false ] ; then +$ECHO "\n**** flush write error test **** " -echo "echo foo | $ZSTD > /dev/full" -echo foo | $ZSTD > /dev/full && die "write error not detected!" -echo "echo foo | $ZSTD | $ZSTD -d > /dev/full" -echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" +$ECHO "$ECHO foo | $ZSTD > /dev/full" +$ECHO foo | $ZSTD > /dev/full && die "write error not detected!" +$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full" +$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" +fi -echo "\n**** test sparse file support **** " +$ECHO "\n**** test sparse file support **** " ./datagen -g5M -P100 > tmpSparse $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen @@ -91,10 +103,10 @@ ls -ls tmpSparse* ./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) ./datagen -s1 -g1200007 -P100 | diff -s - tmpSparseOdd ls -ls tmpSparseOdd -echo "\n Sparse Compatibility with Console :" -echo "Hello World 1 !" | $ZSTD | $ZSTD -d -c -echo "Hello World 2 !" | $ZSTD | $ZSTD -d | cat -echo "\n Sparse Compatibility with Append :" +$ECHO "\n Sparse Compatibility with Console :" +$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c +$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat +$ECHO "\n Sparse Compatibility with Append :" ./datagen -P100 -g1M > tmpSparse1M cat tmpSparse1M tmpSparse1M > tmpSparse2M $ZSTD -v -f tmpSparse1M -o tmpSparseCompressed @@ -105,7 +117,7 @@ diff tmpSparse2M tmpSparseRegenerated # rm tmpSparse* -echo "\n**** dictionary tests **** " +$ECHO "\n**** dictionary tests **** " ./datagen > tmpDict ./datagen -g1M | md5sum > tmp1 @@ -117,41 +129,41 @@ $ZSTD -d tmp -D tmpDict -of result diff xxhash.c result -echo "\n**** multiple files tests **** " +$ECHO "\n**** multiple files tests **** " ./datagen -s1 > tmp1 2> /dev/null ./datagen -s2 -g100K > tmp2 2> /dev/null ./datagen -s3 -g1M > tmp3 2> /dev/null $ZSTD -f tmp* -echo "compress tmp* : " +$ECHO "compress tmp* : " ls -ls tmp* rm tmp1 tmp2 tmp3 -echo "decompress tmp* : " +$ECHO "decompress tmp* : " $ZSTD -df *.zst ls -ls tmp* -echo "compress tmp* into stdout > tmpall : " +$ECHO "compress tmp* into stdout > tmpall : " $ZSTD -c tmp1 tmp2 tmp3 > tmpall ls -ls tmp* -echo "decompress tmpall* into stdout > tmpdec : " +$ECHO "decompress tmpall* into stdout > tmpdec : " cp tmpall tmpall2 $ZSTD -dc tmpall* > tmpdec ls -ls tmp* -echo "compress multiple files including a missing one (notHere) : " +$ECHO "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" -echo "\n**** integrity tests **** " +$ECHO "\n**** integrity tests **** " -echo "test one file (tmp1.zst) " +$ECHO "test one file (tmp1.zst) " $ZSTD -t tmp1.zst $ZSTD --test tmp1.zst -echo "test multiple files (*.zst) " +$ECHO "test multiple files (*.zst) " $ZSTD -t *.zst -echo "test good and bad files (*) " +$ECHO "test good and bad files (*) " $ZSTD -t * && die "bad files not detected !" -echo "\n**** zstd round-trip tests **** " +$ECHO "\n**** zstd round-trip tests **** " roundTripTest roundTripTest -g15K # TableID==3 @@ -165,7 +177,7 @@ roundTripTest -g512K 19 # btopt rm tmp* if [ "$1" != "--test-large-data" ]; then - echo "Skipping large data tests" + $ECHO "Skipping large data tests" exit 0 fi diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh deleted file mode 100755 index 3d5a0e842..000000000 --- a/programs/tests/playTestsMSYS.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/sh -e - -die() { - echo "$@" 1>&2 - exit 1 -} - -roundTripTest() { - if [ -n "$3" ]; then - local c="$3" - local p="$2" - else - local c="$2" - fi - - rm -f tmp1 tmp2 - echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" - ./datagen $1 $p | md5sum > tmp1 - ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 - fc tmp1 tmp2 -} - -[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" - - -echo -e "\n**** simple tests **** " -./datagen > tmp -$ZSTD -f tmp # trivial compression case, creates tmp.zst -$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) -echo "test : too large compression level (must fail)" -$ZSTD -99 tmp && die "too large compression level undetected" -echo "test : compress to stdout" -$ZSTD tmp -c > tmpCompressed -$ZSTD tmp --stdout > tmpCompressed # long command format -echo "test : null-length file roundtrip" -echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout -echo "test : decompress file with wrong suffix (must fail)" -$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" -$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout -$ZSTD --decompress tmpCompressed -c > tmpResult -$ZSTD --decompress tmpCompressed --stdout > tmpResult -#$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout -#$ZSTD -d - < tmp.zst > /dev/null -$ZSTD -dc < tmp.zst > /dev/null -$ZSTD -dc - < tmp.zst > /dev/null -$ZSTD -q tmp && die "overwrite check failed!" -$ZSTD -q -f tmp -$ZSTD -q --force tmp -$ZSTD -df tmp && die "should have refused : wrong extension" - - -echo -e "\n**** Pass-Through mode **** " -echo "Hello world !" | $ZSTD -df -echo "Hello world !" | $ZSTD -dcf - - -echo -e "\n**** frame concatenation **** " - -echo "hello " > hello.tmp -echo "world!" > world.tmp -cat hello.tmp world.tmp > helloworld.tmp -$ZSTD -c hello.tmp > hello.zstd -$ZSTD -c world.tmp > world.zstd -cat hello.zstd world.zstd > helloworld.zstd -$ZSTD -dc helloworld.zstd > result.tmp -cat result.tmp -fc helloworld.tmp result.tmp -rm ./*.tmp ./*.zstd - -echo frame concatenation tests completed - - -echo -e "\n**** test sparse file support **** " - -./datagen -g5M -P100 > tmpSparse -$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen -fc tmpSparse tmpSparseRegen -$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse -fc tmpSparse tmpOutSparse -$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse -fc tmpSparse tmpOutNoSparse -ls -ls tmpSparse* -./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) -./datagen -s1 -g1200007 -P100 | fc - tmpSparseOdd -ls -ls tmpSparseOdd -echo -e "\n Sparse Compatibility with Console :" -echo "Hello World 1 !" | $ZSTD | $ZSTD -d -c -echo "Hello World 2 !" | $ZSTD | $ZSTD -d | cat -echo -e "\n Sparse Compatibility with Append :" -./datagen -P100 -g1M > tmpSparse1M -cat tmpSparse1M tmpSparse1M > tmpSparse2M -$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed -$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated -$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated -ls -ls tmpSparse* -fc tmpSparse2M tmpSparseRegenerated - - - -echo -e "\n**** dictionary tests **** " - -./datagen > tmpDict -./datagen -g1M | md5sum > tmp1 -./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 -fc tmp1 tmp2 -$ZSTD --train *.c *.h -o tmpDict -$ZSTD xxhash.c -D tmpDict -of tmp -$ZSTD -d tmp -D tmpDict -of result -fc xxhash.c result - - -echo -e "\n**** multiple files tests **** " - -./datagen -s1 > tmp1 2> /dev/null -./datagen -s2 -g100K > tmp2 2> /dev/null -./datagen -s3 -g1M > tmp3 2> /dev/null -$ZSTD -f tmp* -echo "compress tmp* : " -ls -ls tmp* -rm tmp1 tmp2 tmp3 -echo "decompress tmp* : " -$ZSTD -df *.zst -ls -ls tmp* -echo "compress tmp* into stdout > tmpall : " -$ZSTD -c tmp1 tmp2 tmp3 > tmpall -ls -ls tmp* -echo "decompress tmpall* into stdout > tmpdec : " -cp tmpall tmpall2 -$ZSTD -dc tmpall* > tmpdec -ls -ls tmp* -echo "compress multiple files including a missing one (notHere) : " -$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" - -echo -e "\n**** integrity tests **** " -echo "test one file (tmp1.zst) " -$ZSTD -t tmp1.zst -$ZSTD --test tmp1.zst -echo "test multiple files (*.zst) " -$ZSTD -t *.zst -echo "test good and bad files (*) " -$ZSTD -t * && die "bad files not detected !" - -echo -e "\n**** zstd round-trip tests **** " - -roundTripTest -roundTripTest -g15K # TableID==3 -roundTripTest -g127K # TableID==2 -roundTripTest -g255K # TableID==1 -roundTripTest -g513K # TableID==0 -roundTripTest -g512K 6 # greedy, hash chain -roundTripTest -g512K 16 # btlazy2 -roundTripTest -g512K 19 # btopt - -rm tmp* - -if [ "$1" != "--test-large-data" ]; then - echo "Skipping large data tests" - exit 0 -fi - -roundTripTest -g270000000 1 -roundTripTest -g270000000 2 -roundTripTest -g270000000 3 - -roundTripTest -g140000000 -P60 4 -roundTripTest -g140000000 -P60 5 -roundTripTest -g140000000 -P60 6 - -roundTripTest -g70000000 -P70 7 -roundTripTest -g70000000 -P70 8 -roundTripTest -g70000000 -P70 9 - -roundTripTest -g35000000 -P75 10 -roundTripTest -g35000000 -P75 11 -roundTripTest -g35000000 -P75 12 - -roundTripTest -g18000000 -P80 13 -roundTripTest -g18000000 -P80 14 -roundTripTest -g18000000 -P80 15 -roundTripTest -g18000000 -P80 16 -roundTripTest -g18000000 -P80 17 - -roundTripTest -g50000000 -P94 18 -roundTripTest -g50000000 -P94 19 - -roundTripTest -g99000000 -P99 20 -roundTripTest -g6000000000 -P99 1 - -rm tmp* - From e75909ef93c87c3b716f3fb85aaf81f5ef0e6310 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 11:31:16 +0200 Subject: [PATCH 23/26] ZBUFF_createCCtx_advanced moved to zbuff_static.h --- lib/common/zbuff.h | 3 --- lib/common/zbuff_static.h | 17 ++++++++++++++++- programs/fileio.c | 1 + programs/zbufftest.c | 15 ++++++++------- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 4d7f12a1f..89207bd24 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -39,7 +39,6 @@ extern "C" { * Dependencies ***************************************/ #include /* size_t */ -#include "zstd_static.h" /* ZSTD_customMem */ /* *************************************************************** @@ -61,7 +60,6 @@ extern "C" { ***************************************/ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); -ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); @@ -114,7 +112,6 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); -ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); diff --git a/lib/common/zbuff_static.h b/lib/common/zbuff_static.h index 4c52eef22..6a4287e4d 100644 --- a/lib/common/zbuff_static.h +++ b/lib/common/zbuff_static.h @@ -46,7 +46,22 @@ extern "C" { ***************************************/ #include "zstd_static.h" /* ZSTD_parameters */ #include "zbuff.h" -#include "zstd_internal.h" /* MIN */ + +#ifndef MIN + #define MIN(a,b) ((a)<(b) ? (a) : (b)) +#endif + + +/*-************************************* +* Advanced functions +***************************************/ +/*! ZBUFF_createCCtx_advanced() : + * Create a ZBUFF compression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZBUFF_createDCtx_advanced() : + * Create a ZBUFF decompression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); /* ************************************* diff --git a/programs/fileio.c b/programs/fileio.c index 51707e8b7..eba90d011 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -57,6 +57,7 @@ #include "mem.h" #include "fileio.h" #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ +#include "zstd_internal.h" /* MIN, KB, MB */ #include "zbuff_static.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index e9e54586c..fce0ab275 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -35,15 +35,16 @@ /*-************************************ * Includes **************************************/ -#include /* free */ -#include /* fgets, sscanf */ -#include /* timeb */ -#include /* strcmp */ +#include /* free */ +#include /* fgets, sscanf */ +#include /* timeb */ +#include /* strcmp */ #include "mem.h" #include "zbuff.h" -#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ -#include "datagen.h" /* RDG_genBuffer */ -#include "xxhash.h" /* XXH64 */ +#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ +#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ +#include "datagen.h" /* RDG_genBuffer */ +#include "xxhash.h" /* XXH64 */ /*-************************************ From 4f1acdd15890cea8cfba011ef470a905a3963d60 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 12:41:41 +0200 Subject: [PATCH 24/26] usage of fileno() only on POSIX and Windows --- programs/zbufftest.c | 2 -- programs/zstdcli.c | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index fce0ab275..c780b8f4e 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -262,8 +262,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) return FUZ_rLogLength(seed, logLength); } -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 13ff7955d..215157873 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -52,9 +52,12 @@ # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else - extern int fileno(FILE *stream); /* triggers fileno() within on POSIX */ +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif #endif From 957823f56f245e6fb2c562fa8f8160aec6be730d Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 15:30:55 +0200 Subject: [PATCH 25/26] zstdcli: -r (operate recursively on directories) works with dictBuilder and compression --- lib/common/zbuff_static.h | 6 ++---- programs/bench.c | 22 ++-------------------- programs/bench.h | 2 +- programs/util.h | 3 +-- programs/zbufftest.c | 2 ++ programs/zstdcli.c | 39 +++++++++++++++++++++++++++++++-------- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/lib/common/zbuff_static.h b/lib/common/zbuff_static.h index 6a4287e4d..c50a3e166 100644 --- a/lib/common/zbuff_static.h +++ b/lib/common/zbuff_static.h @@ -47,9 +47,7 @@ extern "C" { #include "zstd_static.h" /* ZSTD_parameters */ #include "zbuff.h" -#ifndef MIN - #define MIN(a,b) ((a)<(b) ? (a) : (b)) -#endif +#define ZBUFF_MIN(a,b) ((a)<(b) ? (a) : (b)) /*-************************************* @@ -73,7 +71,7 @@ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t length = MIN(dstCapacity, srcSize); + size_t length = ZBUFF_MIN(dstCapacity, srcSize); memcpy(dst, src, length); return length; } diff --git a/programs/bench.c b/programs/bench.c index a2721552e..52c64d525 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -26,7 +26,7 @@ /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_HAS_CREATEFILELIST, UTIL_sleep */ +#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ @@ -495,32 +495,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int recursive) + const char* dictFileName, int cLevel, int cLevelLast) { double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) BMK_syntheticTest(cLevel, cLevelLast, compressibility); else - { -#ifdef UTIL_HAS_CREATEFILELIST - if (recursive) { - char* buf; - const char** filenameTable; - unsigned i; - filenameTable = UTIL_createFileList(fileNamesTable, nbFiles, &buf, &nbFiles); - if (filenameTable) { - for (i=0; i= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; diff --git a/programs/zbufftest.c b/programs/zbufftest.c index c780b8f4e..fce0ab275 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -262,6 +262,8 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) return FUZ_rLogLength(seed, logLength); } +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 215157873..f367f49af 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -129,6 +129,9 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories\n"); +#endif #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); #endif @@ -147,9 +150,6 @@ static int usage_advanced(const char* programName) DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); -#ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories\n"); -#endif DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); #endif return 0; @@ -198,6 +198,11 @@ int main(int argCount, const char** argv) unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; +#ifdef UTIL_HAS_CREATEFILELIST + const char** fileNamesTable = NULL; + char* fileNamesBuf; + unsigned fileNamesNb; +#endif /* init */ (void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ @@ -296,6 +301,9 @@ int main(int argCount, const char** argv) /* dictionary name */ case 'o': nextArgumentIsOutFileName=1; argument++; break; + /* recursive */ + case 'r': recursive=1; argument++; break; + #ifndef ZSTD_NOBENCH /* Benchmark */ case 'b': bench=1; argument++; break; @@ -322,9 +330,6 @@ int main(int argCount, const char** argv) } break; - /* recursive */ - case 'r': recursive=1; argument++; break; - /* cut input into blocks (benchmark only) */ case 'B': { size_t bSize = 0; @@ -395,11 +400,24 @@ int main(int argCount, const char** argv) /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); +#ifdef UTIL_HAS_CREATEFILELIST + if (recursive) { + fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + if (fileNamesTable) { + unsigned i; + for (i=0; i Date: Wed, 25 May 2016 17:40:58 +0200 Subject: [PATCH 26/26] fixed Visual Studio warning --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f367f49af..68dd98cb0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -200,7 +200,7 @@ int main(int argCount, const char** argv) unsigned dictSelect = g_defaultSelectivityLevel; #ifdef UTIL_HAS_CREATEFILELIST const char** fileNamesTable = NULL; - char* fileNamesBuf; + char* fileNamesBuf = NULL; unsigned fileNamesNb; #endif