From 4d9e557f44348a5caaa9a7991d7e8ecfcc68b3d9 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 13 May 2016 13:04:27 +0200 Subject: [PATCH 001/208] Visual Studio: remove MSVC*.DLL dependency from Release executables --- visual/2013/fullbench/fullbench.vcxproj | 2 ++ visual/2013/fuzzer/fuzzer.vcxproj | 2 ++ visual/2013/zstdlib/zstdlib.vcxproj | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/visual/2013/fullbench/fullbench.vcxproj b/visual/2013/fullbench/fullbench.vcxproj index c24105ee3..40935bbf7 100644 --- a/visual/2013/fullbench/fullbench.vcxproj +++ b/visual/2013/fullbench/fullbench.vcxproj @@ -128,6 +128,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false true + MultiThreaded Console @@ -147,6 +148,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true false + MultiThreaded Console diff --git a/visual/2013/fuzzer/fuzzer.vcxproj b/visual/2013/fuzzer/fuzzer.vcxproj index 5fd5f5b48..bd3e849f4 100644 --- a/visual/2013/fuzzer/fuzzer.vcxproj +++ b/visual/2013/fuzzer/fuzzer.vcxproj @@ -128,6 +128,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false true + MultiThreaded Console @@ -147,6 +148,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true false + MultiThreaded Console diff --git a/visual/2013/zstdlib/zstdlib.vcxproj b/visual/2013/zstdlib/zstdlib.vcxproj index 94da3e5d3..c13ff0af0 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj +++ b/visual/2013/zstdlib/zstdlib.vcxproj @@ -176,7 +176,7 @@ true ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false - MultiThreadedDLL + MultiThreaded ProgramDatabase All @@ -199,7 +199,7 @@ ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false false - MultiThreadedDLL + MultiThreaded ProgramDatabase true true From 7c4d041d3cb94d96c6a4aa3bff88f8b99103b879 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 16 May 2016 14:48:04 +0200 Subject: [PATCH 002/208] added Visual Studio 2008 and 2010 projects --- appveyor.yml | 55 +- projects/README.md | 8 + projects/VS2008/fullbench/fullbench.vcproj | 441 +++++++++++++++ projects/VS2008/fuzzer/fuzzer.vcproj | 437 ++++++++++++++ projects/VS2008/zstd.sln | 56 ++ projects/VS2008/zstd/zstd.vcproj | 533 ++++++++++++++++++ projects/VS2008/zstdlib/zstdlib.vcproj | 487 ++++++++++++++++ .../VS2010}/fullbench/fullbench.vcxproj | 18 +- .../fullbench/fullbench.vcxproj.filters | 0 .../VS2010}/fuzzer/fuzzer.vcxproj | 18 +- .../VS2010}/fuzzer/fuzzer.vcxproj.filters | 0 {visual/2013 => projects/VS2010}/zstd.sln | 0 .../VS2010}/zstd/zstd.vcxproj | 30 +- .../VS2010}/zstd/zstd.vcxproj.filters | 0 .../VS2010}/zstdlib/zstdlib.vcxproj | 17 +- .../VS2010}/zstdlib/zstdlib.vcxproj.filters | 9 - visual/2013/zstdlib/resource.h | Bin 812 -> 0 bytes visual/2013/zstdlib/zstdlib.rc | Bin 5196 -> 0 bytes 18 files changed, 2040 insertions(+), 69 deletions(-) create mode 100644 projects/README.md create mode 100644 projects/VS2008/fullbench/fullbench.vcproj create mode 100644 projects/VS2008/fuzzer/fuzzer.vcproj create mode 100644 projects/VS2008/zstd.sln create mode 100644 projects/VS2008/zstd/zstd.vcproj create mode 100644 projects/VS2008/zstdlib/zstdlib.vcproj rename {visual/2013 => projects/VS2010}/fullbench/fullbench.vcxproj (88%) rename {visual/2013 => projects/VS2010}/fullbench/fullbench.vcxproj.filters (100%) rename {visual/2013 => projects/VS2010}/fuzzer/fuzzer.vcxproj (88%) rename {visual/2013 => projects/VS2010}/fuzzer/fuzzer.vcxproj.filters (100%) rename {visual/2013 => projects/VS2010}/zstd.sln (100%) rename {visual/2013 => projects/VS2010}/zstd/zstd.vcxproj (85%) mode change 100755 => 100644 rename {visual/2013 => projects/VS2010}/zstd/zstd.vcxproj.filters (100%) mode change 100755 => 100644 rename {visual/2013 => projects/VS2010}/zstdlib/zstdlib.vcxproj (88%) rename {visual/2013 => projects/VS2010}/zstdlib/zstdlib.vcxproj.filters (90%) delete mode 100644 visual/2013/zstdlib/resource.h delete mode 100644 visual/2013/zstdlib/zstdlib.rc diff --git a/appveyor.yml b/appveyor.yml index df8d15572..848db3836 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,21 +5,46 @@ configuration: platform: - Win32 - x64 -environment: - matrix: -# - PlatformToolset: v100 - - PlatformToolset: v110 - - PlatformToolset: v120 - - PlatformToolset: v140 + build_script: -- cmd: >- - ECHO PlatformToolset=%PlatformToolset% +- cmd: | + ECHO *** + ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% + ECHO *** + REM msbuild "projects\VS2010\zstd.sln" /m /verbosity:detailed /property:PlatformToolset=Windows7.1SDK /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + REM msbuild "projects\VS2010\zstd.sln" "/p:AdditionalLibPaths=C:\Program Files\Microsoft SDKs\Windows\v7.1\Lib\X64" /m /verbosity:detailed /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + msbuild "projects\VS2010\zstd.sln" /p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe + + ECHO *** + ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% + ECHO *** + msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe - msbuild "visual\2013\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=%PlatformToolset% /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" test_script: -- cmd: >- - ECHO %APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST% - - SET FUZZERTEST=-T2mn - - IF %CONFIGURATION%==Release (%APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST%) \ No newline at end of file +- cmd: SET FUZZERTEST=-T1mn +- cmd: | + IF %CONFIGURATION%==Release (projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST%) + IF %CONFIGURATION%==Release (projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%) diff --git a/projects/README.md b/projects/README.md new file mode 100644 index 000000000..d69c71b14 --- /dev/null +++ b/projects/README.md @@ -0,0 +1,8 @@ +projects for various integrated development environments (IDE) +================================ + +#### Included projects + +The following projects are included with the zstd distribution: +- VS2008 - Visual Studio 2008 project +- VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/projects/VS2008/fullbench/fullbench.vcproj new file mode 100644 index 000000000..6ab878cc9 --- /dev/null +++ b/projects/VS2008/fullbench/fullbench.vcproj @@ -0,0 +1,441 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2008/fuzzer/fuzzer.vcproj b/projects/VS2008/fuzzer/fuzzer.vcproj new file mode 100644 index 000000000..9e572da26 --- /dev/null +++ b/projects/VS2008/fuzzer/fuzzer.vcproj @@ -0,0 +1,437 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2008/zstd.sln b/projects/VS2008/zstd.sln new file mode 100644 index 000000000..97b88c8d0 --- /dev/null +++ b/projects/VS2008/zstd.sln @@ -0,0 +1,56 @@ + +Microsoft Visual Studio Solution File, Format Version 10.00 +# Visual Studio 2008 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcproj", "{1A2AB08E-5CE7-4C5B-BE55-458157C14051}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcproj", "{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcproj", "{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcproj", "{99DE2A79-7298-4004-A0ED-030D7A3796CA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.ActiveCfg = Debug|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.Build.0 = Debug|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.ActiveCfg = Debug|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.Build.0 = Debug|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.ActiveCfg = Release|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.Build.0 = Release|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.ActiveCfg = Release|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.Build.0 = Release|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.ActiveCfg = Debug|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.Build.0 = Debug|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.ActiveCfg = Debug|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.Build.0 = Debug|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.ActiveCfg = Release|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.Build.0 = Release|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.ActiveCfg = Release|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.Build.0 = Release|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.ActiveCfg = Debug|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.Build.0 = Debug|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.ActiveCfg = Debug|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.Build.0 = Debug|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.ActiveCfg = Release|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.Build.0 = Release|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.ActiveCfg = Release|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.Build.0 = Release|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.ActiveCfg = Debug|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.Build.0 = Debug|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.ActiveCfg = Debug|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.Build.0 = Debug|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.ActiveCfg = Release|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.Build.0 = Release|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.ActiveCfg = Release|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj new file mode 100644 index 000000000..9e4626500 --- /dev/null +++ b/projects/VS2008/zstd/zstd.vcproj @@ -0,0 +1,533 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2008/zstdlib/zstdlib.vcproj b/projects/VS2008/zstdlib/zstdlib.vcproj new file mode 100644 index 000000000..2a4779aac --- /dev/null +++ b/projects/VS2008/zstdlib/zstdlib.vcproj @@ -0,0 +1,487 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/visual/2013/fullbench/fullbench.vcxproj b/projects/VS2010/fullbench/fullbench.vcxproj similarity index 88% rename from visual/2013/fullbench/fullbench.vcxproj rename to projects/VS2010/fullbench/fullbench.vcxproj index 40935bbf7..2d9b78ce1 100644 --- a/visual/2013/fullbench/fullbench.vcxproj +++ b/projects/VS2010/fullbench/fullbench.vcxproj @@ -29,25 +29,21 @@ Application true MultiByte - v110 Application true - v110 MultiByte Application false - v110 true MultiByte Application false - v110 true MultiByte @@ -69,22 +65,24 @@ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(Platform)\$(Configuration)\ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(Platform)\$(Configuration)\ false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false @@ -127,7 +125,7 @@ true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false - true + false MultiThreaded @@ -146,7 +144,7 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true + false false MultiThreaded diff --git a/visual/2013/fullbench/fullbench.vcxproj.filters b/projects/VS2010/fullbench/fullbench.vcxproj.filters similarity index 100% rename from visual/2013/fullbench/fullbench.vcxproj.filters rename to projects/VS2010/fullbench/fullbench.vcxproj.filters diff --git a/visual/2013/fuzzer/fuzzer.vcxproj b/projects/VS2010/fuzzer/fuzzer.vcxproj similarity index 88% rename from visual/2013/fuzzer/fuzzer.vcxproj rename to projects/VS2010/fuzzer/fuzzer.vcxproj index bd3e849f4..047d5fd55 100644 --- a/visual/2013/fuzzer/fuzzer.vcxproj +++ b/projects/VS2010/fuzzer/fuzzer.vcxproj @@ -28,26 +28,22 @@ Application true - v110 MultiByte Application true - v110 MultiByte Application false - v110 true MultiByte Application false - v110 true MultiByte @@ -69,23 +65,25 @@ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath) false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); @@ -127,7 +125,7 @@ true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false - true + false MultiThreaded @@ -146,7 +144,7 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true + false false MultiThreaded diff --git a/visual/2013/fuzzer/fuzzer.vcxproj.filters b/projects/VS2010/fuzzer/fuzzer.vcxproj.filters similarity index 100% rename from visual/2013/fuzzer/fuzzer.vcxproj.filters rename to projects/VS2010/fuzzer/fuzzer.vcxproj.filters diff --git a/visual/2013/zstd.sln b/projects/VS2010/zstd.sln similarity index 100% rename from visual/2013/zstd.sln rename to projects/VS2010/zstd.sln diff --git a/visual/2013/zstd/zstd.vcxproj b/projects/VS2010/zstd/zstd.vcxproj old mode 100755 new mode 100644 similarity index 85% rename from visual/2013/zstd/zstd.vcxproj rename to projects/VS2010/zstd/zstd.vcxproj index d6f7edd66..8fff6320d --- a/visual/2013/zstd/zstd.vcxproj +++ b/projects/VS2010/zstd/zstd.vcxproj @@ -82,27 +82,23 @@ Application true - v110 MultiByte Application true - v110 MultiByte Application false true - v110 MultiByte Application false true - v110 MultiByte @@ -123,23 +119,29 @@ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath) + $(Platform)\$(Configuration)\ true - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath); false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath) + $(Platform)\$(Configuration)\ false - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false + $(LibraryPath); @@ -154,6 +156,7 @@ Console true + setargv.obj;%(AdditionalDependencies) @@ -163,12 +166,13 @@ Level4 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - false + true false Console true + setargv.obj;%(AdditionalDependencies) @@ -176,12 +180,12 @@ Level4 - Full + MaxSpeed true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false - true + false MultiThreaded @@ -189,7 +193,7 @@ true true true - setargv.obj; kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + setargv.obj;%(AdditionalDependencies) @@ -210,7 +214,7 @@ true true true - setargv.obj;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + setargv.obj;%(AdditionalDependencies) diff --git a/visual/2013/zstd/zstd.vcxproj.filters b/projects/VS2010/zstd/zstd.vcxproj.filters old mode 100755 new mode 100644 similarity index 100% rename from visual/2013/zstd/zstd.vcxproj.filters rename to projects/VS2010/zstd/zstd.vcxproj.filters diff --git a/visual/2013/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj similarity index 88% rename from visual/2013/zstdlib/zstdlib.vcxproj rename to projects/VS2010/zstdlib/zstdlib.vcxproj index c13ff0af0..e8625a501 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -48,9 +48,6 @@ - - - {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} Win32Proj @@ -61,27 +58,23 @@ DynamicLibrary true - v110 MultiByte DynamicLibrary true - v110 MultiByte DynamicLibrary false true - v110 MultiByte DynamicLibrary false true - v110 MultiByte @@ -104,28 +97,28 @@ true zstdlib_x86 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false true zstdlib_x64 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false zstdlib_x86 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false zstdlib_x64 $(Platform)\$(Configuration)\ - $(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false @@ -155,7 +148,7 @@ Level4 Disabled ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - false + true EnableFastChecks MultiThreadedDebugDLL ProgramDatabase diff --git a/visual/2013/zstdlib/zstdlib.vcxproj.filters b/projects/VS2010/zstdlib/zstdlib.vcxproj.filters similarity index 90% rename from visual/2013/zstdlib/zstdlib.vcxproj.filters rename to projects/VS2010/zstdlib/zstdlib.vcxproj.filters index 327f3a26e..e4bebb546 100644 --- a/visual/2013/zstdlib/zstdlib.vcxproj.filters +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj.filters @@ -9,10 +9,6 @@ {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hh;hpp;hxx;hm;inl;inc;xsd - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - @@ -102,9 +98,4 @@ Header Files - - - Resource Files - - \ No newline at end of file diff --git a/visual/2013/zstdlib/resource.h b/visual/2013/zstdlib/resource.h deleted file mode 100644 index db984ef4d6c17963c0691479aa0061d0c8000ffe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 812 zcmezWPoF`bp_-wZ!H>b8A)dj7!IdF^!Ii-e%<^M!X7FTiWe8@dWvFG~W#D1}sa9a{ zWyoYmW+-CFXDDXKXGmixVNhTQW5{GEW+-JyWXNGqU~p#8X3%C(U`S_3WyoVlWhi1u zWGG=sWk_LAV8~?1V@PJmVJKxtVMt|AU`S)gWXNGiWza*iA(WvQsyd0Gl0ku?ilLaH zgdv3?har<8i9wH{h#?uS8sshzhPcg-A(f#5>>`lqX$*-Br3^V>ei_(pr3|SI#b8_W z8HyMbz^*H2P+-VsNMguhNM%T7C}Ajuo2>w5DKlg;q=8+V#-PC9$PmB~%n-s5%HYD_ z2@W++20w-{27d-`23H0bBprEp^aL@uGB`50F!(e0G59bf!r;l^&k)ZL#Nf){$l$`@&)~=4!w|_3&k)QI$>7W2#Nf~1!w?L%59G?h zi9vn@xeP`tFt}rhi8OE=<}xTSlrUs~LpOxMl_7#5gu#=+kHMM22OJ)*3Q($ld=O<7e%V1Do z&;auj!Feu~K@&@?7lTtiNIxjIfx<@-8d9K~3d-Rc42}$L3=z;g6wKhy5Xun5;LPC4 z5YOPk;KSepHn)bMhCu+a|Neb4+b3u1%_aTPzEOk?D8PjgD}JgkdgpfJcDvU7=u59H-jsK3)lsqup68h zZT7dxqv6sRz3a_O(cfLW?2F~l$wF~l(BL2ZY)QGo#@j@?DZWcr#^dz8VYZVE#t z11P6J+y#my;!+zZ>_8Z#V{l@a>tHmf9RRBBiy1N*@)`0NVB#Pegq2ZpyaGcMxMYZD z@L}*|aAJsO2xD+%2x15Z*TH@aAbC)&?1CW^&*018$l%4`&k)3*!w?0oyRoYRb^LHrQmb+W`wJ?4b#2!GtgbfkPEk9>;_0di<_|xdFS0kWv}smN153EO7)1dp8Du zaO)5`USZ-OouC@XlK~X^m~jtrC3d?ZdIK1O82lMr7(yAG8A7n69>ONLF+gm01J@Pq z48h=(70lqxpuk|jP{Cl#kjCHxPRF3qC=Hr^6~J|wD}xh5D1$p#7U2(sNe~kZ;rde< zau|vkQVFOxKvEBCg@DQqh;LzT^=Al%y3T~b01Qo_dJwJ%VTfc10N0TqUl?J?L+VFV zRWO^em-(RD)Stl_95PA_!3-q~Md0>bI)fWSCPNNGDuX9O9zz;KK3XY^-84v9r~s~` z5vCe2fP9q90HTxNwFyiujK*&wOjQ_zBe(?%Y9}Z$I5XrkXHkz+)50`4m)QLdqdfDl!D8 z4+Cha4RJ5RwFnj{Z{Tk4k=M@}oUJ~Pt3bX*#*o?(R9}HgG~(K~Aae#M2Kfx+G8j$i r_=z!=o;2$C3CJc82H83|G01he({~<2K6o4ubF>81+C%L{gG>Sd7At^m From 0ba773878fcbd3cefede2720420748c1cf908e0f Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 16 May 2016 14:54:48 +0200 Subject: [PATCH 003/208] updated .gitignore for Visual Studio --- .gitignore | 6 +++++- appveyor.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b6d5d1325..2d3b6917a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,11 @@ *.app # Visual C++ -visual/ +projects/VS2008 +projects/VS2010 +projects/VS2012 +projects/VS2013 +projects/VS2015 # IDEA solution files *.idea diff --git a/appveyor.yml b/appveyor.yml index 848db3836..064baf9f9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,7 +32,7 @@ build_script: ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% ECHO *** msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe ECHO *** ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% From e30bac4c4b66730fa83981f4927c920b1d76b414 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 16 May 2016 15:10:33 +0200 Subject: [PATCH 004/208] fixed Win32 in AppVeyor --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 064baf9f9..d37f48a60 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,7 @@ version: 1.0.{build} configuration: -- Release - Debug +- Release platform: - Win32 - x64 @@ -19,7 +19,8 @@ build_script: ECHO *** REM msbuild "projects\VS2010\zstd.sln" /m /verbosity:detailed /property:PlatformToolset=Windows7.1SDK /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" REM msbuild "projects\VS2010\zstd.sln" "/p:AdditionalLibPaths=C:\Program Files\Microsoft SDKs\Windows\v7.1\Lib\X64" /m /verbosity:detailed /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - msbuild "projects\VS2010\zstd.sln" /p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + If %PLATFORM%==x64 SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe ECHO *** From 268775dae8a75810f5e261edf0e81afb8fe47965 Mon Sep 17 00:00:00 2001 From: klemens Date: Mon, 16 May 2016 22:23:24 +0200 Subject: [PATCH 005/208] spelling-fixes, as of lintian.debian.org --- lib/compress/huf_compress.c | 2 +- lib/compress/zbuff_compress.c | 2 +- programs/xxhash.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index d126305c6..695b66dae 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -116,7 +116,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */ if ((size <= 1) || (size >= maxSymbolValue/2)) { if (size==1) { /* RLE */ - /* only possible case : serie of 1 (because there are at least 2) */ + /* only possible case : series of 1 (because there are at least 2) */ /* can only be 2^n or (2^n-1), otherwise not an huffman tree */ BYTE code; switch(maxSymbolValue) diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 260aca087..fcbfbb95b 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -81,7 +81,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; -/* *** Ressources *** */ +/* *** Resources *** */ struct ZBUFF_CCtx_s { ZSTD_CCtx* zc; char* inBuff; diff --git a/programs/xxhash.c b/programs/xxhash.c index 2a66e251a..92df90295 100644 --- a/programs/xxhash.c +++ b/programs/xxhash.c @@ -65,7 +65,7 @@ You can contact the author at : /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ /*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. * Results are therefore identical for little-endian and big-endian CPU. * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. * Should endian-independance be of no importance for your application, you may set the #define below to 1, From ea4daa308d5478a8fc11e9e77c66e822624a280b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 17 May 2016 12:01:55 +0200 Subject: [PATCH 006/208] added `ZSTDv05_getFrameParams()` to "zstd_v05.h" --- lib/legacy/zstd_v05.c | 15 --------------- lib/legacy/zstd_v05.h | 10 +++++++++- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 42e925c42..9c57d18fc 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -436,21 +436,6 @@ extern "C" { ***************************************/ #define ZSTDv05_WINDOWLOG_ABSOLUTEMIN 11 -/* from faster to stronger */ -typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; - -typedef struct -{ - U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */ - U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */ - U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /* dispatch table : larger == faster, more memory */ - U32 searchLog; /* nb of searches : larger == more compression, slower */ - U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */ - U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */ - ZSTDv05_strategy strategy; -} ZSTDv05_parameters; - /*-************************************* * Advanced functions diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h index 8ef6bb25a..6dc17a647 100644 --- a/lib/legacy/zstd_v05.h +++ b/lib/legacy/zstd_v05.h @@ -40,7 +40,7 @@ extern "C" { * Dependencies ***************************************/ #include /* size_t */ - +#include "mem.h" /* U64, U32 */ /* ************************************* @@ -91,6 +91,14 @@ size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, /*-************************ * Advanced Streaming API ***************************/ +typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; +typedef struct { + U64 srcSize; + U32 windowLog; /* the only useful information to retrieve */ + U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; +} ZSTDv05_parameters; +size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); + size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); From 78aa05228a17b56816897e55ff6ac06efa2d56dd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 17 May 2016 14:30:19 +0200 Subject: [PATCH 007/208] fixed Visual warning in debug mode --- lib/common/mem.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/common/mem.h b/lib/common/mem.h index 201d9bb0f..99de9cb36 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -49,6 +49,9 @@ extern "C" { /*-**************************************** * Compiler specifics ******************************************/ +#if defined(_MSC_VER) +# include /* _byteswap_ */ +#endif #if defined(__GNUC__) # define MEM_STATIC static __attribute__((unused)) #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) From 5e334be4573dd3ff64f79b8b795a11721d6852ff Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 09:07:28 +0200 Subject: [PATCH 008/208] AppVeyor tests for mingw32, mingw64, clang --- appveyor.yml | 141 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 46 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d37f48a60..b5ffb4a25 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,51 +1,100 @@ version: 1.0.{build} -configuration: -- Debug -- Release -platform: -- Win32 -- x64 +environment: + matrix: + - ZSTD_APPVEYOR: "gcc" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" + - ZSTD_APPVEYOR: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + +install: + - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% + - if [%ZSTD_APPVEYOR%]==[gcc] ( + SET "MINGW_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen" && + SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && + SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && + SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && + COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe + ) else ( + IF [%PLATFORM%]==[x64] SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + ) build_script: -- cmd: | - ECHO *** - ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% - ECHO *** - REM msbuild "projects\VS2010\zstd.sln" /m /verbosity:detailed /property:PlatformToolset=Windows7.1SDK /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - REM msbuild "projects\VS2010\zstd.sln" "/p:AdditionalLibPaths=C:\Program Files\Microsoft SDKs\Windows\v7.1\Lib\X64" /m /verbosity:detailed /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - If %PLATFORM%==x64 SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe - - ECHO *** - ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% - ECHO *** - msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + - ECHO Building %ZSTD_APPVEYOR% + - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building mingw32 && + ECHO *** && + ECHO make %MINGW_PARAMS% && + make %MINGW_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_mingw32.exe && + make clean + ) + - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building mingw64 && + ECHO *** && + ECHO make %MINGW_PARAMS% && + make %MINGW_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_mingw64.exe && + make clean && + ECHO *** && + ECHO *** Building clang && + ECHO *** && + ECHO make %CLANG_PARAMS% && + make %CLANG_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_clang.exe && + make clean + ) + - if [%ZSTD_APPVEYOR%]==[visual] ( + ECHO *** && + ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && + ECHO *** && + ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + ) test_script: -- cmd: SET FUZZERTEST=-T1mn -- cmd: | - IF %CONFIGURATION%==Release (projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST%) - IF %CONFIGURATION%==Release (projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%) + - SET FUZZERTEST=-T1mn + - if [%ZSTD_APPVEYOR%]==[gcc] ( + projects\fuzzer_mingw32.exe %FUZZERTEST% && + projects\fuzzer_mingw64.exe %FUZZERTEST% && + projects\fuzzer_clang.exe %FUZZERTEST% + ) + - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( + projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% + ) From 236d94fa9a4ff8723922971274a119c6084d5dbc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 18 May 2016 12:06:33 +0200 Subject: [PATCH 009/208] reverted default compression level to 1 --- lib/compress/zstd_compress.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1bb75c68c..ceaff74e0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2432,7 +2432,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS /*-===== Pre-defined compression levels =====-*/ -#define ZSTD_DEFAULT_CLEVEL 5 +#define ZSTD_DEFAULT_CLEVEL 1 #define ZSTD_MAX_CLEVEL 22 unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } @@ -2543,17 +2543,16 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, }; -/*! ZSTD_getParams() : -* @return ZSTD_parameters structure for a selected compression level and srcSize. -* `srcSize` value is optional, select 0 if not known */ +/*! ZSTD_getCParams() : +* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. +* Size values are optional, provide 0 if not known or unused */ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize) { ZSTD_compressionParameters cp; size_t const addedSize = srcSize ? 0 : 500; U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ - if (compressionLevel < 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; - if (compressionLevel==0) compressionLevel = 1; + if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; cp = ZSTD_defaultCParameters[tableID][compressionLevel]; if (MEM_32bits()) { /* auto-correction, for 32-bits mode */ From d9967436da0383df4aba85c1e67907e6465a2eb9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 18 May 2016 14:16:49 +0200 Subject: [PATCH 010/208] added memory sanitizer to travis CI --- .travis.yml | 1 + Makefile | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index b7faed3eb..a430fee56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ env: - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=asan + - ZSTD_TRAVIS_CI_ENV=msan - ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" diff --git a/Makefile b/Makefile index 7d1ae713f..b1b9096a9 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,9 @@ usan: clean asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" +msan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" + asan32: clean $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" From f33021552b5036db1c143928f701d2b71355a4ec Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 16:38:36 +0200 Subject: [PATCH 011/208] added playTestsMSYS.sh --- appveyor.yml | 42 ++++---- programs/Makefile | 52 ++++++---- programs/{ => tests}/playTests.sh | 0 programs/tests/playTestsMSYS.sh | 160 ++++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 41 deletions(-) rename programs/{ => tests}/playTests.sh (100%) mode change 100755 => 100644 create mode 100644 programs/tests/playTestsMSYS.sh diff --git a/appveyor.yml b/appveyor.yml index b5ffb4a25..9a493a616 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,11 @@ version: 1.0.{build} environment: matrix: - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw32" + - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw64" - ZSTD_APPVEYOR: "visual" CONFIGURATION: "Debug" PLATFORM: "Win32" @@ -18,7 +23,6 @@ environment: install: - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%ZSTD_APPVEYOR%]==[gcc] ( - SET "MINGW_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen" && SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && @@ -29,24 +33,9 @@ install: build_script: - ECHO Building %ZSTD_APPVEYOR% - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw32 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw32.exe && - make clean - ) - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw64 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw64.exe && + - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] ( make clean && ECHO *** && ECHO *** Building clang && @@ -56,6 +45,15 @@ build_script: COPY programs\fuzzer.exe projects\fuzzer_clang.exe && make clean ) + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building %PLATFORM% && + ECHO *** && + ECHO make %MAKE_PARAMS% && + make %MAKE_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_%PLATFORM%.exe && + make clean + ) - if [%ZSTD_APPVEYOR%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && @@ -82,14 +80,14 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: - SET FUZZERTEST=-T1mn - if [%ZSTD_APPVEYOR%]==[gcc] ( - projects\fuzzer_mingw32.exe %FUZZERTEST% && - projects\fuzzer_mingw64.exe %FUZZERTEST% && - projects\fuzzer_clang.exe %FUZZERTEST% + projects\fuzzer_%PLATFORM%.exe %FUZZERTEST% && + if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && diff --git a/programs/Makefile b/programs/Makefile index 70434cedc..339427e2d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -163,7 +163,11 @@ clean: #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) +HOST_OS = POSIX +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) install: zstd @echo Installing binaries @@ -186,15 +190,41 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled +valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 +valgrindTest: zstd datagen fuzzer fullbench zbufftest + @echo "\n ---- valgrind tests : memory analyzer ----" + $(VALGRIND) ./datagen -g50M > $(VOID) + $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi + ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) + ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) + ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp + $(VALGRIND) ./zstd -vdf tmp -o $(VOID) + ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) + @rm tmp + $(VALGRIND) ./fuzzer -T1mn -t1 + $(VALGRIND) ./fullbench -i1 + $(VALGRIND) ./zbufftest -T1mn + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTestsMSYS.sh $(ZSTDRTTEST) +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) test: test-zstd test-fullbench test-fuzzer test-zbuff test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest -zstd-playTests: datagen - ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) - test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests @@ -223,20 +253,4 @@ test-zbuff: zbufftest test-zbuff32: zbufftest32 ./zbufftest32 $(ZBUFFTEST) - -valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: zstd datagen fuzzer fullbench zbufftest - @echo "\n ---- valgrind tests : memory analyzer ----" - $(VALGRIND) ./datagen -g50M > $(VOID) - $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi - ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) - ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp - $(VALGRIND) ./zstd -vdf tmp -o $(VOID) - ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) - @rm tmp - $(VALGRIND) ./fuzzer -T1mn -t1 - $(VALGRIND) ./fullbench -i1 - $(VALGRIND) ./zbufftest -T1mn - endif diff --git a/programs/playTests.sh b/programs/tests/playTests.sh old mode 100755 new mode 100644 similarity index 100% rename from programs/playTests.sh rename to programs/tests/playTests.sh diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh new file mode 100644 index 000000000..82f2357bd --- /dev/null +++ b/programs/tests/playTestsMSYS.sh @@ -0,0 +1,160 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 + fc tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + + +echo "\n**** simple tests **** " +./datagen > tmp +$ZSTD -f tmp # trivial compression case, creates tmp.zst +$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) +echo "test : too large compression level (must fail)" +$ZSTD -99 tmp && die "too large compression level undetected" +echo "test : compress to stdout" +$ZSTD tmp -c > tmpCompressed +$ZSTD tmp --stdout > tmpCompressed # long command format +echo "test : null-length file roundtrip" +echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout +echo "test : decompress file with wrong suffix (must fail)" +$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" +$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout +$ZSTD --decompress tmpCompressed -c > tmpResult +$ZSTD --decompress tmpCompressed --stdout > tmpResult +#$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout +#$ZSTD -d - < tmp.zst > /dev/null +$ZSTD -dc < tmp.zst > /dev/null +$ZSTD -dc - < tmp.zst > /dev/null +$ZSTD -q tmp && die "overwrite check failed!" +$ZSTD -q -f tmp +$ZSTD -q --force tmp +$ZSTD -df tmp && die "should have refused : wrong extension" +cp tmp tmp2.zst +$ZSTD -df tmp2.zst && die "should have failed : wrong format" +rm tmp2.zst + +echo "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD -c hello.tmp > hello.zstd +$ZSTD -c world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -dc helloworld.zstd > result.tmp +cat result.tmp +fc helloworld.tmp result.tmp +rm ./*.tmp ./*.zstd + +echo frame concatenation test completed + + +echo "\n**** dictionary tests **** " + +./datagen > tmpDict +./datagen -g1M | md5sum > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 +fc tmp1 tmp2 +$ZSTD --train *.c *.h -o tmpDict +$ZSTD xxhash.c -D tmpDict -of tmp +$ZSTD -d tmp -D tmpDict -of result +fc xxhash.c result + + +echo "\n**** multiple files tests **** " + +./datagen -s1 > tmp1 2> /dev/null +./datagen -s2 -g100K > tmp2 2> /dev/null +./datagen -s3 -g1M > tmp3 2> /dev/null +$ZSTD -f tmp* +echo "compress tmp* : " +ls -ls tmp* +rm tmp1 tmp2 tmp3 +echo "decompress tmp* : " +$ZSTD -df *.zst +ls -ls tmp* +echo "compress tmp* into stdout > tmpall : " +$ZSTD -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* +echo "decompress tmpall* into stdout > tmpdec : " +cp tmpall tmpall2 +# $ZSTD -dc tmpall* > tmpdec +ls -ls tmp* +echo "compress multiple files including a missing one (notHere) : " +$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" + +echo "\n**** integrity tests **** " +echo "test one file (tmp1.zst) " +$ZSTD -t tmp1.zst +$ZSTD --test tmp1.zst +echo "test multiple files (*.zst) " +$ZSTD -t *.zst +echo "test good and bad files (*) " +$ZSTD -t * && die "bad files not detected !" + +echo "\n**** zstd round-trip tests **** " + +roundTripTest +roundTripTest -g15K # TableID==3 +roundTripTest -g127K # TableID==2 +roundTripTest -g255K # TableID==1 +roundTripTest -g513K # TableID==0 +roundTripTest -g512K 6 # greedy, hash chain +roundTripTest -g512K 16 # btlazy2 +roundTripTest -g512K 19 # btopt + +rm tmp* + +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 + +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 1 + +rm tmp* + From bde926fce7b9bf1e7c4688569819c3c0bd2facd2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 18 May 2016 17:18:48 +0200 Subject: [PATCH 012/208] removed msan tests --- .travis.yml | 1 - Makefile | 2 +- programs/datagen.c | 102 ++++++++++++++++++++++--------------------- programs/zbufftest.c | 22 +++++----- 4 files changed, 64 insertions(+), 63 deletions(-) diff --git a/.travis.yml b/.travis.yml index a430fee56..b7faed3eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,6 @@ env: - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=asan - - ZSTD_TRAVIS_CI_ENV=msan - ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" diff --git a/Makefile b/Makefile index b1b9096a9..e7fca85fe 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" msan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason asan32: clean $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" diff --git a/programs/datagen.c b/programs/datagen.c index ecc783e1b..a17debcf0 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -48,22 +48,25 @@ * Macros **************************************/ #define KB *(1 <<10) +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define RDG_DEBUG 0 +#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ ) /*-************************************ -* Local types +* Local constants **************************************/ #define LTLOG 13 #define LTSIZE (1<> (32 - r))) -static unsigned int RDG_rand(U32* src) +static U32 RDG_rand(U32* src) { static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; @@ -72,39 +75,42 @@ static unsigned int RDG_rand(U32* src) rand32 ^= prime2; rand32 = RDG_rotl32(rand32, 13); *src = rand32; - return rand32; + return rand32 >> 5; } -static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) +static void RDG_fillLiteralDistrib(BYTE* ldt, double ld) { - U32 i = 0; + BYTE const firstChar = (ld<=0.0) ? 0 : '('; + BYTE const lastChar = (ld<=0.0) ? 255 : '}'; BYTE character = (ld<=0.0) ? 0 : '0'; - BYTE const firstChar = (ld<=0.0) ? 0 : '('; - BYTE const lastChar = (ld<=0.0) ?255: '}'; + U32 u; - while (i LTSIZE) weight = LTSIZE-i; - end = i + weight; - while (i < end) lt[i++] = character; + if (ld<=0.0) ld = 0.0; + //TRACE(" percent:%5.2f%% \n", ld*100.); + //TRACE(" start:(%c)[%02X] ", character, character); + for (u=0; u lastChar) character = firstChar; } } -static BYTE RDG_genChar(U32* seed, const litDistribTable lt) +static BYTE RDG_genChar(U32* seed, const BYTE* ldt) { U32 const id = RDG_rand(seed) & LTMASK; - return (lt[id]); + //TRACE(" %u : \n", id); + //TRACE(" %4u [%4u] ; val : %4u \n", id, id&255, ldt[id]); + return (ldt[id]); /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with 0.0. Checked : table is fully initialized */ } -#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 0x7FFF) -#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) -void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) +#define RDG_RAND15BITS ( RDG_rand(seed) & 0x7FFF ) +#define RDG_RANDLENGTH ( (RDG_rand(seed) & 7) ? (RDG_rand(seed) & 0xF) : (RDG_rand(seed) & 0x1FF) + 0xF) +void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr) { BYTE* buffPtr = (BYTE*)buffer; const U32 matchProba32 = (U32)(32768 * matchProba); @@ -123,75 +129,73 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match } memset(buffPtr+pos, 0, size0); pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, lt); + buffPtr[pos-1] = RDG_genChar(seed, ldt); continue; } /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; + if (pos==0) buffPtr[0] = RDG_genChar(seed, ldt), pos=1; /* Generate compressible data */ while (pos < buffSize) { /* Select : Literal (char) or Match (within 32K) */ if (RDG_RAND15BITS < matchProba32) { /* Copy (within 32K) */ - size_t match; - size_t d; - size_t const length = RDG_RANDLENGTH + 4; - U32 offset = RDG_RAND15BITS + 1; - U32 repeatOffset = (RDG_rand(seed) & 15) == 2; - if (repeatOffset) offset = prevOffset; - if (offset > pos) offset = (U32)pos; - prevOffset = offset; - match = pos - offset; - d = pos + length; - if (d > buffSize) d = buffSize; + U32 const length = RDG_RANDLENGTH + 4; + U32 const d = MIN (pos + length , buffSize); + U32 const repeatOffset = (RDG_rand(seed) & 15) == 2; + U32 const randOffset = RDG_RAND15BITS + 1; + U32 const offset = repeatOffset ? prevOffset : MIN(randOffset , pos); + size_t match = pos - offset; + //TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length); while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ + prevOffset = offset; } else { /* Literal (noise) */ - size_t const length = RDG_RANDLENGTH; - size_t d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); + U32 const length = RDG_RANDLENGTH; + U32 const d = MIN(pos + length, buffSize); + while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt); } } } void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) { - litDistribTable lt; - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); + BYTE ldt[LTSIZE]; + memset(ldt, '0', sizeof(ldt)); + if (litProba<=0.0) litProba = matchProba / 4.5; + //TRACE(" percent:%5.2f%% \n", litProba*100.); + RDG_fillLiteralDistrib(ldt, litProba); + RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed); } -#define RDG_DICTSIZE (32 KB) -#define RDG_BLOCKSIZE (128 KB) -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed) { - BYTE* buff = (BYTE*)malloc(RDG_DICTSIZE + RDG_BLOCKSIZE); + size_t const stdBlockSize = 128 KB; + size_t const stdDictSize = 32 KB; + BYTE* buff = (BYTE*)malloc(stdDictSize + stdBlockSize); U64 total = 0; - litDistribTable ldt; + BYTE ldt[LTSIZE]; /* init */ if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); } if (litProba<=0.0) litProba = matchProba / 4.5; + memset(ldt, '0', sizeof(ldt)); RDG_fillLiteralDistrib(ldt, litProba); SET_BINARY_MODE(stdout); /* Generate initial dict */ - RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldt, &seed); + RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed); /* Generate compressible data */ while (total < size) { - size_t const genBlockSize = (size_t) (MIN (RDG_BLOCKSIZE, size-total)); - RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldt, &seed); + size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total)); + RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed); total += genBlockSize; { size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; } /* update dict */ - memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); + memcpy(buff, buff + stdBlockSize, stdDictSize); } /* cleanup */ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f8d527ef2..eb63117ab 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -140,7 +140,6 @@ static int basicUnitTests(U32 seed, double compressibility) void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); - U32 randState = seed; size_t result, cSize, readSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx(); @@ -151,7 +150,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAY("Not enough memory, aborting\n"); goto _output_error; } - RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState); + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); @@ -234,19 +233,13 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max) { const BYTE* b1 = (const BYTE*)buf1; const BYTE* b2 = (const BYTE*)buf2; - size_t i; - for (i=0; i "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } - - static size_t FUZ_rLogLength(U32* seed, U32 logLength) { size_t const lengthMask = ((size_t)1 << logLength) - 1; @@ -259,6 +252,11 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) return FUZ_rLogLength(seed, logLength); } +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { static const U32 maxSrcLog = 24; From 35276a944143fb752158a0643708d3f6379d41a8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 18 May 2016 17:26:23 +0200 Subject: [PATCH 013/208] fixed minor conversion warnings --- programs/datagen.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index a17debcf0..fd9b2316f 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -142,10 +142,10 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match if (RDG_RAND15BITS < matchProba32) { /* Copy (within 32K) */ U32 const length = RDG_RANDLENGTH + 4; - U32 const d = MIN (pos + length , buffSize); + U32 const d = (U32) MIN(pos + length , buffSize); U32 const repeatOffset = (RDG_rand(seed) & 15) == 2; U32 const randOffset = RDG_RAND15BITS + 1; - U32 const offset = repeatOffset ? prevOffset : MIN(randOffset , pos); + U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos); size_t match = pos - offset; //TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length); while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ @@ -153,7 +153,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match } else { /* Literal (noise) */ U32 const length = RDG_RANDLENGTH; - U32 const d = MIN(pos + length, buffSize); + U32 const d = (U32) MIN(pos + length, buffSize); while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt); } } } From 877cc84f859cd910355682d65e502349afae5ca8 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:03:02 +0200 Subject: [PATCH 014/208] changed appveyor.yml --- .gitattributes | 3 +++ appveyor.yml | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index 3959727b8..f314af9ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -14,3 +14,6 @@ *.vcxproj* text eol=crlf *.suo binary *.rc binary + +# Windows +*.bat text eol=crlf diff --git a/appveyor.yml b/appveyor.yml index 9a493a616..b79ef2a59 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,9 @@ install: SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe ) else ( - IF [%PLATFORM%]==[x64] SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") && + COPY programs\tests\playTests.bat playTests.bat && + COPY programs\tests\roundTripTest.bat roundTripTest.bat ) build_script: @@ -51,7 +53,6 @@ build_script: ECHO *** && ECHO make %MAKE_PARAMS% && make %MAKE_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_%PLATFORM%.exe && make clean ) - if [%ZSTD_APPVEYOR%]==[visual] ( @@ -79,14 +80,13 @@ build_script: ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: - SET FUZZERTEST=-T1mn - if [%ZSTD_APPVEYOR%]==[gcc] ( - projects\fuzzer_%PLATFORM%.exe %FUZZERTEST% && if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( @@ -94,5 +94,6 @@ test_script: projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && + zstd_VS2015_%PLATFORM%_Release.exe ) From 3aa4b2b598686973525a540e00247405d37a3d92 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:04:10 +0200 Subject: [PATCH 015/208] update file permission --- programs/tests/playTests.sh | 0 programs/tests/playTestsMSYS.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 programs/tests/playTests.sh mode change 100644 => 100755 programs/tests/playTestsMSYS.sh diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh old mode 100644 new mode 100755 diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh old mode 100644 new mode 100755 From 2bd6b13e5337ed10af0292963e51b1ac38ab3258 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:05:56 +0200 Subject: [PATCH 016/208] added playTests.bat --- programs/tests/playTests.bat | 149 +++++++++++++++++++++++++++++++ programs/tests/roundTripTest.bat | 8 ++ 2 files changed, 157 insertions(+) create mode 100644 programs/tests/playTests.bat create mode 100644 programs/tests/roundTripTest.bat diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat new file mode 100644 index 000000000..19c57d5fc --- /dev/null +++ b/programs/tests/playTests.bat @@ -0,0 +1,149 @@ +@echo off +if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 + +echo. && echo **** simple tests **** +datagen > tmp +%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst +%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) +echo test : too large compression level (must fail) +%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1) +echo test : compress to stdout +%ZSTD% tmp -c > tmpCompressed +%ZSTD% tmp --stdout > tmpCompressed && REM long command format +echo test : null-length file roundtrip +echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1) +echo test : decompress file with wrong suffix (must fail) +%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1) +%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout +%ZSTD% --decompress tmpCompressed -c > tmpResult +%ZSTD% --decompress tmpCompressed --stdout > tmpResult +REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout +REM %ZSTD% -d - < tmp.zst > NUL +%ZSTD% -dc < tmp.zst > NUL +%ZSTD% -dc - < tmp.zst > NUL +%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1) +%ZSTD% -q -f tmp +%ZSTD% -q --force tmp +%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) +cp tmp tmp2.zst +%ZSTD% -df tmp2.zst && (echo should have failed : wrong format && exit /b 1) +rm tmp2.zst + +echo. && echo **** frame concatenation **** + +echo hello > hello.tmp +echo world! > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +%ZSTD% -c hello.tmp > hello.zstd +%ZSTD% -c world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +%ZSTD% -dc helloworld.zstd > result.tmp +cat result.tmp +fc /b helloworld.tmp result.tmp +rm *.tmp *.zstd + +echo frame concatenation test completed + + +REM echo. && echo **** flush write error test **** + +REM echo echo foo ^| %ZSTD% ^> v:\full +REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1) +REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full" +REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1) + + +echo. && echo **** dictionary tests **** + +datagen > tmpDict +datagen -g1M | md5sum > tmp1 +datagen -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 +fc tmp1 tmp2 +%ZSTD% --train *.c *.h -o tmpDict +%ZSTD% xxhash.c -D tmpDict -of tmp +%ZSTD% -d tmp -D tmpDict -of result +fc xxhash.c result + + +echo. && echo **** multiple files tests **** + +datagen -s1 > tmp1 2> NUL +datagen -s2 -g100K > tmp2 2> NUL +datagen -s3 -g1M > tmp3 2> NUL +%ZSTD% -f tmp* +echo compress tmp* : +ls -ls tmp* +rm tmp1 tmp2 tmp3 +echo decompress tmp* : +%ZSTD% -df *.zst +ls -ls tmp* + +echo compress tmp* into stdout ^> tmpall : +%ZSTD% -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* +echo decompress tmpall* into stdout ^> tmpdec : +cp tmpall tmpall2 +%ZSTD% -dc tmpall* > tmpdec +ls -ls tmp* +echo compress multiple files including a missing one (notHere) : +%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1) + + +echo. && echo **** integrity tests **** +echo test one file (tmp1.zst) +%ZSTD% -t tmp1.zst +%ZSTD% --test tmp1.zst +echo test multiple files (*.zst) +%ZSTD% -t *.zst +echo test good and bad files (*) +%ZSTD% -t * && (echo bad files not detected! && exit /b 1) + + +echo. && echo **** zstd round-trip tests **** + +CALL roundTripTest.bat +CALL roundTripTest.bat -g15K && REM TableID==3 +CALL roundTripTest.bat -g127K && REM TableID==2 +CALL roundTripTest.bat -g255K && REM TableID==1 +CALL roundTripTest.bat -g513K && REM TableID==0 +CALL roundTripTest.bat -g512K 6 && REM greedy, hash chain +CALL roundTripTest.bat -g512K 16 && REM btlazy2 +CALL roundTripTest.bat -g512K 19 && REM btopt + +rm tmp* +echo Param = %1 +if NOT "%1"=="--test-large-data" ( + echo skipping large data tests + exit /b 0 +) + +CALL roundTripTest.bat -g270000000 1 +CALL roundTripTest.bat -g270000000 2 +CALL roundTripTest.bat -g270000000 3 + +CALL roundTripTest.bat -g140000000 -P60 4 +CALL roundTripTest.bat -g140000000 -P60 5 +CALL roundTripTest.bat -g140000000 -P60 6 + +CALL roundTripTest.bat -g70000000 -P70 7 +CALL roundTripTest.bat -g70000000 -P70 8 +CALL roundTripTest.bat -g70000000 -P70 9 + +CALL roundTripTest.bat -g35000000 -P75 10 +CALL roundTripTest.bat -g35000000 -P75 11 +CALL roundTripTest.bat -g35000000 -P75 12 + +CALL roundTripTest.bat -g18000000 -P80 13 +CALL roundTripTest.bat -g18000000 -P80 14 +CALL roundTripTest.bat -g18000000 -P80 15 +CALL roundTripTest.bat -g18000000 -P80 16 +CALL roundTripTest.bat -g18000000 -P80 17 + +CALL roundTripTest.bat -g50000000 -P94 18 +CALL roundTripTest.bat -g50000000 -P94 19 + +CALL roundTripTest.bat -g99000000 -P99 20 +CALL roundTripTest.bat -g6000000000 -P99 1 + +rm tmp* +exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat new file mode 100644 index 000000000..b1541bd14 --- /dev/null +++ b/programs/tests/roundTripTest.bat @@ -0,0 +1,8 @@ +@echo off +if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) +rm -f tmp1 tmp2 +echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d +datagen %1 %P% | md5sum > tmp1 +datagen %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 +fc tmp1 tmp2 +EXIT /B %ERRORLEVEL% From 60af95d9718a54c165937fb8f325e1d278b61565 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 10:29:49 +0200 Subject: [PATCH 017/208] fixed compression with -c with multiple files on Windows --- appveyor.yml | 2 +- programs/fileio.c | 3 ++- programs/tests/playTestsMSYS.sh | 14 +++++++------- programs/zstdcli.c | 5 +---- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b79ef2a59..e002d89ef 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -95,5 +95,5 @@ test_script: projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - zstd_VS2015_%PLATFORM%_Release.exe + REM zstd_VS2015_%PLATFORM%_Release.exe ) diff --git a/programs/fileio.c b/programs/fileio.c index b3f324ac6..638e63b57 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -71,7 +71,7 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ # include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); } #else # include /* isatty */ # define SET_BINARY_MODE(file) @@ -439,6 +439,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile if (!strcmp(suffix, stdoutmark)) { unsigned u; ress.dstFile = stdout; + SET_BINARY_MODE(stdout); for (u=0; u tmp $ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) @@ -51,7 +51,7 @@ cp tmp tmp2.zst $ZSTD -df tmp2.zst && die "should have failed : wrong format" rm tmp2.zst -echo "\n**** frame concatenation **** " +echo -e "\n**** frame concatenation **** " echo "hello " > hello.tmp echo "world!" > world.tmp @@ -67,7 +67,7 @@ rm ./*.tmp ./*.zstd echo frame concatenation test completed -echo "\n**** dictionary tests **** " +echo -e "\n**** dictionary tests **** " ./datagen > tmpDict ./datagen -g1M | md5sum > tmp1 @@ -79,7 +79,7 @@ $ZSTD -d tmp -D tmpDict -of result fc xxhash.c result -echo "\n**** multiple files tests **** " +echo -e "\n**** multiple files tests **** " ./datagen -s1 > tmp1 2> /dev/null ./datagen -s2 -g100K > tmp2 2> /dev/null @@ -96,12 +96,12 @@ $ZSTD -c tmp1 tmp2 tmp3 > tmpall ls -ls tmp* echo "decompress tmpall* into stdout > tmpdec : " cp tmpall tmpall2 -# $ZSTD -dc tmpall* > tmpdec +$ZSTD -dc tmpall* > tmpdec ls -ls tmp* echo "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" -echo "\n**** integrity tests **** " +echo -e "\n**** integrity tests **** " echo "test one file (tmp1.zst) " $ZSTD -t tmp1.zst $ZSTD --test tmp1.zst @@ -110,7 +110,7 @@ $ZSTD -t *.zst echo "test good and bad files (*) " $ZSTD -t * && die "bad files not detected !" -echo "\n**** zstd round-trip tests **** " +echo -e "\n**** zstd round-trip tests **** " roundTripTest roundTripTest -g15K # TableID==3 diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 310db7f88..8a8af3a2c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -49,14 +49,11 @@ * OS-specific Includes **************************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else extern int fileno(FILE *stream); /* triggers fileno() within on POSIX */ # include /* isatty */ -# define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #endif From 44bb2f6c89161915c8ff46ddc3b3492116b1c377 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 12:16:14 +0200 Subject: [PATCH 018/208] added datagen in Visual Studio project --- appveyor.yml | 12 +- programs/datagencli.c | 2 +- programs/tests/playTests.bat | 75 ++++---- programs/tests/roundTripTest.bat | 4 +- projects/VS2010/datagen/datagen.vcxproj | 168 ++++++++++++++++++ .../VS2010/datagen/datagen.vcxproj.filters | 26 +++ projects/VS2010/zstd.sln | 10 ++ 7 files changed, 253 insertions(+), 44 deletions(-) create mode 100644 projects/VS2010/datagen/datagen.vcxproj create mode 100644 projects/VS2010/datagen/datagen.vcxproj.filters diff --git a/appveyor.yml b/appveyor.yml index e002d89ef..a68e12eab 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,9 +28,7 @@ install: SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe ) else ( - IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") && - COPY programs\tests\playTests.bat playTests.bat && - COPY programs\tests\roundTripTest.bat roundTripTest.bat + IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") ) build_script: @@ -81,7 +79,8 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\datagen.exe projects\datagen_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: @@ -95,5 +94,8 @@ test_script: projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - REM zstd_VS2015_%PLATFORM%_Release.exe + CD programs && + SET ZSTD=..\projects\zstd_VS2015_%PLATFORM%_Release.exe && + SET DATAGEN=..\projects\datagen_VS2015_%PLATFORM%_Release.exe && + CALL tests\playTests.bat --test-large-data ) diff --git a/programs/datagencli.c b/programs/datagencli.c index 6122c29bd..d437d5cb3 100644 --- a/programs/datagencli.c +++ b/programs/datagencli.c @@ -27,8 +27,8 @@ /*-************************************ * Includes **************************************/ +#include "util.h" /* Compiler options */ #include /* fprintf, stderr */ -#include "mem.h" #include "datagen.h" /* RDG_generate */ diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index 19c57d5fc..6e9fd53ff 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -1,8 +1,11 @@ @echo off if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 +if [%DATAGEN%]==[] echo DATAGEN variable must be defined! && exit /b 1 + +SET ROUNDTRIPTEST=tests\roundTripTest.bat echo. && echo **** simple tests **** -datagen > tmp +%DATAGEN% > tmp %ZSTD% -f tmp && REM trivial compression case, creates tmp.zst %ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) echo test : too large compression level (must fail) @@ -55,9 +58,9 @@ REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! echo. && echo **** dictionary tests **** -datagen > tmpDict -datagen -g1M | md5sum > tmp1 -datagen -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 +%DATAGEN% > tmpDict +%DATAGEN% -g1M | md5sum > tmp1 +%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 fc tmp1 tmp2 %ZSTD% --train *.c *.h -o tmpDict %ZSTD% xxhash.c -D tmpDict -of tmp @@ -67,9 +70,9 @@ fc xxhash.c result echo. && echo **** multiple files tests **** -datagen -s1 > tmp1 2> NUL -datagen -s2 -g100K > tmp2 2> NUL -datagen -s3 -g1M > tmp3 2> NUL +%DATAGEN% -s1 > tmp1 2> NUL +%DATAGEN% -s2 -g100K > tmp2 2> NUL +%DATAGEN% -s3 -g1M > tmp3 2> NUL %ZSTD% -f tmp* echo compress tmp* : ls -ls tmp* @@ -101,14 +104,14 @@ echo test good and bad files (*) echo. && echo **** zstd round-trip tests **** -CALL roundTripTest.bat -CALL roundTripTest.bat -g15K && REM TableID==3 -CALL roundTripTest.bat -g127K && REM TableID==2 -CALL roundTripTest.bat -g255K && REM TableID==1 -CALL roundTripTest.bat -g513K && REM TableID==0 -CALL roundTripTest.bat -g512K 6 && REM greedy, hash chain -CALL roundTripTest.bat -g512K 16 && REM btlazy2 -CALL roundTripTest.bat -g512K 19 && REM btopt +CALL %ROUNDTRIPTEST% +CALL %ROUNDTRIPTEST% -g15K && REM TableID==3 +CALL %ROUNDTRIPTEST% -g127K && REM TableID==2 +CALL %ROUNDTRIPTEST% -g255K && REM TableID==1 +CALL %ROUNDTRIPTEST% -g513K && REM TableID==0 +CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain +CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2 +CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt rm tmp* echo Param = %1 @@ -117,33 +120,33 @@ if NOT "%1"=="--test-large-data" ( exit /b 0 ) -CALL roundTripTest.bat -g270000000 1 -CALL roundTripTest.bat -g270000000 2 -CALL roundTripTest.bat -g270000000 3 +CALL %ROUNDTRIPTEST% -g270000000 1 +CALL %ROUNDTRIPTEST% -g270000000 2 +CALL %ROUNDTRIPTEST% -g270000000 3 -CALL roundTripTest.bat -g140000000 -P60 4 -CALL roundTripTest.bat -g140000000 -P60 5 -CALL roundTripTest.bat -g140000000 -P60 6 +CALL %ROUNDTRIPTEST% -g140000000 -P60 4 +CALL %ROUNDTRIPTEST% -g140000000 -P60 5 +CALL %ROUNDTRIPTEST% -g140000000 -P60 6 -CALL roundTripTest.bat -g70000000 -P70 7 -CALL roundTripTest.bat -g70000000 -P70 8 -CALL roundTripTest.bat -g70000000 -P70 9 +CALL %ROUNDTRIPTEST% -g70000000 -P70 7 +CALL %ROUNDTRIPTEST% -g70000000 -P70 8 +CALL %ROUNDTRIPTEST% -g70000000 -P70 9 -CALL roundTripTest.bat -g35000000 -P75 10 -CALL roundTripTest.bat -g35000000 -P75 11 -CALL roundTripTest.bat -g35000000 -P75 12 +CALL %ROUNDTRIPTEST% -g35000000 -P75 10 +CALL %ROUNDTRIPTEST% -g35000000 -P75 11 +CALL %ROUNDTRIPTEST% -g35000000 -P75 12 -CALL roundTripTest.bat -g18000000 -P80 13 -CALL roundTripTest.bat -g18000000 -P80 14 -CALL roundTripTest.bat -g18000000 -P80 15 -CALL roundTripTest.bat -g18000000 -P80 16 -CALL roundTripTest.bat -g18000000 -P80 17 +CALL %ROUNDTRIPTEST% -g18000000 -P80 13 +CALL %ROUNDTRIPTEST% -g18000000 -P80 14 +CALL %ROUNDTRIPTEST% -g18000000 -P80 15 +CALL %ROUNDTRIPTEST% -g18000000 -P80 16 +CALL %ROUNDTRIPTEST% -g18000000 -P80 17 -CALL roundTripTest.bat -g50000000 -P94 18 -CALL roundTripTest.bat -g50000000 -P94 19 +CALL %ROUNDTRIPTEST% -g50000000 -P94 18 +CALL %ROUNDTRIPTEST% -g50000000 -P94 19 -CALL roundTripTest.bat -g99000000 -P99 20 -CALL roundTripTest.bat -g6000000000 -P99 1 +CALL %ROUNDTRIPTEST% -g99000000 -P99 20 +CALL %ROUNDTRIPTEST% -g6000000000 -P99 1 rm tmp* exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat index b1541bd14..759891f0b 100644 --- a/programs/tests/roundTripTest.bat +++ b/programs/tests/roundTripTest.bat @@ -2,7 +2,7 @@ if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) rm -f tmp1 tmp2 echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d -datagen %1 %P% | md5sum > tmp1 -datagen %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 +%DATAGEN% %1 %P% | md5sum > tmp1 +%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 fc tmp1 tmp2 EXIT /B %ERRORLEVEL% diff --git a/projects/VS2010/datagen/datagen.vcxproj b/projects/VS2010/datagen/datagen.vcxproj new file mode 100644 index 000000000..fe2b7eec6 --- /dev/null +++ b/projects/VS2010/datagen/datagen.vcxproj @@ -0,0 +1,168 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {037E781E-81A6-494B-B1B3-438AB1200523} + Win32Proj + datagen + $(SolutionDir)bin\$(Platform)\$(Configuration)\ + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + true + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ + + + true + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + + + false + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ + + + false + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + + + + + + + + + \ No newline at end of file diff --git a/projects/VS2010/datagen/datagen.vcxproj.filters b/projects/VS2010/datagen/datagen.vcxproj.filters new file mode 100644 index 000000000..1ebbd6b03 --- /dev/null +++ b/projects/VS2010/datagen/datagen.vcxproj.filters @@ -0,0 +1,26 @@ + + + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + Source Files + + + + + Header Files + + + \ No newline at end of file diff --git a/projects/VS2010/zstd.sln b/projects/VS2010/zstd.sln index 825d7a130..698b8fe50 100644 --- a/projects/VS2010/zstd.sln +++ b/projects/VS2010/zstd.sln @@ -9,6 +9,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\full EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -49,6 +51,14 @@ Global {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32 {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64 {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.ActiveCfg = Debug|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.Build.0 = Debug|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.ActiveCfg = Debug|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.Build.0 = Debug|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.ActiveCfg = Release|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.Build.0 = Release|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.ActiveCfg = Release|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 96abb5a806941d07ae5e6de91ad62c566c7c40c5 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 15:50:58 +0200 Subject: [PATCH 019/208] AppVeyor tests use playTests.bat --- appveyor.yml | 64 +++++++++++++++++++----------------- programs/tests/playTests.bat | 7 ++-- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a68e12eab..c6f735ea3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,28 +1,29 @@ version: 1.0.{build} environment: matrix: - - ZSTD_APPVEYOR: "gcc" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" + - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw32" - - ZSTD_APPVEYOR: "gcc" + - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw64" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" install: - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% + - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% + - if [%COMPILER%]==[gcc] ( SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && @@ -32,7 +33,7 @@ install: ) build_script: - - ECHO Building %ZSTD_APPVEYOR% + - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] ( @@ -45,7 +46,7 @@ build_script: COPY programs\fuzzer.exe projects\fuzzer_clang.exe && make clean ) - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - if [%COMPILER%]==[gcc] ( ECHO *** && ECHO *** Building %PLATFORM% && ECHO *** && @@ -53,7 +54,7 @@ build_script: make %MAKE_PARAMS% && make clean ) - - if [%ZSTD_APPVEYOR%]==[visual] ( + - if [%COMPILER%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -62,7 +63,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && @@ -77,25 +78,28 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\datagen.exe projects\datagen_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\ ) test_script: + - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - SET FUZZERTEST=-T1mn - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - if [%COMPILER%]==[gcc] ( if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( + - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( + CD programs && + SET ZSTD=..\projects\zstd.exe && + SET DATAGEN=..\projects\datagen.exe && + CALL tests\playTests.bat --test-large-data && + CD .. && + projects\fullbench.exe -i1 && + projects\fullbench.exe -i1 -P0 && projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - CD programs && - SET ZSTD=..\projects\zstd_VS2015_%PLATFORM%_Release.exe && - SET DATAGEN=..\projects\datagen_VS2015_%PLATFORM%_Release.exe && - CALL tests\playTests.bat --test-large-data + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index 6e9fd53ff..edbbcb340 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -1,6 +1,9 @@ @echo off -if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 -if [%DATAGEN%]==[] echo DATAGEN variable must be defined! && exit /b 1 +ECHO Start playTests.bat +ECHO ZSTD^: %ZSTD% +ECHO DATAGEN^: %DATAGEN% +if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1 +if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1 SET ROUNDTRIPTEST=tests\roundTripTest.bat From e162aceeb66e39fd059bbaf012560b2eb2e342d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 20 May 2016 11:24:35 +0200 Subject: [PATCH 020/208] minor simplification --- programs/bench.c | 33 ++++++++++++++------------------- programs/util.h | 15 +++++++-------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index bed7d16bc..a2721552e 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -188,7 +188,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Bench */ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); - U64 crcCheck = 0; UTIL_time_t coolTime; U32 testNb; size_t cSize = 0; @@ -282,7 +281,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, (double)srcSize / fastestD ); /* CRC Checking */ - { crcCheck = XXH64(resultBuffer, srcSize, 0); + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); if (crcOrig!=crcCheck) { size_t u; DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); @@ -308,12 +307,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, #endif } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ - if (crcOrig == crcCheck) { - result->ratio = ratio; - result->cSize = cSize; - result->cSpeed = (double)srcSize / fastestC; - result->dSpeed = (double)srcSize / fastestD; - } + result->ratio = ratio; + result->cSize = cSize; + result->cSpeed = (double)srcSize / fastestC; + result->dSpeed = (double)srcSize / fastestD; DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ @@ -402,9 +399,9 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, const char** fileNamesTable, unsigned nbFiles) { size_t pos = 0, totalSize = 0; - FILE* f; unsigned n; for (n=0; n 1) displayName = mfName; - else displayName = fileNamesTable[0]; - - BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); + { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); + } /* clean up */ free(srcBuffer); @@ -482,7 +477,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility { char name[20] = {0}; size_t benchedSize = 10000000; - void* srcBuffer = malloc(benchedSize); + void* const srcBuffer = malloc(benchedSize); /* Memory allocation */ if (!srcBuffer) EXM_THROW(21, "not enough memory"); diff --git a/programs/util.h b/programs/util.h index e6eb5f41e..44ab43488 100644 --- a/programs/util.h +++ b/programs/util.h @@ -37,7 +37,6 @@ extern "C" { # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ #define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ - //#define snprintf _snprintf #endif #endif @@ -47,7 +46,7 @@ extern "C" { # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # if defined(__sun__) /* Sun Solaris 32-bits requires specific definitions */ # define _LARGEFILE_SOURCE /* fseeko, ftello */ -# else +# else # define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ # endif #endif @@ -91,7 +90,7 @@ extern "C" { # define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) # include # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ @@ -140,8 +139,8 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) { UTIL_time_t clockStart, clockEnd; UTIL_getTime(&clockStart); - do { - UTIL_getTime(&clockEnd); + do { + UTIL_getTime(&clockEnd); } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); } @@ -280,7 +279,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); return 0; } - + while ((entry = readdir(dir)) != NULL) { if (strcmp (entry->d_name, "..") == 0 || strcmp (entry->d_name, ".") == 0) continue; @@ -324,8 +323,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #endif // #ifdef _WIN32 -/* - * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, +/* + * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. From 7591a7f224b3eb6e56c350e2632bc9e17a0356f5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 20 May 2016 11:44:43 +0200 Subject: [PATCH 021/208] minor refactoring --- lib/compress/zstd_compress.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ceaff74e0..f01029cc7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -942,7 +942,7 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLim const BYTE* const pStart = pIn; while ((pIn iEnd) vEnd = iEnd; - matchLength = ZSTD_count(ip, match, vEnd); + const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); + size_t matchLength = ZSTD_count(ip, match, vEnd); if (match + matchLength == mEnd) matchLength += ZSTD_count(ip+matchLength, iStart, iEnd); return matchLength; From de4d4da52d62d26f191a28a6f66a39adeee7e5de Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 20 May 2016 12:20:12 +0200 Subject: [PATCH 022/208] changed version to 0.6.2 removed make-controlled version (now used from zstd.h) removed std=c99 by default added : c90/gno90/c99/gnu99/c11 tests --- .travis.yml | 2 ++ Makefile | 20 +++++++++++++------- lib/Makefile | 2 +- lib/common/zstd.h | 2 +- programs/Makefile | 15 ++------------- programs/zstdcli.c | 2 ++ 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7faed3eb..536b5a845 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ env: - ZSTD_TRAVIS_CI_ENV=clangtest - ZSTD_TRAVIS_CI_ENV=gpptest - ZSTD_TRAVIS_CI_ENV=gnu90test + - ZSTD_TRAVIS_CI_ENV=c99test + - ZSTD_TRAVIS_CI_ENV=gnu99test - ZSTD_TRAVIS_CI_ENV=armtest-w-install - ZSTD_TRAVIS_CI_ENV=test - ZSTD_TRAVIS_CI_ENV="-C programs test32" diff --git a/Makefile b/Makefile index e7fca85fe..95ecc007c 100644 --- a/Makefile +++ b/Makefile @@ -30,9 +30,6 @@ # - zstd homepage : http://www.zstd.net/ # ################################################################ -# force a version number : uncomment below export (otherwise, default to the one declared into zstd.h) -#export VERSION := 0.6.1 - PRGDIR = programs ZSTDDIR = lib @@ -88,11 +85,20 @@ clangtest: clean gpptest: clean $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" -gnu90test: clean - $(MAKE) all CFLAGS="-std=gnu90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Wdeclaration-after-statement -Werror" - c90test: clean - $(MAKE) all CFLAGS="-std=c90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Werror" # will fail, due to // and long long + CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long + +gnu90test: clean + CFLAGS="-std=gnu90" $(MAKE) all + +c99test: clean + CFLAGS="-std=c99" $(MAKE) all + +gnu99test: clean + CFLAGS="-std=gnu99" $(MAKE) all + +c11test: clean + CFLAGS="-std=c11" $(MAKE) all bmix64test: clean CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test diff --git a/lib/Makefile b/lib/Makefile index 5ea014dd1..904366959 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -45,7 +45,7 @@ DESTDIR?= PREFIX ?= /usr/local CPPFLAGS= -I./common CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) LIBDIR ?= $(PREFIX)/lib diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 7b5708c42..dbcf62ba4 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -61,7 +61,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 6 -#define ZSTD_VERSION_RELEASE 1 +#define ZSTD_VERSION_RELEASE 2 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str diff --git a/programs/Makefile b/programs/Makefile index 70434cedc..682927192 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -31,22 +31,11 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -# Version numbers -LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h` -LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h` -LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h` -LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) -LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) -LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) -LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) -VERSION?= $(LIBVER) - DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I../lib/common -DZSTD_VERSION=\"$(VERSION)\" +CPPFLAGS= -I../lib/common CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial -CFLAGS += -std=c99 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) BINDIR = $(PREFIX)/bin diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 310db7f88..3bb4db70a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -133,6 +133,7 @@ static int usage_advanced(const char* programName) DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); #endif #ifndef ZSTD_NODICT + DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); DISPLAY( "--train : create a dictionary from a training set of files \n"); DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName); @@ -140,6 +141,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); #endif #ifndef ZSTD_NOBENCH + DISPLAY( "\n"); DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); From 45f84ab7ccbe4dc0ae7308a1b375fd3becfc5fe4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 20 May 2016 12:34:40 +0200 Subject: [PATCH 023/208] all programs use `ZSTD_VERSION_STRING` from `zstd.h` to display their version --- lib/compress/fse_compress.c | 6 ++---- programs/fullbench.c | 7 ++----- programs/fuzzer.c | 8 ++------ programs/paramgrill.c | 5 +---- programs/zbufftest.c | 6 +----- 5 files changed, 8 insertions(+), 24 deletions(-) diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index dad0be794..cde30acff 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -111,8 +111,6 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned U32 highThreshold = tableSize-1; /* CTable header */ - - tableU16[-2] = (U16) tableLog; tableU16[-1] = (U16) maxSymbolValue; @@ -147,10 +145,10 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned } /* Build table */ - { U32 u; for (u=0; u /* clock_t, clock, CLOCKS_PER_SEC */ #include "mem.h" -#include "zstd_static.h" +#include "zstd_static.h" /* ZSTD_VERSION_STRING */ #include "fse_static.h" #include "zbuff.h" #include "datagen.h" @@ -42,11 +42,8 @@ * Constants **************************************/ #define PROGRAM_DESCRIPTION "Zstandard speed analyzer" -#ifndef ZSTD_VERSION -# define ZSTD_VERSION "" -#endif #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__ +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR, __DATE__ #define KB *(1<<10) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 5bf532e1a..e664bf433 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -40,7 +40,7 @@ #include /* timeb */ #include /* strcmp */ #include /* clock_t */ -#include "zstd_static.h" +#include "zstd_static.h" /* ZSTD_VERSION_STRING */ #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" /* XXH64 */ #include "mem.h" @@ -49,10 +49,6 @@ /*-************************************ * Constants **************************************/ -#ifndef ZSTD_VERSION -# define ZSTD_VERSION "" -#endif - #define KB *(1U<<10) #define MB *(1U<<20) #define GB *(1U<<30) @@ -805,7 +801,7 @@ int main(int argc, const char** argv) } } } } /* for (argNb=1; argNb Date: Fri, 20 May 2016 14:36:36 +0200 Subject: [PATCH 024/208] huff0 dynamic reduction --- lib/common/bitstream.h | 6 +- lib/common/fse_static.h | 17 ++-- lib/common/huf.h | 1 + lib/common/huf_static.h | 29 +++--- lib/compress/fse_compress.c | 11 ++- lib/compress/huf_compress.c | 157 +++++++++++++++++--------------- lib/decompress/huf_decompress.c | 58 ++++++------ 7 files changed, 153 insertions(+), 126 deletions(-) diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h index 48021dfaf..694df4e3b 100644 --- a/lib/common/bitstream.h +++ b/lib/common/bitstream.h @@ -162,14 +162,12 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val) # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; - unsigned r; v |= v >> 1; v |= v >> 2; v |= v >> 4; v |= v >> 8; v |= v >> 16; - r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; - return r; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; # endif } @@ -375,7 +373,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ return BIT_DStream_overflow; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { diff --git a/lib/common/fse_static.h b/lib/common/fse_static.h index 0661dbd3e..797258f75 100644 --- a/lib/common/fse_static.h +++ b/lib/common/fse_static.h @@ -64,19 +64,22 @@ extern "C" { * FSE advanced API *******************************************/ size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); -/* same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */ +/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); -/* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */ +/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); -/* build a fake FSE_CTable, designed to compress always the same symbolValue */ +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); -/* build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */ +/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); -/* build a fake FSE_DTable, designed to always generate the same symbolValue */ +/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ /* ***************************************** @@ -103,7 +106,7 @@ static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsig static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); -/*! +/**< These functions are inner components of FSE_compress_usingCTable(). They allow the creation of custom streams, mixing multiple tables and bit sources. @@ -163,7 +166,7 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); -/*! +/**< Let's now decompose FSE_decompress_usingDTable() into its unitary components. You will decode FSE-encoded symbols from the bitStream, and also any other bitFields you put in, **in reverse order**. diff --git a/lib/common/huf.h b/lib/common/huf.h index d07080b15..a06fd3e18 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -77,6 +77,7 @@ HUF_decompress() : /* **************************************** * Tool functions ******************************************/ +#define HUF_BLOCKSIZE_MAX (128 * 1024) size_t HUF_compressBound(size_t size); /**< maximum compressed size */ /* Error Management */ diff --git a/lib/common/huf_static.h b/lib/common/huf_static.h index e68ec33dd..ea3eb62b8 100644 --- a/lib/common/huf_static.h +++ b/lib/common/huf_static.h @@ -85,15 +85,17 @@ size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cS /*! HUF_compress() does the following: 1. count symbol occurrence from source[] into table count[] using FSE_count() -2. build Huffman table from count using HUF_buildCTable() -3. save Huffman table to memory buffer using HUF_writeCTable() -4. encode the data stream using HUF_compress4X_usingCTable() +2. (optional) refine tableLog using HUF_optimalTableLog() +3. build Huffman table from count using HUF_buildCTable() +4. save Huffman table to memory buffer using HUF_writeCTable() +5. encode the data stream using HUF_compress4X_usingCTable() The following API allows targeting specific sub-functions for advanced tasks. For example, it's possible to compress several blocks using the same 'CTable', or to save and regenerate 'CTable' using external methods. */ /* FSE_count() : find it within "fse.h" */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); @@ -137,16 +139,19 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* sr /* ************************************************************** * Constants ****************************************************************/ -#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */ -#define HUF_MAX_SYMBOL_VALUE 255 -#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG) -# error "HUF_MAX_TABLELOG is too large !" +#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */ +#define HUF_SYMBOLVALUE_MAX 255 +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" #endif +/* ************************************************************** +* Needed by zstd in both compression and decompression +****************************************************************/ /*! HUF_readStats() : Read compact Huffman tree, saved by HUF_writeCTable(). `huffWeight` is destination buffer. @@ -188,17 +193,17 @@ MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, } /* collect weight stats */ - memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); + memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); weightTotal = 0; { U32 n; for (n=0; n= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } /* get last non-null symbol weight (implied, total must be 2^n) */ { U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); *tableLogPtr = tableLog; /* determine last weight */ { U32 const total = 1 << tableLog; diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index cde30acff..f1068433a 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -442,7 +442,7 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) return (FSE_CTable*)malloc(size); } -void FSE_freeCTable (FSE_CTable* ct) { free(ct); } +void FSE_freeCTable (FSE_CTable* ct) { free(ct); } /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) @@ -453,9 +453,9 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) return minBits; } -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) { - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - 2; + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; U32 tableLog = maxTableLog; U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; @@ -466,6 +466,11 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS return tableLog; } +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + /* Secondary normalization method. To be used when primary method fails. */ diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 695b66dae..a1004fba3 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -64,7 +64,7 @@ #include /* printf (debug) */ #include "huf_static.h" #include "bitstream.h" -#include "fse.h" /* header compression */ +#include "fse_static.h" /* header compression */ /* ************************************************************** @@ -73,6 +73,15 @@ #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +/* ************************************************************** +* Utils +****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + + /* ******************************************************* * HUF : Huffman block compression *********************************************************/ @@ -94,14 +103,14 @@ typedef struct nodeElt_s { size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) { - BYTE bitsToWeight[HUF_MAX_TABLELOG + 1]; - BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; U32 n; BYTE* op = (BYTE*)dst; size_t size; /* check conditions */ - if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1) + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX + 1) return ERROR(GENERIC); /* convert to weight */ @@ -159,19 +168,19 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize) { - BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ U32 tableLog = 0; size_t readSize; U32 nbSymbols = 0; //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ /* get symbol weights */ - readSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize); + readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(readSize)) return readSize; /* check result */ - if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); /* Prepare base value per rank */ @@ -189,12 +198,12 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si }} /* fill val */ - { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; - U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; + { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; { U32 n; for (n=0; n0; n--) { + U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) { valPerRank[n] = min; /* get starting value within each rank */ min += nbPerRank[n]; min >>= 1; @@ -229,7 +238,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) /* repay normalized cost */ { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_MAX_TABLELOG+1]; + U32 rankLast[HUF_TABLELOG_MAX+1]; int pos; /* Get pos of last (smallest) symbol per rank */ @@ -253,7 +262,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) if (highTotal <= lowTotal) break; } } /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ nBitsToDecrease ++; totalCost -= 1 << (nBitsToDecrease-1); if (rankLast[nBitsToDecrease-1] == noSymbol) @@ -312,10 +321,10 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) } -#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1) +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) { - nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1]; + nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1]; nodeElt* huffNode = huffNode0 + 1; U32 n, nonNullRank; int lowS, lowN; @@ -323,8 +332,8 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 U32 nodeRoot; /* safety checks */ - if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; - if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); memset(huffNode0, 0, sizeof(huffNode0)); /* sort, decreasing order */ @@ -360,9 +369,9 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); /* fill result into tree (val, nbBits) */ - { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; - U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; - if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */ + { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ for (n=0; n<=nonNullRank; n++) nbPerRank[huffNode[n].nbBits]++; /* determine stating value per rank */ @@ -391,10 +400,10 @@ size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } #define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) #define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream) + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) #define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream) + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) { @@ -441,44 +450,47 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) { - size_t segmentSize = (srcSize+3)/4; /* first 3 segments */ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; BYTE* op = ostart; - size_t errorCode; if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ if (srcSize < 12) return 0; /* no saving possible : too small input */ op += 6; /* jumpTable */ - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart, (U16)errorCode); + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart+2, (U16)errorCode); + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart+4, (U16)errorCode); + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + op += cSize; + } - op += errorCode; return op-ostart; } @@ -493,43 +505,46 @@ static size_t HUF_compress_internal ( BYTE* const oend = ostart + dstSize; BYTE* op = ostart; - U32 count[HUF_MAX_SYMBOL_VALUE+1]; - HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1]; - size_t errorCode; + U32 count[HUF_SYMBOLVALUE_MAX+1]; + HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1]; /* checks & inits */ - if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */ - if (dstSize < 1) return 0; /* not compressible within dst budget */ - if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */ - if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE; - if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG; + if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ + if (!dstSize) return 0; /* cannot fit within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; /* Scan input and build symbol stats */ - errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } - if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */ + { size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); + if (HUF_isError(largest)) return largest; + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* rle */ + if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ + } /* Build Huffman Tree */ - errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog); - if (HUF_isError(errorCode)) return errorCode; - huffLog = (U32)errorCode; + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog); + if (HUF_isError(maxBits)) return maxBits; + huffLog = (U32)maxBits; + } /* Write table description header */ - errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */ - op += errorCode; + { size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); + if (HUF_isError(hSize)) return hSize; + if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ + op += hSize; + } /* Compress */ - if (singleStream) - errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */ - else - errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - op += errorCode; + { size_t const cSize = (singleStream) ? + HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */ + HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; /* uncompressible */ + op += cSize; + } /* check compressibility */ if ((size_t)(op-ostart) >= srcSize-1) @@ -556,5 +571,5 @@ size_t HUF_compress2 (void* dst, size_t dstSize, size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG); + return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT); } diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index e42f76300..01e9c07b2 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -92,8 +92,8 @@ typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) { - BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ U32 tableLog = 0; size_t iSize; U32 nbSymbols = 0; @@ -105,7 +105,7 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ @@ -148,7 +148,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) #define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) #define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ @@ -203,7 +203,7 @@ size_t HUF_decompress1X2_usingDTable( size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG); + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); @@ -312,7 +312,7 @@ size_t HUF_decompress4X2_usingDTable( size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG); + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); @@ -335,7 +335,7 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co U32 nbBitsBaseline, U16 baseSeq) { HUF_DEltX4 DElt; - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* get pre-calculated rankVal */ memcpy(rankVal, rankValOrigin, sizeof(rankVal)); @@ -369,14 +369,14 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co }} } -typedef U32 rankVal_t[HUF_ABSOLUTEMAX_TABLELOG][HUF_ABSOLUTEMAX_TABLELOG + 1]; +typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1]; static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, const sortedSymbol_t* sortedList, const U32 sortedListSize, const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline) { - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ const U32 minBits = nbBitsBaseline - maxWeight; U32 s; @@ -415,10 +415,10 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) { - BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1]; - sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1]; - U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 }; - U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 }; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; + U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; U32* const rankStart = rankStart0+1; rankVal_t rankVal; U32 tableLog, maxW, sizeOfSort, nbSymbols; @@ -428,10 +428,10 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1; HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ - if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); + if (memLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ @@ -517,7 +517,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) #define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) #define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ @@ -580,7 +580,7 @@ size_t HUF_decompress1X4_usingDTable( size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG); + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize); @@ -688,7 +688,7 @@ size_t HUF_decompress4X4_usingDTable( size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG); + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; size_t hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize); @@ -716,7 +716,7 @@ static void HUF_fillDTableX6LevelN(HUF_DDescX6* DDescription, HUF_DSeqX6* DSeque const int scaleLog = nbBitsBaseline - sizeLog; /* note : targetLog >= (nbBitsBaseline-1), hence scaleLog <= 1 */ const int minBits = nbBitsBaseline - maxWeight; const U32 level = DDesc.nbBytes; - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; U32 symbolStartPos, s; /* local rankVal, will be modified */ @@ -766,20 +766,20 @@ static void HUF_fillDTableX6LevelN(HUF_DDescX6* DDescription, HUF_DSeqX6* DSeque /* note : same preparation as X4 */ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) { - BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1]; - sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1]; - U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 }; - U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 }; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; + U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; U32* const rankStart = rankStart0+1; U32 tableLog, maxW, sizeOfSort, nbSymbols; rankVal_t rankVal; const U32 memLog = DTable[0]; size_t iSize; - if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); + if (memLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ @@ -838,7 +838,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) DDesc.nbBits = 0; DDesc.nbBytes = 0; HUF_fillDTableX6LevelN(DDescription, DSequence, memLog, - (const U32 (*)[HUF_ABSOLUTEMAX_TABLELOG + 1])rankVal, 0, 1, maxW, + (const U32 (*)[HUF_TABLELOG_ABSOLUTEMAX + 1])rankVal, 0, 1, maxW, sortedSymbol, sizeOfSort, rankStart0, tableLog+1, DSeq, DDesc); } @@ -879,7 +879,7 @@ static U32 HUF_decodeLastSymbolsX6(void* op, U32 const maxL, BIT_DStream_t* DStr ptr += HUF_decodeSymbolX6(ptr, DStreamPtr, dd, ds, dtLog) #define HUF_DECODE_SYMBOLX6_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr) #define HUF_DECODE_SYMBOLX6_2(ptr, DStreamPtr) \ @@ -939,7 +939,7 @@ size_t HUF_decompress1X6_usingDTable( size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_MAX_TABLELOG); + HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize); @@ -1068,7 +1068,7 @@ size_t HUF_decompress4X6_usingDTable( size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_MAX_TABLELOG); + HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize); From 35b891c5b02608ddb09b5765104c52bf0819745d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 20 May 2016 19:42:20 +0200 Subject: [PATCH 025/208] zstd_stats.h updated to v0.6 --- lib/common/zstd_internal.h | 14 +--- lib/compress/.debug/zstd_stats.h | 135 ++++++++++++++++--------------- lib/compress/zstd_compress.c | 11 ++- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 4c9a7615a..8d9883d0f 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -186,6 +186,8 @@ MEM_STATIC unsigned ZSTD_highbit(U32 val) /*-******************************************* * Private interfaces *********************************************/ +typedef struct ZSTD_stats_s ZSTD_stats_t; + typedef struct { U32 off; U32 len; @@ -199,16 +201,6 @@ typedef struct { U32 rep[ZSTD_REP_INIT]; } ZSTD_optimal_t; -#if ZSTD_OPT_DEBUG == 3 - #include ".debug/zstd_stats.h" -#else - typedef struct { U32 unused; } ZSTD_stats_t; - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif - typedef struct { void* buffer; U32* offsetStart; @@ -245,7 +237,7 @@ typedef struct { U32 cachedPrice; U32 cachedLitLength; const BYTE* cachedLiterals; - ZSTD_stats_t stats; + ZSTD_stats_t* stats; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index d0189f872..51406a930 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -37,31 +37,21 @@ extern "C" { #endif -/*-************************************* -* Dependencies -***************************************/ -//#include "zstd.h" -//#include "mem.h" - - -/*-************************************* -* Constants -***************************************/ -//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#if ZSTD_OPT_DEBUG == 3 /*-************************************* * Types ***************************************/ -typedef struct { - U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength; +struct ZSTD_stats_s { + U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength; U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum; U32 litSum, matchLengthSum, litLengthSum, offCodeSum; - U32 matchLengthFreq[1<totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); - printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); - printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8); + printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); + printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8); } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1; - stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0; + stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { unsigned u; - stats->litSum = (1<litLengthSum = (1<matchLengthSum = (1<offCodeSum = (1<litSum = (2<litLengthSum = MaxLL+1; + stats->matchLengthSum = MaxML+1; + stats->offCodeSum = (MaxOff+1); for (u=0; u<=MaxLit; u++) stats->litFreq[u] = 1; @@ -99,62 +91,75 @@ MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) stats->offCodeFreq[u] = 1; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { - /* offset */ - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; - stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); - stats->priceOffset += (offCode-1) + (!offCode); - - /* match Length */ - stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); - stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1); - - if (litLength) { - /* literals */ - U32 u; - stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); - for (u=0; u < litLength; u++) - stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); - - /* literal Length */ - stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1); - } else { - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); - } - - - if (offset == 0) stats->totalRepSum++; - stats->totalSeqSum++; - stats->totalMatchSum += matchLength; - stats->totalLitSum += litLength; - U32 u; /* literals */ + stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); + for (u=0; u < litLength; u++) + stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); stats->litSum += litLength; for (u=0; u < litLength; u++) stats->litFreq[literals[u]]++; /* literal Length */ - stats->litLengthSum++; - if (litLength >= MaxLL) - stats->litLengthFreq[MaxLL]++; - else - stats->litLengthFreq[litLength]++; + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + if (litLength) { + stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1); + } else { + stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); + } + stats->litLengthFreq[llCode]++; + stats->litLengthSum++; + } /* match offset */ - stats->offCodeSum++; - stats->offCodeFreq[offCode]++; + { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); + stats->priceOffset += offCode; + stats->offCodeSum++; + stats->offCodeFreq[offCode]++; + } /* match Length */ - stats->matchLengthSum++; - if (matchLength >= MaxML) - stats->matchLengthFreq[MaxML]++; - else - stats->matchLengthFreq[matchLength]++; + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1); + stats->matchLengthFreq[mlCode]++; + stats->matchLengthSum++; + } + + if (offset == 0) stats->totalRepSum++; + stats->totalSeqSum++; + stats->totalMatchSum += matchLength; + stats->totalLitSum += litLength; } +#else + struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } +#endif // #if ZSTD_OPT_DEBUG == 3 #if defined (__cplusplus) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1bb75c68c..988a74fb7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -57,6 +57,7 @@ #include "fse_static.h" #include "huf_static.h" #include "zstd_internal.h" +#include ".debug/zstd_stats.h" /*-************************************* @@ -856,7 +857,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode); + ZSTD_statsUpdatePrices(seqStorePtr->stats, litLength, literals, offsetCode, matchCode); /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); @@ -1748,7 +1749,7 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; - ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); + ZSTD_statsUpdatePrices(seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -2020,8 +2021,9 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = &zc->seqStore.stats; - + ZSTD_stats_t* stats = malloc(sizeof(ZSTD_stats_t)); + if (!stats) return ERROR(memory_allocation); + zc->seqStore.stats = stats; ZSTD_statsInit(stats); while (remaining) { @@ -2059,6 +2061,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); + free(stats); return op-ostart; } From 04773ac90a2a697961b98fe173fac4d351dd0063 Mon Sep 17 00:00:00 2001 From: "Peter (Stig) Edwards" Date: Sat, 21 May 2016 12:15:48 +0100 Subject: [PATCH 026/208] Minor changes needed for OpenVMS Define util sleep functions on VMS. If PRIO_PROCESS is not defined then do not attempt to use it in the definition of SET_HIGH_PRIORITY. --- programs/util.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/programs/util.h b/programs/util.h index 44ab43488..f8b4dcb6c 100644 --- a/programs/util.h +++ b/programs/util.h @@ -90,11 +90,15 @@ extern "C" { # define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) # include # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ -# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# if defined(PRIO_PROCESS) +# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# else +# define SET_HIGH_PRIORITY /* disabled */ +# endif # define UTIL_sleep(s) sleep(s) # if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L) # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } From dca60f29fa98fd72935d5c74a1da0bcd1b5ab9e1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 23 May 2016 14:23:55 +0200 Subject: [PATCH 027/208] Added : new cross-version test : check new versions can decode older ones --- .travis.yml | 1 + versionsTest/.gitignore | 3 + versionsTest/Makefile | 39 ++++++++ versionsTest/test-zstd-versions.py | 153 +++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+) create mode 100644 versionsTest/.gitignore create mode 100644 versionsTest/Makefile create mode 100644 versionsTest/test-zstd-versions.py diff --git a/.travis.yml b/.travis.yml index 536b5a845..f583a803d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ env: - ZSTD_TRAVIS_CI_ENV=test - ZSTD_TRAVIS_CI_ENV="-C programs test32" - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" + - ZSTD_TRAVIS_CI_ENV="-C versionsTest" - ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=asan - ZSTD_TRAVIS_CI_ENV=asan32 diff --git a/versionsTest/.gitignore b/versionsTest/.gitignore new file mode 100644 index 000000000..bdb2cbdff --- /dev/null +++ b/versionsTest/.gitignore @@ -0,0 +1,3 @@ +# Tmp test directory +zstdtest + diff --git a/versionsTest/Makefile b/versionsTest/Makefile new file mode 100644 index 000000000..420b81e9f --- /dev/null +++ b/versionsTest/Makefile @@ -0,0 +1,39 @@ +# ########################################################################## +# Zstd tests - Makefile +# Based on LZ4 version test, by Takayuki Matsuoka - 2015-2016 +# +# GPL v2 License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# You can contact the author at : +# - ZSTD homepage : http://www.zstd.net/ +# ########################################################################## +# versionstest : Compatibility test between zstd versions stored on Github (v0.1+) +# ########################################################################## + +PYTHON?= python3 +TESTDIR := zstdtest + +default: all + +all: versionsTest + +versionsTest: + $(PYTHON) test-zstd-versions.py + +clean: + @rm -fR $(TESTDIR) + @echo Cleaning completed diff --git a/versionsTest/test-zstd-versions.py b/versionsTest/test-zstd-versions.py new file mode 100644 index 000000000..744d30dca --- /dev/null +++ b/versionsTest/test-zstd-versions.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 + +# Based on LZ4 version test script, by Takayuki Matsuoka + +import glob +import subprocess +import filecmp +import os +import shutil +import sys +import hashlib + +repo_url = 'https://github.com/Cyan4973/zstd.git' +tmp_dir_name = 'versionsTest/zstdtest' +make_cmd = 'make' +git_cmd = 'git' +test_dat_src = 'README.md' +test_dat = 'test_dat' +head = 'vdevel' + +def proc(cmd_args, pipe=True, dummy=False): + if dummy: + return + if pipe: + subproc = subprocess.Popen(cmd_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + subproc = subprocess.Popen(cmd_args) + return subproc.communicate() + +def make(args, pipe=True): + return proc([make_cmd] + args, pipe) + +def git(args, pipe=True): + return proc([git_cmd] + args, pipe) + +def get_git_tags(): + stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]']) + tags = stdout.decode('utf-8').split() + return tags + +def compress_sample(tag, sample): + print(tag) + if subprocess.call(['./zstd.' + tag, '-f' , sample])==0: # for some reason, compressed file is not created (or is overwritten ?) when pipe=True + os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst') + if subprocess.call(['./zstd.' + tag, '-5f' , sample])==0: + os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst') + if subprocess.call(['./zstd.' + tag, '-9f' , sample])==0 : + os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst') + if subprocess.call(['./zstd.' + tag, '-15f', sample])==0 : + os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst') + if subprocess.call(['./zstd.' + tag, '-18f', sample])==0: + os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst') + # zstdFiles = glob.glob("*.zst*") + # print(zstdFiles) + +# http://stackoverflow.com/a/19711609/2132223 +def sha1_of_file(filepath): + with open(filepath, 'rb') as f: + return hashlib.sha1(f.read()).hexdigest() + +def remove_duplicates(): + list_of_zst = sorted(glob.glob('*.zst')) + for i, ref_zst in enumerate(list_of_zst): + if not os.path.isfile(ref_zst): + continue + for j in range(i+1, len(list_of_zst)): + compared_zst = list_of_zst[j] + if not os.path.isfile(compared_zst): + continue + if filecmp.cmp(ref_zst, compared_zst): + os.remove(compared_zst) + print('duplicated : {} == {}'.format(ref_zst, compared_zst)) + +def decompress_zst(tag): + dec_error = 0 + list_zst = sorted(glob.glob('*.zst')) + for file_zst in list_zst: + print(file_zst, end=" ") + print(tag, end=" ") + file_dec = file_zst + '_d64_' + tag + '.dec' + if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec])==0: + if not filecmp.cmp(file_dec, test_dat): + print('ERR !! ') + dec_error = 1 + else: + print('OK ') + return dec_error + +if __name__ == '__main__': + error_code = 0 + base_dir = os.getcwd() + '/..' # /path/to/zstd + tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/versionsTest/zstdtest + clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/versionsTest/zstdtest/zstd + programs_dir = base_dir + '/programs' # /path/to/zstd/programs + os.makedirs(tmp_dir, exist_ok=True) + + # since Travis clones limited depth, we should clone full repository + if not os.path.isdir(clone_dir): + git(['clone', repo_url, clone_dir]) + + shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat) + + # Retrieve all release tags + print('Retrieve all release tags :') + os.chdir(clone_dir) + tags = get_git_tags() + [head] + print(tags); + + # Build all release zstd + for tag in tags: + os.chdir(base_dir) + dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/zstd. + if not os.path.isfile(dst_zstd) or tag == head: + if tag != head: + r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/ + os.makedirs(r_dir, exist_ok=True) + os.chdir(clone_dir) + git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) + os.chdir(r_dir + '/programs') # /path/to/zstd/zstdtest//programs + make(['clean', 'zstd'], False) + else: + os.chdir(programs_dir) + make(['zstd'], False) + shutil.copy2('zstd', dst_zstd) + + # remove any remaining *.zst and *.dec from previous test + os.chdir(tmp_dir) + for compressed in glob.glob("*.zst"): + os.remove(compressed) + for dec in glob.glob("*.dec"): + os.remove(dec) + + print('Compress test.dat by all released zstd') + + error_code = 0; + for tag in tags: + print(tag) + compress_sample(tag, test_dat) + remove_duplicates() + error_code += decompress_zst(tag) + + print('') + print('Enumerate different compressed files') + zstds = sorted(glob.glob('*.zst')) + for zstd in zstds: + print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd)) + + if error_code != 0: + print('==== ERROR !!! =====') + + sys.exit(error_code) From 99b23ba1b77be51f6c49d2319760df94fdbb37fc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 23 May 2016 15:04:14 +0200 Subject: [PATCH 028/208] cleaner script output --- versionsTest/test-zstd-versions.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/versionsTest/test-zstd-versions.py b/versionsTest/test-zstd-versions.py index 744d30dca..f3359ba6e 100644 --- a/versionsTest/test-zstd-versions.py +++ b/versionsTest/test-zstd-versions.py @@ -41,16 +41,16 @@ def get_git_tags(): return tags def compress_sample(tag, sample): - print(tag) - if subprocess.call(['./zstd.' + tag, '-f' , sample])==0: # for some reason, compressed file is not created (or is overwritten ?) when pipe=True + from subprocess import DEVNULL + if subprocess.call(['./zstd.' + tag, '-f' , sample], stderr=DEVNULL)==0: os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-5f' , sample])==0: + if subprocess.call(['./zstd.' + tag, '-5f' , sample], stderr=DEVNULL)==0: os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-9f' , sample])==0 : + if subprocess.call(['./zstd.' + tag, '-9f' , sample], stderr=DEVNULL)==0 : os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-15f', sample])==0 : + if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL)==0 : os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-18f', sample])==0: + if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL)==0: os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst') # zstdFiles = glob.glob("*.zst*") # print(zstdFiles) @@ -76,16 +76,19 @@ def remove_duplicates(): def decompress_zst(tag): dec_error = 0 list_zst = sorted(glob.glob('*.zst')) + from subprocess import DEVNULL for file_zst in list_zst: print(file_zst, end=" ") print(tag, end=" ") file_dec = file_zst + '_d64_' + tag + '.dec' - if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec])==0: + if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL)==0: if not filecmp.cmp(file_dec, test_dat): print('ERR !! ') dec_error = 1 else: print('OK ') + else: + print('command does not work') return dec_error if __name__ == '__main__': From da4fe741b8fcb8fc9c33e0b4047e949966a6d800 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 23 May 2016 15:43:17 +0200 Subject: [PATCH 029/208] more portable DEVNULL definition --- versionsTest/test-zstd-versions.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/versionsTest/test-zstd-versions.py b/versionsTest/test-zstd-versions.py index f3359ba6e..b18c3d3c2 100644 --- a/versionsTest/test-zstd-versions.py +++ b/versionsTest/test-zstd-versions.py @@ -41,7 +41,10 @@ def get_git_tags(): return tags def compress_sample(tag, sample): - from subprocess import DEVNULL + try: + from subprocess import DEVNULL # py3k + except ImportError: + DEVNULL = open(os.devnull, 'wb') if subprocess.call(['./zstd.' + tag, '-f' , sample], stderr=DEVNULL)==0: os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst') if subprocess.call(['./zstd.' + tag, '-5f' , sample], stderr=DEVNULL)==0: @@ -76,7 +79,10 @@ def remove_duplicates(): def decompress_zst(tag): dec_error = 0 list_zst = sorted(glob.glob('*.zst')) - from subprocess import DEVNULL + try: + from subprocess import DEVNULL # py3k + except ImportError: + DEVNULL = open(os.devnull, 'wb') for file_zst in list_zst: print(file_zst, end=" ") print(tag, end=" ") @@ -91,6 +97,7 @@ def decompress_zst(tag): print('command does not work') return dec_error + if __name__ == '__main__': error_code = 0 base_dir = os.getcwd() + '/..' # /path/to/zstd From 50e82c015d56c00879bb8d2d551d117b840a9804 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 15:49:09 +0200 Subject: [PATCH 030/208] added ZSTD_createCCtx_advanced --- lib/common/zstd_static.h | 9 ++++++++ lib/compress/zstd_compress.c | 45 +++++++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index ef42b0a97..912eb1335 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -94,10 +94,19 @@ typedef struct { ZSTD_frameParameters fParams; } ZSTD_parameters; +typedef void* (*ZSTD_allocFunction) (size_t size); +typedef void (*ZSTD_freeFunction) (void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem; + /*-************************************* * Advanced functions ***************************************/ +/*! ZSTD_createCCtx_advanced() : + * Create ZSTD context using external alloc and free functions */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); + + ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getCParams() : diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 988a74fb7..00106f5fe 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -104,6 +104,8 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -118,13 +120,32 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - return (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + ZSTD_CCtx* ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + ctx->customAlloc = malloc; + ctx->customFree = free; + return ctx; +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + if (!customMem.customAlloc || !customMem.customFree) + return ZSTD_createCCtx(); + + ZSTD_CCtx* ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + memset(ctx, 0, sizeof(ZSTD_CCtx)); + ctx->customAlloc = customMem.customAlloc; + ctx->customFree = customMem.customFree; + return ctx; } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - free(cctx->workSpace); - free(cctx); + cctx->customFree(cctx->workSpace); + cctx->customFree(cctx); return 0; /* reserved as a potential error code in the future */ } @@ -229,8 +250,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); if (zc->workSpaceSize < neededSpace) { - free(zc->workSpace); - zc->workSpace = malloc(neededSpace); + zc->customFree(zc->workSpace); + zc->workSpace = zc->customAlloc(neededSpace); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; } } @@ -2021,7 +2042,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = malloc(sizeof(ZSTD_stats_t)); + ZSTD_stats_t* stats = (ZSTD_stats_t*) zc->customAlloc(sizeof(ZSTD_stats_t)); if (!stats) return ERROR(memory_allocation); zc->seqStore.stats = stats; ZSTD_statsInit(stats); @@ -2030,7 +2051,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { zc->customFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2041,11 +2062,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; + if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; + if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2061,7 +2082,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - free(stats); + zc->customFree(stats); return op-ostart; } @@ -2427,8 +2448,10 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); + ctxBody.customAlloc = malloc; + ctxBody.customFree = free; result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); - free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ + ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } From 107e2431955a7bb0d78f31d74bddd880f2250f75 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 16:24:52 +0200 Subject: [PATCH 031/208] added ZSTD_createDCtx_advanced --- lib/common/zstd_static.h | 5 ++++- lib/compress/zstd_compress.c | 27 +++++++++++++++++---------- lib/decompress/zstd_decompress.c | 31 ++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index 912eb1335..e0f50cf40 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -103,9 +103,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } * Advanced functions ***************************************/ /*! ZSTD_createCCtx_advanced() : - * Create ZSTD context using external alloc and free functions */ + * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API unsigned ZSTD_maxCLevel (void); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 00106f5fe..966d9acbd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -120,20 +120,25 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - ZSTD_CCtx* ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); - if (!ctx) return NULL; - - ctx->customAlloc = malloc; - ctx->customFree = free; - return ctx; + ZSTD_customMem customMem = { NULL, NULL }; + return ZSTD_createCCtx_advanced(customMem); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { - if (!customMem.customAlloc || !customMem.customFree) - return ZSTD_createCCtx(); + ZSTD_CCtx* ctx; - ZSTD_CCtx* ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + if (!customMem.customAlloc || !customMem.customFree) + { + ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + ctx->customAlloc = malloc; + ctx->customFree = free; + return ctx; + } + + ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); if (!ctx) return NULL; memset(ctx, 0, sizeof(ZSTD_CCtx)); @@ -307,7 +312,9 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { if (srcCCtx->stage!=1) return ERROR(stage_wrong); - dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ + dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ + dstCCtx->customAlloc = srcCCtx->customAlloc; + dstCCtx->customFree = srcCCtx->customFree; ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 177e9c8a2..a3eee709e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -116,6 +116,8 @@ struct ZSTD_DCtx_s size_t expected; size_t headerSize; ZSTD_frameParams fParams; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 flagRepeatTable; @@ -143,15 +145,38 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) ZSTD_DCtx* ZSTD_createDCtx(void) { - ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx)); - if (dctx==NULL) return NULL; + ZSTD_customMem customMem = { NULL, NULL }; + return ZSTD_createDCtx_advanced(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_DCtx* dctx; + + if (!customMem.customAlloc || !customMem.customFree) + { + dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); + if (!dctx) return NULL; + dctx->customAlloc = malloc; + dctx->customFree = free; + + ZSTD_decompressBegin(dctx); + return dctx; + } + + dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); + if (!dctx) return NULL; + dctx->customAlloc = customMem.customAlloc; + dctx->customFree = customMem.customFree; + ZSTD_decompressBegin(dctx); return dctx; } + size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { - free(dctx); + dctx->customFree(dctx); return 0; /* reserved as a potential error code in the future */ } From 75424d11390e3d1cae616b3afc22d6c1fab1361a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 23 May 2016 16:56:56 +0200 Subject: [PATCH 032/208] Added : Sparse write support --[no-]sparse command --- programs/fileio.c | 90 +++++++++++++++++++++++++++++++++++++++++++--- programs/fileio.h | 3 +- programs/zstdcli.c | 3 ++ 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b3f324ac6..cdfe6a3dc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -130,6 +130,8 @@ static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } static U32 g_maxWLog = 23; void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; } +static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ +void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } /*-************************************* @@ -178,6 +180,10 @@ static FILE* FIO_openDstFile(const char* dstFileName) DISPLAYLEVEL(4,"Using stdout for output\n"); f = stdout; SET_BINARY_MODE(stdout); + if (g_sparseFileSupport==1) { + g_sparseFileSupport = 0; + DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); + } } else { if (!g_overwrite) { /* Check if destination file already exists */ f = fopen( dstFileName, "rb" ); @@ -189,8 +195,7 @@ static FILE* FIO_openDstFile(const char* dstFileName) return 0; } DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName); - { - int ch = getchar(); + { int ch = getchar(); if ((ch!='Y') && (ch!='y')) { DISPLAY(" not overwritten \n"); return 0; @@ -512,6 +517,81 @@ static void FIO_freeDResources(dRess_t ress) } +/** FIO_fwriteSparse() : +* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ +static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + size_t bufferSizeT = bufferSize / sizeof(size_t); + const size_t* const bufferTEnd = bufferT + bufferSizeT; + const size_t* ptrT = bufferT; + static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ + + if (!g_sparseFileSupport) { /* normal write */ + size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) { + int const seekResult = fseek(file, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) { + size_t seg0SizeT = segmentSizeT; + size_t nb0T; + + /* count leading zeros */ + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeof(size_t)); + + if (nb0T != seg0SizeT) { /* not all 0s */ + int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + seg0SizeT -= nb0T; + ptrT += nb0T; + { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); + if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); + } } + ptrT += seg0SizeT; + } + + { static size_t const maskT = sizeof(size_t)-1; + if (bufferSize & maskT) { /* size not multiple of sizeof(size_t) : implies end of block */ + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + size_t restSize = bufferSize & maskT; + const char* const restEnd = restStart + restSize; + for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) { + int seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); + } } } } + + return storedSkips; +} + +static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) +{ + if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */ + int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + { const char lastZeroByte[1] = { 0 }; + size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); + } } +} + /** FIO_decompressFrame() : @return : size of decoded frame */ @@ -520,6 +600,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, { U64 frameSize = 0; size_t readSize; + U32 storedSkips = 0; ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); @@ -538,8 +619,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, readSize -= inSize; /* Write block */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); } + storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips); frameSize += decodedSize; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); @@ -553,6 +633,8 @@ unsigned long long FIO_decompressFrame(dRess_t ress, EXM_THROW(35, "Read error"); } + FIO_fwriteSparseEnd(foutput, storedSkips); + return frameSize; } diff --git a/programs/fileio.h b/programs/fileio.h index d5aae449b..6e7912380 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -46,7 +46,8 @@ extern "C" { ***************************************/ void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); -void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ +void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ +void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3bb4db70a..3fdc558bd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -132,6 +132,7 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); #endif + DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); @@ -229,6 +230,8 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } + if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } + if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){ From 13ba880b4974817cca5962c0c94acbd1cc3aec16 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 17:04:23 +0200 Subject: [PATCH 033/208] added ZBUFF_createCCtx_advanced and ZBUFF_createDCtx_advanced --- lib/common/zbuff.h | 5 +++- lib/compress/zbuff_compress.c | 45 +++++++++++++++++++++++------- lib/compress/zstd_compress.c | 5 +++- lib/decompress/zbuff_decompress.c | 46 ++++++++++++++++++++++++------- lib/decompress/zstd_decompress.c | 5 +++- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 3caa39caf..4d7f12a1f 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -38,7 +38,8 @@ extern "C" { /* ************************************* * Dependencies ***************************************/ -#include /* size_t */ +#include /* size_t */ +#include "zstd_static.h" /* ZSTD_customMem */ /* *************************************************************** @@ -60,6 +61,7 @@ extern "C" { ***************************************/ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); @@ -112,6 +114,7 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 260aca087..cabf447a4 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -95,14 +95,39 @@ struct ZBUFF_CCtx_s { size_t outBuffContentSize; size_t outBuffFlushedSize; ZBUFF_cStage stage; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; }; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ ZBUFF_CCtx* ZBUFF_createCCtx(void) { - ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx)); + ZSTD_customMem customMem = { NULL, NULL }; + return ZBUFF_createCCtx_advanced(customMem); +} + +ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_CCtx* zbc; + + if (!customMem.customAlloc && !customMem.customFree) + { + zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx)); + if (zbc==NULL) return NULL; + zbc->customAlloc = malloc; + zbc->customFree = free; + zbc->zc = ZSTD_createCCtx(); + return zbc; + } + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; - memset(zbc, 0, sizeof(*zbc)); - zbc->zc = ZSTD_createCCtx(); + memset(zbc, 0, sizeof(ZBUFF_CCtx)); + zbc->customAlloc = customMem.customAlloc; + zbc->customFree = customMem.customFree; + zbc->zc = ZSTD_createCCtx_advanced(customMem); return zbc; } @@ -110,9 +135,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) { if (zbc==NULL) return 0; /* support free on NULL */ ZSTD_freeCCtx(zbc->zc); - free(zbc->inBuff); - free(zbc->outBuff); - free(zbc); + zbc->customFree(zbc->inBuff); + zbc->customFree(zbc->outBuff); + zbc->customFree(zbc); return 0; } @@ -127,16 +152,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zbc->inBuffSize < neededInBuffSize) { zbc->inBuffSize = neededInBuffSize; - free(zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)malloc(neededInBuffSize); + zbc->customFree(zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); } if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; - free(zbc->outBuff); /* should not be necessary */ - zbc->outBuff = (char*)malloc(zbc->outBuffSize); + zbc->customFree(zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize); if (zbc->outBuff == NULL) return ERROR(memory_allocation); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 966d9acbd..bca725f54 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -128,7 +128,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ZSTD_CCtx* ctx; - if (!customMem.customAlloc || !customMem.customFree) + if (!customMem.customAlloc && !customMem.customFree) { ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); if (!ctx) return NULL; @@ -138,6 +138,9 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) return ctx; } + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); if (!ctx) return NULL; diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 858b442f7..d4ca550a7 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -82,15 +82,41 @@ struct ZBUFF_DCtx_s { size_t blockSize; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ ZBUFF_DCtx* ZBUFF_createDCtx(void) { - ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); + ZSTD_customMem customMem = { NULL, NULL }; + return ZBUFF_createDCtx_advanced(customMem); +} + +ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_DCtx* zbd; + + if (!customMem.customAlloc && !customMem.customFree) + { + zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx)); + if (zbd==NULL) return NULL; + zbd->customAlloc = malloc; + zbd->customFree = free; + zbd->zd = ZSTD_createDCtx(); + zbd->stage = ZBUFFds_init; + return zbd; + } + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx)); if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(*zbd)); - zbd->zd = ZSTD_createDCtx(); + memset(zbd, 0, sizeof(ZBUFF_DCtx)); + zbd->customAlloc = customMem.customAlloc; + zbd->customFree = customMem.customFree; + zbd->zd = ZSTD_createDCtx_advanced(customMem); zbd->stage = ZBUFFds_init; return zbd; } @@ -99,9 +125,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) { if (zbd==NULL) return 0; /* support free on null */ ZSTD_freeDCtx(zbd->zd); - free(zbd->inBuff); - free(zbd->outBuff); - free(zbd); + zbd->customFree(zbd->inBuff); + zbd->customFree(zbd->outBuff); + zbd->customFree(zbd); return 0; } @@ -170,16 +196,16 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; if (zbd->inBuffSize < blockSize) { - free(zbd->inBuff); + zbd->customFree(zbd->inBuff); zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)malloc(blockSize); + zbd->inBuff = (char*)zbd->customAlloc(blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; if (zbd->outBuffSize < neededOutSize) { - free(zbd->outBuff); + zbd->customFree(zbd->outBuff); zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)malloc(neededOutSize); + zbd->outBuff = (char*)zbd->customAlloc(neededOutSize); if (zbd->outBuff == NULL) return ERROR(memory_allocation); } } } zbd->stage = ZBUFFds_read; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index a3eee709e..b670c548c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -153,7 +153,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; - if (!customMem.customAlloc || !customMem.customFree) + if (!customMem.customAlloc && !customMem.customFree) { dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; @@ -164,6 +164,9 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) return dctx; } + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; dctx->customAlloc = customMem.customAlloc; From 32990b5daeb0710b424caaa336911d5a0420bd89 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 23 May 2016 17:48:57 +0200 Subject: [PATCH 034/208] Added tests for Sparse mode support Fixed : complex cli arg case involving a mix of `stdin` and `-o` --- programs/playTests.sh | 27 +++++++++++++++++++++++++++ programs/zstdcli.c | 20 ++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/programs/playTests.sh b/programs/playTests.sh index 1c5f00a13..ee29bec06 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -75,6 +75,33 @@ echo "echo foo | $ZSTD | $ZSTD -d > /dev/full" echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" +echo "\n**** test sparse file support **** " + +./datagen -g5M -P100 > tmpSparse +$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen +diff -s tmpSparse tmpSparseRegen +$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse +diff -s tmpSparse tmpOutSparse +$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse +diff -s tmpSparse tmpOutNoSparse +ls -ls tmpSparse* +./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) +./datagen -s1 -g1200007 -P100 | diff -s - tmpSparseOdd +ls -ls tmpSparseOdd +echo "\n Sparse Compatibility with Console :" +echo "Hello World 1 !" | $ZSTD | $ZSTD -d -c +echo "Hello World 2 !" | $ZSTD | $ZSTD -d | cat +echo "\n Sparse Compatibility with Append :" +./datagen -P100 -g1M > tmpSparse1M +cat tmpSparse1M tmpSparse1M > tmpSparse2M +$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed +$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated +$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated +ls -ls tmpSparse* +diff tmpSparse2M tmpSparseRegenerated +# rm tmpSparse* + + echo "\n**** dictionary tests **** " ./datagen > tmpDict diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3fdc558bd..d09dbe440 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -176,7 +176,7 @@ static void waitEnter(void) int main(int argCount, const char** argv) { - int i, + int argNb, bench=0, decode=0, forceStdout=0, @@ -203,18 +203,21 @@ int main(int argCount, const char** argv) (void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } + filenameTable[0] = stdinmark; displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ - for (i = (int)strlen(programName); i > 0; i--) { if (programName[i] == '/') { i++; break; } } - programName += i; + { size_t pos; + for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } + programName += pos; + } /* preset behaviors */ if (!strcmp(programName, ZSTD_UNZSTD)) decode=1; if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } /* command switches */ - for(i=1; i use stdin and stdout */ - if(!filenameIdx) filenameIdx=1, filenameTable[0]=stdinmark, outFileName=stdoutmark; + filenameIdx += !filenameIdx; /*< default input is stdin */ + if (!strcmp(filenameTable[0], stdinmark) && !outFileName ) outFileName = stdoutmark; /*< when input is stdin, default output is stdout */ /* Check if input/output defined as console; trigger an error in this case */ if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); @@ -443,9 +447,9 @@ int main(int argCount, const char** argv) { /* decompression */ #ifndef ZSTD_NODECOMPRESS if (filenameIdx==1 && outFileName) - operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); + operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName); + operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName); #else DISPLAY("Decompression not supported\n"); #endif From de95f9646263e7a35ddbad806cc6fc3c83dc34d7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 23 May 2016 19:46:47 +0200 Subject: [PATCH 035/208] Pass-Through mode support (using `-df`), for compatibility with gzip --- NEWS | 4 ++++ programs/fileio.c | 31 ++++++++++++++++++++++++++++--- programs/playTests.sh | 16 +++++++++++----- programs/zstdcli.c | 5 +++-- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/NEWS b/NEWS index 0cf171b2c..c100700aa 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +v0.6.2 +New : Support for Sparse File-systems (do not use space for zero-filled sectors) +New : Support pass-through mode, when using `-df` + v0.6.1 New : zlib wrapper API, thanks to Przemyslaw Skibinski New : Ability to compile compressor / decompressor separately diff --git a/programs/fileio.c b/programs/fileio.c index cdfe6a3dc..da70936af 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -639,6 +639,28 @@ unsigned long long FIO_decompressFrame(dRess_t ress, } +/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode + @return : 0 (no error) */ +static unsigned FIO_passThrough(FILE* foutput, FILE* finput, unsigned char* buffer, size_t bufferSize) +{ + size_t const blockSize = MIN (64 KB, bufferSize); + size_t read = 1; + unsigned storedSkips = 0; + + /* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */ + { size_t const sizeCheck = fwrite(buffer, 1, 4, foutput); + if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); } + + while (read) { + read = fread(buffer, 1, blockSize, finput); + storedSkips = FIO_fwriteSparse(foutput, buffer, read, storedSkips); + } + + FIO_fwriteSparseEnd(foutput, storedSkips); + return 0; +} + + /** FIO_decompressSrcFile() : Decompression `srcFileName` into `ress.dstFile` @return : 0 : OK @@ -666,9 +688,12 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) } #endif if (magic != ZSTD_MAGICNUMBER) { - DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); - return 1; - } } + if (g_overwrite) /* -df : pass-through mode */ + return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + else { + DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); + return 1; + } } } filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead); } diff --git a/programs/playTests.sh b/programs/playTests.sh index ee29bec06..58455c0f9 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -24,6 +24,7 @@ roundTripTest() { echo "\n**** simple tests **** " + ./datagen > tmp $ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) @@ -47,9 +48,12 @@ $ZSTD -q tmp && die "overwrite check failed!" $ZSTD -q -f tmp $ZSTD -q --force tmp $ZSTD -df tmp && die "should have refused : wrong extension" -cp tmp tmp2.zst -$ZSTD -df tmp2.zst && die "should have failed : wrong format" -rm tmp2.zst + + +echo "\n**** Pass-Through mode **** " +echo "Hello world !" | $ZSTD -df +echo "Hello world !" | $ZSTD -dcf + echo "\n**** frame concatenation **** " @@ -63,8 +67,7 @@ $ZSTD -dc helloworld.zstd > result.tmp cat result.tmp sdiff helloworld.tmp result.tmp rm ./*.tmp ./*.zstd - -echo frame concatenation test completed +echo "frame concatenation tests completed" echo "\n**** flush write error test **** " @@ -136,7 +139,9 @@ ls -ls tmp* echo "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" + echo "\n**** integrity tests **** " + echo "test one file (tmp1.zst) " $ZSTD -t tmp1.zst $ZSTD --test tmp1.zst @@ -145,6 +150,7 @@ $ZSTD -t *.zst echo "test good and bad files (*) " $ZSTD -t * && die "bad files not detected !" + echo "\n**** zstd round-trip tests **** " roundTripTest diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d09dbe440..80f037868 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -279,7 +279,7 @@ int main(int argCount, const char** argv) case 'D': nextEntryIsDictionary = 1; argument++; break; /* Overwrite */ - case 'f': FIO_overwriteMode(); argument++; break; + case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; /* Verbose mode */ case 'v': displayLevel=4; argument++; break; @@ -422,7 +422,8 @@ int main(int argCount, const char** argv) /* Check if input/output defined as console; trigger an error in this case */ if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); - if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) CLEAN_RETURN(badusage(programName)); + if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode)) + CLEAN_RETURN(badusage(programName)); /* user-selected output filename, only possible with a single file */ if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) { From ddbb8e27bfc0a7890aadbf630d58d5676431380c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 24 May 2016 00:52:14 +0200 Subject: [PATCH 036/208] fixed minor conversion warning --- programs/fileio.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index da70936af..4367c2095 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -641,19 +641,19 @@ unsigned long long FIO_decompressFrame(dRess_t ress, /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @return : 0 (no error) */ -static unsigned FIO_passThrough(FILE* foutput, FILE* finput, unsigned char* buffer, size_t bufferSize) +static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize) { size_t const blockSize = MIN (64 KB, bufferSize); - size_t read = 1; + size_t readFromInput = 1; unsigned storedSkips = 0; /* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */ { size_t const sizeCheck = fwrite(buffer, 1, 4, foutput); if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); } - while (read) { - read = fread(buffer, 1, blockSize, finput); - storedSkips = FIO_fwriteSparse(foutput, buffer, read, storedSkips); + while (readFromInput) { + readFromInput = fread(buffer, 1, blockSize, finput); + storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips); } FIO_fwriteSparseEnd(foutput, storedSkips); From 71dd67cf57182c31686d1262b785512d2149c8ae Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 10:57:14 +0200 Subject: [PATCH 037/208] fullbench: tests for ZBUFF_createCCtx_advanced and ZBUFF_createDCtx_advanced --- lib/common/mem.h | 7 +++-- lib/compress/.debug/zstd_stats.h | 7 ++++- lib/compress/zstd_compress.c | 11 ++++--- programs/fullbench.c | 51 +++++++++++++++++++++++++++++++- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 5abd83d1a..7f5a2ef7c 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -42,8 +42,11 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ -#include /* size_t, ptrdiff_t */ -#include /* memcpy */ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +#endif /*-**************************************** diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index 51406a930..428514bfe 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -55,8 +55,11 @@ struct ZSTD_stats_s { /*-************************************* -* Advanced functions +* Stats functions ***************************************/ +MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc() { return malloc(sizeof(ZSTD_stats_t)); } +MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { free(stats); } + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); @@ -155,6 +158,8 @@ MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, co #else struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc(void) { return NULL; } + MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { (void)stats; } MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bca725f54..93efa99c9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2052,8 +2052,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = (ZSTD_stats_t*) zc->customAlloc(sizeof(ZSTD_stats_t)); - if (!stats) return ERROR(memory_allocation); + ZSTD_stats_t* stats = ZSTD_statsAlloc(); zc->seqStore.stats = stats; ZSTD_statsInit(stats); @@ -2061,7 +2060,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { zc->customFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { ZSTD_statsFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2072,11 +2071,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } + if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } + if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2092,7 +2091,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - zc->customFree(stats); + ZSTD_statsFree(stats); return op-ostart; } diff --git a/programs/fullbench.c b/programs/fullbench.c index 6f5d6a782..fe1abf1a2 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -213,6 +213,19 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, /*_******************************************************* * Bench functions *********************************************************/ +void* BMK_allocFunction(size_t size) +{ + void* address = malloc(size); + /* printf("alloc %p, %d \n", address, (int)size); */ + return address; +} + +void BMK_freeFunction(void* address) +{ +/* printf("free %p \n", address); */ + free(address); +} + static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { BYTE* dstBuff; @@ -220,6 +233,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) BYTE* buff2; const char* benchName; size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); + ZSTD_customMem customMem = { BMK_allocFunction, BMK_freeFunction }; double bestTime = 100000000.; /* Selection */ @@ -247,8 +261,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue"; break; case 42: + benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_createCCtx_advanced/ZBUFF_compressContinue"; + break; + case 43: benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue"; break; + case 44: + benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_createDCtx_advanced/ZBUFF_decompressContinue"; + break; default : return 0; } @@ -323,9 +343,16 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx(); break; case 42 : + if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx_advanced(customMem); + break; + case 43 : if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx(); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); break; + case 44 : + if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx_advanced(customMem); + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + break; /* test functions */ /* by convention, test functions can be added > 100 */ @@ -355,7 +382,29 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds; if (averageTime < bestTime) bestTime = averageTime; DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult); - }} + } + + /* free allocated structures */ + switch(benchNb) + { + case 11 : + if (g_zcc) { ZSTD_freeCCtx(g_zcc); g_zcc=NULL; } + break; + case 12 : + case 31: + case 32: + if (g_zdc) { ZSTD_freeDCtx(g_zdc); g_zdc=NULL; } + break; + case 41 : + case 42 : + if (g_zbcc) { ZBUFF_freeCCtx(g_zbcc); g_zbcc=NULL; } + break; + case 43 : + case 44 : + if (g_zbdc) { ZBUFF_freeDCtx(g_zbdc); g_zbdc=NULL; } + break; + default : ; + } } DISPLAY("%2u\n", benchNb); _cleanOut: From a1653fbfd05653c2c3f18741da925a5dade2f5b9 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 15:35:48 +0200 Subject: [PATCH 038/208] zbufftest.c: tests for ZBUFF_createCCtx_advanced/ZBUFF_createDCtx_advanced --- programs/zbufftest.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f8d527ef2..d7f7ac00d 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -131,7 +131,20 @@ static unsigned FUZ_highbit32(U32 v32) } */ -static int basicUnitTests(U32 seed, double compressibility) +void* ZBUFF_allocFunction(size_t size) +{ + void* address = malloc(size); + /* DISPLAYLEVEL(4, "alloc %p, %d \n", address, (int)size); */ + return address; +} + +void ZBUFF_freeFunction(void* address) +{ + /* if (address) DISPLAYLEVEL(4, "free %p \n", address); */ + free(address); +} + +static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) { int testResult = 0; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; @@ -143,8 +156,8 @@ static int basicUnitTests(U32 seed, double compressibility) U32 randState = seed; size_t result, cSize, readSize, genSize; U32 testNb=0; - ZBUFF_CCtx* zc = ZBUFF_createCCtx(); - ZBUFF_DCtx* zd = ZBUFF_createDCtx(); + ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); + ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); /* Create compressible test buffer */ if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { @@ -489,6 +502,8 @@ int main(int argc, const char** argv) int result=0; U32 mainPause = 0; const char* programName = argv[0]; + ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction }; + ZSTD_customMem customNULL = { NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Tue, 24 May 2016 15:36:37 +0200 Subject: [PATCH 039/208] zstd_stats.h included only in debug mode --- lib/common/zstd_internal.h | 12 +++++++- lib/compress/.debug/zstd_stats.h | 12 -------- lib/compress/zstd_compress.c | 15 ++++------ programs/fullbench.c | 51 +------------------------------- 4 files changed, 18 insertions(+), 72 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8d9883d0f..2eea5feee 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -201,6 +201,16 @@ typedef struct { U32 rep[ZSTD_REP_INIT]; } ZSTD_optimal_t; +#if ZSTD_OPT_DEBUG == 3 + #include ".debug/zstd_stats.h" +#else + struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } +#endif // #if ZSTD_OPT_DEBUG == 3 + typedef struct { void* buffer; U32* offsetStart; @@ -237,7 +247,7 @@ typedef struct { U32 cachedPrice; U32 cachedLitLength; const BYTE* cachedLiterals; - ZSTD_stats_t* stats; + ZSTD_stats_t stats; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index 428514bfe..5cefa3115 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -38,8 +38,6 @@ extern "C" { #endif -#if ZSTD_OPT_DEBUG == 3 - /*-************************************* * Types ***************************************/ @@ -156,16 +154,6 @@ MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, co stats->totalLitSum += litLength; } -#else - struct ZSTD_stats_s { U32 unused; }; - MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc(void) { return NULL; } - MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif // #if ZSTD_OPT_DEBUG == 3 - #if defined (__cplusplus) } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 93efa99c9..01d276fe3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -57,7 +57,6 @@ #include "fse_static.h" #include "huf_static.h" #include "zstd_internal.h" -#include ".debug/zstd_stats.h" /*-************************************* @@ -888,7 +887,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - ZSTD_statsUpdatePrices(seqStorePtr->stats, litLength, literals, offsetCode, matchCode); + ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode); /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); @@ -1780,7 +1779,7 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; - ZSTD_statsUpdatePrices(seqStorePtr->stats, lastLLSize, anchor, 0, 0); + ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -2052,15 +2051,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = ZSTD_statsAlloc(); - zc->seqStore.stats = stats; + ZSTD_stats_t* stats = &zc->seqStore.stats; ZSTD_statsInit(stats); while (remaining) { size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { ZSTD_statsFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2071,11 +2069,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } + if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } + if (ZSTD_isError(cSize)) return cSize; } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2091,7 +2089,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - ZSTD_statsFree(stats); return op-ostart; } diff --git a/programs/fullbench.c b/programs/fullbench.c index fe1abf1a2..6f5d6a782 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -213,19 +213,6 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, /*_******************************************************* * Bench functions *********************************************************/ -void* BMK_allocFunction(size_t size) -{ - void* address = malloc(size); - /* printf("alloc %p, %d \n", address, (int)size); */ - return address; -} - -void BMK_freeFunction(void* address) -{ -/* printf("free %p \n", address); */ - free(address); -} - static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { BYTE* dstBuff; @@ -233,7 +220,6 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) BYTE* buff2; const char* benchName; size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); - ZSTD_customMem customMem = { BMK_allocFunction, BMK_freeFunction }; double bestTime = 100000000.; /* Selection */ @@ -261,14 +247,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue"; break; case 42: - benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_createCCtx_advanced/ZBUFF_compressContinue"; - break; - case 43: benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue"; break; - case 44: - benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_createDCtx_advanced/ZBUFF_decompressContinue"; - break; default : return 0; } @@ -343,16 +323,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx(); break; case 42 : - if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx_advanced(customMem); - break; - case 43 : if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx(); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); break; - case 44 : - if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx_advanced(customMem); - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); - break; /* test functions */ /* by convention, test functions can be added > 100 */ @@ -382,29 +355,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds; if (averageTime < bestTime) bestTime = averageTime; DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult); - } - - /* free allocated structures */ - switch(benchNb) - { - case 11 : - if (g_zcc) { ZSTD_freeCCtx(g_zcc); g_zcc=NULL; } - break; - case 12 : - case 31: - case 32: - if (g_zdc) { ZSTD_freeDCtx(g_zdc); g_zdc=NULL; } - break; - case 41 : - case 42 : - if (g_zbcc) { ZBUFF_freeCCtx(g_zbcc); g_zbcc=NULL; } - break; - case 43 : - case 44 : - if (g_zbdc) { ZBUFF_freeDCtx(g_zbdc); g_zbdc=NULL; } - break; - default : ; - } } + }} DISPLAY("%2u\n", benchNb); _cleanOut: From 33b8a893edd3dadc960e5060044cc35a141324fd Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 17:44:14 +0200 Subject: [PATCH 040/208] CMake project moved to projects --- Makefile | 2 +- projects/README.md | 1 + {contrib => projects}/cmake/CMakeLists.txt | 0 .../cmake/CMakeModules/AddExtraCompilationFlags.cmake | 0 projects/cmake/build/.keep | 0 {contrib => projects}/cmake/cmake_uninstall.cmake.in | 0 {contrib => projects}/cmake/lib/CMakeLists.txt | 0 {contrib => projects}/cmake/programs/CMakeLists.txt | 0 8 files changed, 2 insertions(+), 1 deletion(-) rename {contrib => projects}/cmake/CMakeLists.txt (100%) rename {contrib => projects}/cmake/CMakeModules/AddExtraCompilationFlags.cmake (100%) create mode 100644 projects/cmake/build/.keep rename {contrib => projects}/cmake/cmake_uninstall.cmake.in (100%) rename {contrib => projects}/cmake/lib/CMakeLists.txt (100%) rename {contrib => projects}/cmake/programs/CMakeLists.txt (100%) diff --git a/Makefile b/Makefile index 95ecc007c..0894f75d7 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ travis-install: $(MAKE) install PREFIX=~/install_test_dir cmaketest: - cd contrib/cmake ; cmake . ; $(MAKE) + cd projects/cmake/build ; cmake .. ; $(MAKE) clangtest: clean clang -v diff --git a/projects/README.md b/projects/README.md index d69c71b14..b6831ce29 100644 --- a/projects/README.md +++ b/projects/README.md @@ -4,5 +4,6 @@ projects for various integrated development environments (IDE) #### Included projects The following projects are included with the zstd distribution: +- cmake - CMake project contributed by Artyom Dymchenko - VS2008 - Visual Studio 2008 project - VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) diff --git a/contrib/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt similarity index 100% rename from contrib/cmake/CMakeLists.txt rename to projects/cmake/CMakeLists.txt diff --git a/contrib/cmake/CMakeModules/AddExtraCompilationFlags.cmake b/projects/cmake/CMakeModules/AddExtraCompilationFlags.cmake similarity index 100% rename from contrib/cmake/CMakeModules/AddExtraCompilationFlags.cmake rename to projects/cmake/CMakeModules/AddExtraCompilationFlags.cmake diff --git a/projects/cmake/build/.keep b/projects/cmake/build/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/cmake/cmake_uninstall.cmake.in b/projects/cmake/cmake_uninstall.cmake.in similarity index 100% rename from contrib/cmake/cmake_uninstall.cmake.in rename to projects/cmake/cmake_uninstall.cmake.in diff --git a/contrib/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt similarity index 100% rename from contrib/cmake/lib/CMakeLists.txt rename to projects/cmake/lib/CMakeLists.txt diff --git a/contrib/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt similarity index 100% rename from contrib/cmake/programs/CMakeLists.txt rename to projects/cmake/programs/CMakeLists.txt From 460a4ee6d4d6659d46c052b7477c085e7646f369 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 18:22:38 +0200 Subject: [PATCH 041/208] playTests updated with Pass-Through mode --- programs/tests/playTests.bat | 10 +++++---- programs/tests/playTestsMSYS.sh | 38 +++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index edbbcb340..5f7ebd685 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -31,9 +31,11 @@ REM %ZSTD% -d - < tmp.zst > NUL %ZSTD% -q -f tmp %ZSTD% -q --force tmp %ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) -cp tmp tmp2.zst -%ZSTD% -df tmp2.zst && (echo should have failed : wrong format && exit /b 1) -rm tmp2.zst + +echo. && echo **** Pass-Through mode **** +echo "Hello world !" | %ZSTD% -df +echo "Hello world !" | %ZSTD% -dcf + echo. && echo **** frame concatenation **** @@ -48,7 +50,7 @@ cat result.tmp fc /b helloworld.tmp result.tmp rm *.tmp *.zstd -echo frame concatenation test completed +echo frame concatenation tests completed REM echo. && echo **** flush write error test **** diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh index 4289977a0..3d5a0e842 100755 --- a/programs/tests/playTestsMSYS.sh +++ b/programs/tests/playTestsMSYS.sh @@ -47,9 +47,12 @@ $ZSTD -q tmp && die "overwrite check failed!" $ZSTD -q -f tmp $ZSTD -q --force tmp $ZSTD -df tmp && die "should have refused : wrong extension" -cp tmp tmp2.zst -$ZSTD -df tmp2.zst && die "should have failed : wrong format" -rm tmp2.zst + + +echo -e "\n**** Pass-Through mode **** " +echo "Hello world !" | $ZSTD -df +echo "Hello world !" | $ZSTD -dcf + echo -e "\n**** frame concatenation **** " @@ -64,7 +67,34 @@ cat result.tmp fc helloworld.tmp result.tmp rm ./*.tmp ./*.zstd -echo frame concatenation test completed +echo frame concatenation tests completed + + +echo -e "\n**** test sparse file support **** " + +./datagen -g5M -P100 > tmpSparse +$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen +fc tmpSparse tmpSparseRegen +$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse +fc tmpSparse tmpOutSparse +$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse +fc tmpSparse tmpOutNoSparse +ls -ls tmpSparse* +./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) +./datagen -s1 -g1200007 -P100 | fc - tmpSparseOdd +ls -ls tmpSparseOdd +echo -e "\n Sparse Compatibility with Console :" +echo "Hello World 1 !" | $ZSTD | $ZSTD -d -c +echo "Hello World 2 !" | $ZSTD | $ZSTD -d | cat +echo -e "\n Sparse Compatibility with Append :" +./datagen -P100 -g1M > tmpSparse1M +cat tmpSparse1M tmpSparse1M > tmpSparse2M +$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed +$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated +$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated +ls -ls tmpSparse* +fc tmpSparse2M tmpSparseRegenerated + echo -e "\n**** dictionary tests **** " From ebc13bc180028bd0a0be0ec5075133e85a76ad6c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 25 May 2016 10:12:39 +0200 Subject: [PATCH 042/208] cleaner versionsTest script and output --- versionsTest/test-zstd-versions.py | 41 ++++++++++++++++++------------ 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/versionsTest/test-zstd-versions.py b/versionsTest/test-zstd-versions.py index b18c3d3c2..5b3d9463b 100644 --- a/versionsTest/test-zstd-versions.py +++ b/versionsTest/test-zstd-versions.py @@ -2,13 +2,13 @@ # Based on LZ4 version test script, by Takayuki Matsuoka -import glob -import subprocess import filecmp +import glob +import hashlib import os import shutil +import subprocess import sys -import hashlib repo_url = 'https://github.com/Cyan4973/zstd.git' tmp_dir_name = 'versionsTest/zstdtest' @@ -18,6 +18,7 @@ test_dat_src = 'README.md' test_dat = 'test_dat' head = 'vdevel' + def proc(cmd_args, pipe=True, dummy=False): if dummy: return @@ -29,40 +30,46 @@ def proc(cmd_args, pipe=True, dummy=False): subproc = subprocess.Popen(cmd_args) return subproc.communicate() + def make(args, pipe=True): return proc([make_cmd] + args, pipe) + def git(args, pipe=True): return proc([git_cmd] + args, pipe) + def get_git_tags(): stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]']) tags = stdout.decode('utf-8').split() return tags + def compress_sample(tag, sample): try: - from subprocess import DEVNULL # py3k + from subprocess import DEVNULL # py3k except ImportError: DEVNULL = open(os.devnull, 'wb') - if subprocess.call(['./zstd.' + tag, '-f' , sample], stderr=DEVNULL)==0: + if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-5f' , sample], stderr=DEVNULL)==0: + if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-9f' , sample], stderr=DEVNULL)==0 : + if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0: os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL)==0 : + if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0: os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst') - if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL)==0: + if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0: os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst') # zstdFiles = glob.glob("*.zst*") # print(zstdFiles) + # http://stackoverflow.com/a/19711609/2132223 def sha1_of_file(filepath): with open(filepath, 'rb') as f: return hashlib.sha1(f.read()).hexdigest() + def remove_duplicates(): list_of_zst = sorted(glob.glob('*.zst')) for i, ref_zst in enumerate(list_of_zst): @@ -76,18 +83,19 @@ def remove_duplicates(): os.remove(compared_zst) print('duplicated : {} == {}'.format(ref_zst, compared_zst)) + def decompress_zst(tag): dec_error = 0 list_zst = sorted(glob.glob('*.zst')) try: - from subprocess import DEVNULL # py3k + from subprocess import DEVNULL # py3k except ImportError: DEVNULL = open(os.devnull, 'wb') for file_zst in list_zst: print(file_zst, end=" ") print(tag, end=" ") file_dec = file_zst + '_d64_' + tag + '.dec' - if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL)==0: + if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0: if not filecmp.cmp(file_dec, test_dat): print('ERR !! ') dec_error = 1 @@ -116,12 +124,12 @@ if __name__ == '__main__': print('Retrieve all release tags :') os.chdir(clone_dir) tags = get_git_tags() + [head] - print(tags); + print(tags) # Build all release zstd for tag in tags: os.chdir(base_dir) - dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/zstd. + dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/zstd. if not os.path.isfile(dst_zstd) or tag == head: if tag != head: r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/ @@ -139,17 +147,18 @@ if __name__ == '__main__': os.chdir(tmp_dir) for compressed in glob.glob("*.zst"): os.remove(compressed) - for dec in glob.glob("*.dec"): + for dec in glob.glob("*.dec"): os.remove(dec) print('Compress test.dat by all released zstd') - error_code = 0; + error_code = 0 for tag in tags: print(tag) compress_sample(tag, test_dat) remove_duplicates() - error_code += decompress_zst(tag) + if tag >= 'v0.5.1': + error_code += decompress_zst(tag) print('') print('Enumerate different compressed files') From 5d589562fe93d517abf71aa8fc71c5cbb545d0cf Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 10:50:28 +0200 Subject: [PATCH 043/208] playTests.sh joined with playTestsMSYS.sh --- appveyor.yml | 30 ++--- programs/Makefile | 8 +- programs/tests/playTests.sh | 94 +++++++++------- programs/tests/playTestsMSYS.sh | 190 -------------------------------- 4 files changed, 71 insertions(+), 251 deletions(-) delete mode 100755 programs/tests/playTestsMSYS.sh diff --git a/appveyor.yml b/appveyor.yml index c6f735ea3..4721fa951 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,24 +1,24 @@ version: 1.0.{build} environment: matrix: - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - COMPILER: "gcc" - MAKE_PARAMS: "test" - PLATFORM: "mingw32" - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw64" + - COMPILER: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% diff --git a/programs/Makefile b/programs/Makefile index 11709300a..c4e19229a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -155,9 +155,6 @@ clean: #------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) HOST_OS = POSIX -zstd-playTests: datagen - ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) - install: zstd @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @@ -199,8 +196,6 @@ endif ifneq (,$(filter MSYS%,$(shell uname))) HOST_OS = MSYS -zstd-playTests: datagen - ZSTD=$(ZSTD) ./tests/playTestsMSYS.sh $(ZSTDRTTEST) endif @@ -208,6 +203,9 @@ endif #make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets #------------------------------------------------------------------------ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) + test: test-zstd test-fullbench test-fuzzer test-zbuff test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index 58455c0f9..dbeb916d0 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -1,7 +1,7 @@ #!/bin/sh -e die() { - echo "$@" 1>&2 + $ECHO "$@" 1>&2 exit 1 } @@ -14,34 +14,44 @@ roundTripTest() { fi rm -f tmp1 tmp2 - echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" ./datagen $1 $p | md5sum > tmp1 ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 diff -q tmp1 tmp2 } +isWindows=false +ECHO="echo" +if [[ "$OS" == "Windows"* ]]; then + isWindows=true + ECHO="echo -e" +fi + +$ECHO "\nStarting playTests.sh isWindows=$isWindows" + [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" - -echo "\n**** simple tests **** " +$ECHO "\n**** simple tests **** " ./datagen > tmp $ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) -echo "test : too large compression level (must fail)" +$ECHO "test : too large compression level (must fail)" $ZSTD -99 tmp && die "too large compression level undetected" -echo "test : compress to stdout" +$ECHO "test : compress to stdout" $ZSTD tmp -c > tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format -echo "test : null-length file roundtrip" -echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout -echo "test : decompress file with wrong suffix (must fail)" +$ECHO "test : null-length file roundtrip" +$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout +$ECHO "test : decompress file with wrong suffix (must fail)" $ZSTD -d tmpCompressed && die "wrong suffix error not detected!" $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout $ZSTD --decompress tmpCompressed -c > tmpResult $ZSTD --decompress tmpCompressed --stdout > tmpResult -$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout -$ZSTD -d - < tmp.zst > /dev/null +if [ "$isWindows" = false ] ; then + $ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout + $ZSTD -d - < tmp.zst > /dev/null +fi $ZSTD -dc < tmp.zst > /dev/null $ZSTD -dc - < tmp.zst > /dev/null $ZSTD -q tmp && die "overwrite check failed!" @@ -50,15 +60,15 @@ $ZSTD -q --force tmp $ZSTD -df tmp && die "should have refused : wrong extension" -echo "\n**** Pass-Through mode **** " -echo "Hello world !" | $ZSTD -df -echo "Hello world !" | $ZSTD -dcf +$ECHO "\n**** Pass-Through mode **** " +$ECHO "Hello world !" | $ZSTD -df +$ECHO "Hello world !" | $ZSTD -dcf -echo "\n**** frame concatenation **** " +$ECHO "\n**** frame concatenation **** " -echo "hello " > hello.tmp -echo "world!" > world.tmp +$ECHO "hello " > hello.tmp +$ECHO "world!" > world.tmp cat hello.tmp world.tmp > helloworld.tmp $ZSTD -c hello.tmp > hello.zstd $ZSTD -c world.tmp > world.zstd @@ -67,18 +77,20 @@ $ZSTD -dc helloworld.zstd > result.tmp cat result.tmp sdiff helloworld.tmp result.tmp rm ./*.tmp ./*.zstd -echo "frame concatenation tests completed" +$ECHO "frame concatenation tests completed" -echo "\n**** flush write error test **** " +if [ "$isWindows" = false ] ; then +$ECHO "\n**** flush write error test **** " -echo "echo foo | $ZSTD > /dev/full" -echo foo | $ZSTD > /dev/full && die "write error not detected!" -echo "echo foo | $ZSTD | $ZSTD -d > /dev/full" -echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" +$ECHO "$ECHO foo | $ZSTD > /dev/full" +$ECHO foo | $ZSTD > /dev/full && die "write error not detected!" +$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full" +$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" +fi -echo "\n**** test sparse file support **** " +$ECHO "\n**** test sparse file support **** " ./datagen -g5M -P100 > tmpSparse $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen @@ -91,10 +103,10 @@ ls -ls tmpSparse* ./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) ./datagen -s1 -g1200007 -P100 | diff -s - tmpSparseOdd ls -ls tmpSparseOdd -echo "\n Sparse Compatibility with Console :" -echo "Hello World 1 !" | $ZSTD | $ZSTD -d -c -echo "Hello World 2 !" | $ZSTD | $ZSTD -d | cat -echo "\n Sparse Compatibility with Append :" +$ECHO "\n Sparse Compatibility with Console :" +$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c +$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat +$ECHO "\n Sparse Compatibility with Append :" ./datagen -P100 -g1M > tmpSparse1M cat tmpSparse1M tmpSparse1M > tmpSparse2M $ZSTD -v -f tmpSparse1M -o tmpSparseCompressed @@ -105,7 +117,7 @@ diff tmpSparse2M tmpSparseRegenerated # rm tmpSparse* -echo "\n**** dictionary tests **** " +$ECHO "\n**** dictionary tests **** " ./datagen > tmpDict ./datagen -g1M | md5sum > tmp1 @@ -117,41 +129,41 @@ $ZSTD -d tmp -D tmpDict -of result diff xxhash.c result -echo "\n**** multiple files tests **** " +$ECHO "\n**** multiple files tests **** " ./datagen -s1 > tmp1 2> /dev/null ./datagen -s2 -g100K > tmp2 2> /dev/null ./datagen -s3 -g1M > tmp3 2> /dev/null $ZSTD -f tmp* -echo "compress tmp* : " +$ECHO "compress tmp* : " ls -ls tmp* rm tmp1 tmp2 tmp3 -echo "decompress tmp* : " +$ECHO "decompress tmp* : " $ZSTD -df *.zst ls -ls tmp* -echo "compress tmp* into stdout > tmpall : " +$ECHO "compress tmp* into stdout > tmpall : " $ZSTD -c tmp1 tmp2 tmp3 > tmpall ls -ls tmp* -echo "decompress tmpall* into stdout > tmpdec : " +$ECHO "decompress tmpall* into stdout > tmpdec : " cp tmpall tmpall2 $ZSTD -dc tmpall* > tmpdec ls -ls tmp* -echo "compress multiple files including a missing one (notHere) : " +$ECHO "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" -echo "\n**** integrity tests **** " +$ECHO "\n**** integrity tests **** " -echo "test one file (tmp1.zst) " +$ECHO "test one file (tmp1.zst) " $ZSTD -t tmp1.zst $ZSTD --test tmp1.zst -echo "test multiple files (*.zst) " +$ECHO "test multiple files (*.zst) " $ZSTD -t *.zst -echo "test good and bad files (*) " +$ECHO "test good and bad files (*) " $ZSTD -t * && die "bad files not detected !" -echo "\n**** zstd round-trip tests **** " +$ECHO "\n**** zstd round-trip tests **** " roundTripTest roundTripTest -g15K # TableID==3 @@ -165,7 +177,7 @@ roundTripTest -g512K 19 # btopt rm tmp* if [ "$1" != "--test-large-data" ]; then - echo "Skipping large data tests" + $ECHO "Skipping large data tests" exit 0 fi diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh deleted file mode 100755 index 3d5a0e842..000000000 --- a/programs/tests/playTestsMSYS.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/sh -e - -die() { - echo "$@" 1>&2 - exit 1 -} - -roundTripTest() { - if [ -n "$3" ]; then - local c="$3" - local p="$2" - else - local c="$2" - fi - - rm -f tmp1 tmp2 - echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" - ./datagen $1 $p | md5sum > tmp1 - ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 - fc tmp1 tmp2 -} - -[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" - - -echo -e "\n**** simple tests **** " -./datagen > tmp -$ZSTD -f tmp # trivial compression case, creates tmp.zst -$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) -echo "test : too large compression level (must fail)" -$ZSTD -99 tmp && die "too large compression level undetected" -echo "test : compress to stdout" -$ZSTD tmp -c > tmpCompressed -$ZSTD tmp --stdout > tmpCompressed # long command format -echo "test : null-length file roundtrip" -echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout -echo "test : decompress file with wrong suffix (must fail)" -$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" -$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout -$ZSTD --decompress tmpCompressed -c > tmpResult -$ZSTD --decompress tmpCompressed --stdout > tmpResult -#$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout -#$ZSTD -d - < tmp.zst > /dev/null -$ZSTD -dc < tmp.zst > /dev/null -$ZSTD -dc - < tmp.zst > /dev/null -$ZSTD -q tmp && die "overwrite check failed!" -$ZSTD -q -f tmp -$ZSTD -q --force tmp -$ZSTD -df tmp && die "should have refused : wrong extension" - - -echo -e "\n**** Pass-Through mode **** " -echo "Hello world !" | $ZSTD -df -echo "Hello world !" | $ZSTD -dcf - - -echo -e "\n**** frame concatenation **** " - -echo "hello " > hello.tmp -echo "world!" > world.tmp -cat hello.tmp world.tmp > helloworld.tmp -$ZSTD -c hello.tmp > hello.zstd -$ZSTD -c world.tmp > world.zstd -cat hello.zstd world.zstd > helloworld.zstd -$ZSTD -dc helloworld.zstd > result.tmp -cat result.tmp -fc helloworld.tmp result.tmp -rm ./*.tmp ./*.zstd - -echo frame concatenation tests completed - - -echo -e "\n**** test sparse file support **** " - -./datagen -g5M -P100 > tmpSparse -$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen -fc tmpSparse tmpSparseRegen -$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse -fc tmpSparse tmpOutSparse -$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse -fc tmpSparse tmpOutNoSparse -ls -ls tmpSparse* -./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) -./datagen -s1 -g1200007 -P100 | fc - tmpSparseOdd -ls -ls tmpSparseOdd -echo -e "\n Sparse Compatibility with Console :" -echo "Hello World 1 !" | $ZSTD | $ZSTD -d -c -echo "Hello World 2 !" | $ZSTD | $ZSTD -d | cat -echo -e "\n Sparse Compatibility with Append :" -./datagen -P100 -g1M > tmpSparse1M -cat tmpSparse1M tmpSparse1M > tmpSparse2M -$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed -$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated -$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated -ls -ls tmpSparse* -fc tmpSparse2M tmpSparseRegenerated - - - -echo -e "\n**** dictionary tests **** " - -./datagen > tmpDict -./datagen -g1M | md5sum > tmp1 -./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 -fc tmp1 tmp2 -$ZSTD --train *.c *.h -o tmpDict -$ZSTD xxhash.c -D tmpDict -of tmp -$ZSTD -d tmp -D tmpDict -of result -fc xxhash.c result - - -echo -e "\n**** multiple files tests **** " - -./datagen -s1 > tmp1 2> /dev/null -./datagen -s2 -g100K > tmp2 2> /dev/null -./datagen -s3 -g1M > tmp3 2> /dev/null -$ZSTD -f tmp* -echo "compress tmp* : " -ls -ls tmp* -rm tmp1 tmp2 tmp3 -echo "decompress tmp* : " -$ZSTD -df *.zst -ls -ls tmp* -echo "compress tmp* into stdout > tmpall : " -$ZSTD -c tmp1 tmp2 tmp3 > tmpall -ls -ls tmp* -echo "decompress tmpall* into stdout > tmpdec : " -cp tmpall tmpall2 -$ZSTD -dc tmpall* > tmpdec -ls -ls tmp* -echo "compress multiple files including a missing one (notHere) : " -$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" - -echo -e "\n**** integrity tests **** " -echo "test one file (tmp1.zst) " -$ZSTD -t tmp1.zst -$ZSTD --test tmp1.zst -echo "test multiple files (*.zst) " -$ZSTD -t *.zst -echo "test good and bad files (*) " -$ZSTD -t * && die "bad files not detected !" - -echo -e "\n**** zstd round-trip tests **** " - -roundTripTest -roundTripTest -g15K # TableID==3 -roundTripTest -g127K # TableID==2 -roundTripTest -g255K # TableID==1 -roundTripTest -g513K # TableID==0 -roundTripTest -g512K 6 # greedy, hash chain -roundTripTest -g512K 16 # btlazy2 -roundTripTest -g512K 19 # btopt - -rm tmp* - -if [ "$1" != "--test-large-data" ]; then - echo "Skipping large data tests" - exit 0 -fi - -roundTripTest -g270000000 1 -roundTripTest -g270000000 2 -roundTripTest -g270000000 3 - -roundTripTest -g140000000 -P60 4 -roundTripTest -g140000000 -P60 5 -roundTripTest -g140000000 -P60 6 - -roundTripTest -g70000000 -P70 7 -roundTripTest -g70000000 -P70 8 -roundTripTest -g70000000 -P70 9 - -roundTripTest -g35000000 -P75 10 -roundTripTest -g35000000 -P75 11 -roundTripTest -g35000000 -P75 12 - -roundTripTest -g18000000 -P80 13 -roundTripTest -g18000000 -P80 14 -roundTripTest -g18000000 -P80 15 -roundTripTest -g18000000 -P80 16 -roundTripTest -g18000000 -P80 17 - -roundTripTest -g50000000 -P94 18 -roundTripTest -g50000000 -P94 19 - -roundTripTest -g99000000 -P99 20 -roundTripTest -g6000000000 -P99 1 - -rm tmp* - From 0d0f7e4357f91d27421ab3d694bb2792ff95f769 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 25 May 2016 10:58:11 +0200 Subject: [PATCH 044/208] minor linter modifications --- versionsTest/test-zstd-versions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versionsTest/test-zstd-versions.py b/versionsTest/test-zstd-versions.py index 5b3d9463b..f7428bc22 100644 --- a/versionsTest/test-zstd-versions.py +++ b/versionsTest/test-zstd-versions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 - +"""Test zstd interoperability between versions""" # Based on LZ4 version test script, by Takayuki Matsuoka import filecmp @@ -75,7 +75,7 @@ def remove_duplicates(): for i, ref_zst in enumerate(list_of_zst): if not os.path.isfile(ref_zst): continue - for j in range(i+1, len(list_of_zst)): + for j in range(i + 1, len(list_of_zst)): compared_zst = list_of_zst[j] if not os.path.isfile(compared_zst): continue @@ -92,8 +92,8 @@ def decompress_zst(tag): except ImportError: DEVNULL = open(os.devnull, 'wb') for file_zst in list_zst: - print(file_zst, end=" ") - print(tag, end=" ") + print(file_zst, end=' ') + print(tag, end=' ') file_dec = file_zst + '_d64_' + tag + '.dec' if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0: if not filecmp.cmp(file_dec, test_dat): From e75909ef93c87c3b716f3fb85aaf81f5ef0e6310 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 11:31:16 +0200 Subject: [PATCH 045/208] ZBUFF_createCCtx_advanced moved to zbuff_static.h --- lib/common/zbuff.h | 3 --- lib/common/zbuff_static.h | 17 ++++++++++++++++- programs/fileio.c | 1 + programs/zbufftest.c | 15 ++++++++------- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 4d7f12a1f..89207bd24 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -39,7 +39,6 @@ extern "C" { * Dependencies ***************************************/ #include /* size_t */ -#include "zstd_static.h" /* ZSTD_customMem */ /* *************************************************************** @@ -61,7 +60,6 @@ extern "C" { ***************************************/ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); -ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); @@ -114,7 +112,6 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); -ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); diff --git a/lib/common/zbuff_static.h b/lib/common/zbuff_static.h index 4c52eef22..6a4287e4d 100644 --- a/lib/common/zbuff_static.h +++ b/lib/common/zbuff_static.h @@ -46,7 +46,22 @@ extern "C" { ***************************************/ #include "zstd_static.h" /* ZSTD_parameters */ #include "zbuff.h" -#include "zstd_internal.h" /* MIN */ + +#ifndef MIN + #define MIN(a,b) ((a)<(b) ? (a) : (b)) +#endif + + +/*-************************************* +* Advanced functions +***************************************/ +/*! ZBUFF_createCCtx_advanced() : + * Create a ZBUFF compression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZBUFF_createDCtx_advanced() : + * Create a ZBUFF decompression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); /* ************************************* diff --git a/programs/fileio.c b/programs/fileio.c index 51707e8b7..eba90d011 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -57,6 +57,7 @@ #include "mem.h" #include "fileio.h" #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ +#include "zstd_internal.h" /* MIN, KB, MB */ #include "zbuff_static.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index e9e54586c..fce0ab275 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -35,15 +35,16 @@ /*-************************************ * Includes **************************************/ -#include /* free */ -#include /* fgets, sscanf */ -#include /* timeb */ -#include /* strcmp */ +#include /* free */ +#include /* fgets, sscanf */ +#include /* timeb */ +#include /* strcmp */ #include "mem.h" #include "zbuff.h" -#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ -#include "datagen.h" /* RDG_genBuffer */ -#include "xxhash.h" /* XXH64 */ +#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ +#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ +#include "datagen.h" /* RDG_genBuffer */ +#include "xxhash.h" /* XXH64 */ /*-************************************ From 4f1acdd15890cea8cfba011ef470a905a3963d60 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 12:41:41 +0200 Subject: [PATCH 046/208] usage of fileno() only on POSIX and Windows --- programs/zbufftest.c | 2 -- programs/zstdcli.c | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index fce0ab275..c780b8f4e 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -262,8 +262,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) return FUZ_rLogLength(seed, logLength); } -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 13ff7955d..215157873 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -52,9 +52,12 @@ # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else - extern int fileno(FILE *stream); /* triggers fileno() within on POSIX */ +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif #endif From 957823f56f245e6fb2c562fa8f8160aec6be730d Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 25 May 2016 15:30:55 +0200 Subject: [PATCH 047/208] zstdcli: -r (operate recursively on directories) works with dictBuilder and compression --- lib/common/zbuff_static.h | 6 ++---- programs/bench.c | 22 ++-------------------- programs/bench.h | 2 +- programs/util.h | 3 +-- programs/zbufftest.c | 2 ++ programs/zstdcli.c | 39 +++++++++++++++++++++++++++++++-------- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/lib/common/zbuff_static.h b/lib/common/zbuff_static.h index 6a4287e4d..c50a3e166 100644 --- a/lib/common/zbuff_static.h +++ b/lib/common/zbuff_static.h @@ -47,9 +47,7 @@ extern "C" { #include "zstd_static.h" /* ZSTD_parameters */ #include "zbuff.h" -#ifndef MIN - #define MIN(a,b) ((a)<(b) ? (a) : (b)) -#endif +#define ZBUFF_MIN(a,b) ((a)<(b) ? (a) : (b)) /*-************************************* @@ -73,7 +71,7 @@ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t length = MIN(dstCapacity, srcSize); + size_t length = ZBUFF_MIN(dstCapacity, srcSize); memcpy(dst, src, length); return length; } diff --git a/programs/bench.c b/programs/bench.c index a2721552e..52c64d525 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -26,7 +26,7 @@ /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_HAS_CREATEFILELIST, UTIL_sleep */ +#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ @@ -495,32 +495,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int recursive) + const char* dictFileName, int cLevel, int cLevelLast) { double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) BMK_syntheticTest(cLevel, cLevelLast, compressibility); else - { -#ifdef UTIL_HAS_CREATEFILELIST - if (recursive) { - char* buf; - const char** filenameTable; - unsigned i; - filenameTable = UTIL_createFileList(fileNamesTable, nbFiles, &buf, &nbFiles); - if (filenameTable) { - for (i=0; i= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; diff --git a/programs/zbufftest.c b/programs/zbufftest.c index c780b8f4e..fce0ab275 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -262,6 +262,8 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) return FUZ_rLogLength(seed, logLength); } +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 215157873..f367f49af 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -129,6 +129,9 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories\n"); +#endif #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); #endif @@ -147,9 +150,6 @@ static int usage_advanced(const char* programName) DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); -#ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories\n"); -#endif DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); #endif return 0; @@ -198,6 +198,11 @@ int main(int argCount, const char** argv) unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; +#ifdef UTIL_HAS_CREATEFILELIST + const char** fileNamesTable = NULL; + char* fileNamesBuf; + unsigned fileNamesNb; +#endif /* init */ (void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ @@ -296,6 +301,9 @@ int main(int argCount, const char** argv) /* dictionary name */ case 'o': nextArgumentIsOutFileName=1; argument++; break; + /* recursive */ + case 'r': recursive=1; argument++; break; + #ifndef ZSTD_NOBENCH /* Benchmark */ case 'b': bench=1; argument++; break; @@ -322,9 +330,6 @@ int main(int argCount, const char** argv) } break; - /* recursive */ - case 'r': recursive=1; argument++; break; - /* cut input into blocks (benchmark only) */ case 'B': { size_t bSize = 0; @@ -395,11 +400,24 @@ int main(int argCount, const char** argv) /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); +#ifdef UTIL_HAS_CREATEFILELIST + if (recursive) { + fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + if (fileNamesTable) { + unsigned i; + for (i=0; i Date: Wed, 25 May 2016 17:40:58 +0200 Subject: [PATCH 048/208] fixed Visual Studio warning --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f367f49af..68dd98cb0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -200,7 +200,7 @@ int main(int argCount, const char** argv) unsigned dictSelect = g_defaultSelectivityLevel; #ifdef UTIL_HAS_CREATEFILELIST const char** fileNamesTable = NULL; - char* fileNamesBuf; + char* fileNamesBuf = NULL; unsigned fileNamesNb; #endif From 993a9dfc5a0815b8e397b03cf5b6c5ba4191c5c5 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 27 May 2016 10:07:46 +0200 Subject: [PATCH 049/208] playTests.sh updated to work better with /bin/sh --- programs/tests/playTests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index dbeb916d0..5ff4c4e07 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -22,10 +22,12 @@ roundTripTest() { isWindows=false ECHO="echo" -if [[ "$OS" == "Windows"* ]]; then +case "$OS" in + Windows*) isWindows=true ECHO="echo -e" -fi + ;; +esac $ECHO "\nStarting playTests.sh isWindows=$isWindows" From e9dfdf6924e89f4af45309a1bb3bb70721501429 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 27 May 2016 11:56:15 +0200 Subject: [PATCH 050/208] added zlibwrapper test for Travis --- .travis.yml | 3 ++- Makefile | 8 +++++++- zlibWrapper/Makefile | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f583a803d..70f4becee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,8 @@ env: - ZSTD_TRAVIS_CI_ENV=asan - ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" - + - ZSTD_TRAVIS_CI_ENV=zlibwrapper + compiler: gcc script: diff --git a/Makefile b/Makefile index 0894f75d7..6be618bc1 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ PRGDIR = programs ZSTDDIR = lib +ZWRAPDIR = zlibWrapper # Define nul output ifneq (,$(filter Windows%,$(OS))) @@ -40,7 +41,7 @@ else VOID = /dev/null endif -.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan +.PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan default: zstdprogram @@ -51,12 +52,17 @@ all: zstdprogram: $(MAKE) -C $(PRGDIR) +zlibwrapper: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(ZWRAPDIR) all + test: $(MAKE) -C $(PRGDIR) $@ clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) + @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) @echo Cleaning completed diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index fd45ee6d3..c74107d40 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -15,14 +15,14 @@ IMPLIB = -lz ../lib/libzstd.a endif ZLIBWRAPPER_PATH = . -EXAMPLE_PATH = examples/ +EXAMPLE_PATH = examples CC = gcc CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 LDFLAGS = $(LOC) RM = rm -f -all: example example_d +all: clean test testdll test: example ./example From d0c386247974c021f1119e6b90741676b9357a14 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 27 May 2016 12:33:19 +0200 Subject: [PATCH 051/208] #define z_const if not defined --- zlibWrapper/zstd_zlibwrapper.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index c9039ddb5..c2c908ddb 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -40,6 +40,9 @@ extern "C" { #define Z_PREFIX #include +#if !defined(z_const) +# define z_const const +#endif void useZSTD(int turn_on); int isUsingZSTD(); From 6c903a8dd5d913a5e968646a47e30029d37b5ccd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 28 May 2016 13:34:07 +0200 Subject: [PATCH 052/208] updated xxhash to latest "dev" version --- programs/.clang_complete | 3 + programs/fuzzer.c | 11 +- programs/xxhash.c | 621 ++++++++++++++++----------------------- programs/xxhash.h | 97 ++++-- programs/zbufftest.c | 9 +- 5 files changed, 341 insertions(+), 400 deletions(-) create mode 100644 programs/.clang_complete diff --git a/programs/.clang_complete b/programs/.clang_complete new file mode 100644 index 000000000..658aa00b0 --- /dev/null +++ b/programs/.clang_complete @@ -0,0 +1,3 @@ +-I../lib/common +-I../lib/legacy +-I./legacy diff --git a/programs/fuzzer.c b/programs/fuzzer.c index e664bf433..4b5697c4b 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -42,8 +42,9 @@ #include /* clock_t */ #include "zstd_static.h" /* ZSTD_VERSION_STRING */ #include "datagen.h" /* RDG_genBuffer */ -#include "xxhash.h" /* XXH64 */ #include "mem.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" /* XXH64 */ /*-************************************ @@ -463,7 +464,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t cSize, dSize, totalCSize, totalGenSize; U32 sampleSizeLog, nbChunks, n; - XXH64_CREATESTATE_STATIC(xxh64); + XXH64_state_t xxhState; U64 crcOrig; BYTE* sampleBuffer; const BYTE* dict; @@ -614,7 +615,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx); CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); } } - XXH64_reset(xxh64, 0); + XXH64_reset(&xxhState, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; for (totalTestSize=0, cSize=0, n=0 ; n +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/* for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + /* ************************************* * Compiler Specific Options @@ -103,27 +127,12 @@ You can contact the author at : #endif -/* ************************************* -* Includes & Memory related functions -***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for malloc(), free() */ -#include -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free (void* p) { free(p); } -/* for memcpy() */ -#include -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - -#include "xxhash.h" - - /* ************************************* * Basic Types ***************************************/ #ifndef MEM_MODULE # define MEM_MODULE -# if !defined (__VMS) && ( defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ ) +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -250,6 +259,11 @@ FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) return XXH_readLE32_align(ptr, endian, XXH_unaligned); } +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) @@ -263,6 +277,11 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) return XXH_readLE64_align(ptr, endian, XXH_unaligned); } +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + /* ************************************* * Macros @@ -273,17 +292,17 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) /* ************************************* * Constants ***************************************/ -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; -#define PRIME64_1 11400714785074694791ULL -#define PRIME64_2 14029467366897019727ULL -#define PRIME64_3 1609587929392839161ULL -#define PRIME64_4 9650029242287828579ULL -#define PRIME64_5 2870177450012600261ULL +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } @@ -291,6 +310,15 @@ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } /* *************************** * Simple Hash Functions *****************************/ + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; @@ -299,60 +327,40 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH #define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) #ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) - { + if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; } #endif - if (len>=16) - { + if (len>=16) { const BYTE* const limit = bEnd - 16; U32 v1 = seed + PRIME32_1 + PRIME32_2; U32 v2 = seed + PRIME32_2; U32 v3 = seed + 0; U32 v4 = seed - PRIME32_1; - do - { - v1 += XXH_get32bits(p) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p+=4; - v2 += XXH_get32bits(p) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p+=4; - v3 += XXH_get32bits(p) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p+=4; - v4 += XXH_get32bits(p) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p+=4; - } - while (p<=limit); + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p<=limit); h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } - else - { + } else { h32 = seed + PRIME32_5; } h32 += (U32) len; - while (p+4<=bEnd) - { + while (p+4<=bEnd) { h32 += XXH_get32bits(p) * PRIME32_3; h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; p+=4; } - while (p=32) - { + if (len>=32) { const BYTE* const limit = bEnd - 32; U64 v1 = seed + PRIME64_1 + PRIME64_2; U64 v2 = seed + PRIME64_2; U64 v3 = seed + 0; U64 v4 = seed - PRIME64_1; - do - { - v1 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - v2 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - v3 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - v4 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - } - while (p<=limit); + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64 * PRIME64_1 + PRIME64_4; - } - else - { + } else { h64 = seed + PRIME64_5; } h64 += (U64) len; - while (p+8<=bEnd) - { - U64 k1 = XXH_get64bits(p); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1,31); - k1 *= PRIME64_1; + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; p+=8; } - if (p+4<=bEnd) - { + if (p+4<=bEnd) { h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; p+=4; } - while (p= sizeof(XXH32_state_t)); /* A compilation error here means XXH32_state_t is not large enough */ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) @@ -579,7 +531,6 @@ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH64_stateBody_t) >= sizeof(XXH64_state_t)); /* A compilation error here means XXH64_state_t is not large enough */ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) @@ -630,67 +581,37 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void state->total_len += len; - if (state->memsize + len < 16) /* fill in tmp buffer */ - { + if (state->memsize + len < 16) { /* fill in tmp buffer */ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) /* some data left from previous update */ - { + if (state->memsize) { /* some data left from previous update */ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { - const U32* p32 = state->mem32; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v1 = XXH_rotl32(state->v1, 13); - state->v1 *= PRIME32_1; - p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v2 = XXH_rotl32(state->v2, 13); - state->v2 *= PRIME32_1; - p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v3 = XXH_rotl32(state->v3, 13); - state->v3 *= PRIME32_1; - p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v4 = XXH_rotl32(state->v4, 13); - state->v4 *= PRIME32_1; - p32++; + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; } p += 16-state->memsize; state->memsize = 0; } - if (p <= bEnd-16) - { + if (p <= bEnd-16) { const BYTE* const limit = bEnd - 16; U32 v1 = state->v1; U32 v2 = state->v2; U32 v3 = state->v3; U32 v4 = state->v4; - do - { - v1 += XXH_readLE32(p, endian) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p+=4; - v2 += XXH_readLE32(p, endian) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p+=4; - v3 += XXH_readLE32(p, endian) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p+=4; - v4 += XXH_readLE32(p, endian) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p+=4; - } - while (p<=limit); + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); state->v1 = v1; state->v2 = v2; @@ -698,8 +619,7 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void state->v4 = v4; } - if (p < bEnd) - { + if (p < bEnd) { XXH_memcpy(state->mem32, p, bEnd-p); state->memsize = (int)(bEnd-p); } @@ -722,31 +642,26 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { const BYTE * p = (const BYTE*)state->mem32; - const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; - if (state->total_len >= 16) - { + if (state->total_len >= 16) { h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; + } else { + h32 = state->seed + PRIME32_5; } h32 += (U32) state->total_len; - while (p+4<=bEnd) - { + while (p+4<=bEnd) { h32 += XXH_readLE32(p, endian) * PRIME32_3; h32 = XXH_rotl32(h32, 17) * PRIME32_4; p+=4; } - while (ptotal_len += len; - if (state->memsize + len < 32) /* fill in tmp buffer */ - { + if (state->memsize + len < 32) { /* fill in tmp buffer */ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) /* some data left from previous update */ - { + if (state->memsize) { /* tmp buffer is full */ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - { - const U64* p64 = state->mem64; - state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v1 = XXH_rotl64(state->v1, 31); - state->v1 *= PRIME64_1; - p64++; - state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v2 = XXH_rotl64(state->v2, 31); - state->v2 *= PRIME64_1; - p64++; - state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v3 = XXH_rotl64(state->v3, 31); - state->v3 *= PRIME64_1; - p64++; - state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v4 = XXH_rotl64(state->v4, 31); - state->v4 *= PRIME64_1; - p64++; - } + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); p += 32-state->memsize; state->memsize = 0; } - if (p+32 <= bEnd) - { + if (p+32 <= bEnd) { const BYTE* const limit = bEnd - 32; U64 v1 = state->v1; U64 v2 = state->v2; U64 v3 = state->v3; U64 v4 = state->v4; - do - { - v1 += XXH_readLE64(p, endian) * PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - p+=8; - v2 += XXH_readLE64(p, endian) * PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - p+=8; - v3 += XXH_readLE64(p, endian) * PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - p+=8; - v4 += XXH_readLE64(p, endian) * PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - p+=8; - } - while (p<=limit); + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); state->v1 = v1; state->v2 = v2; @@ -850,8 +736,7 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void state->v4 = v4; } - if (p < bEnd) - { + if (p < bEnd) { XXH_memcpy(state->mem64, p, bEnd-p); state->memsize = (int)(bEnd-p); } @@ -874,71 +759,42 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { const BYTE * p = (const BYTE*)state->mem64; - const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; - if (state->total_len >= 32) - { - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64*PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64*PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64*PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64*PRIME64_1 + PRIME64_4; - } - else - { + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { h64 = state->seed + PRIME64_5; } h64 += (U64) state->total_len; - while (p+8<=bEnd) - { - U64 k1 = XXH_readLE64(p, endian); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1,31); - k1 *= PRIME64_1; + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; p+=8; } - if (p+4<=bEnd) - { + if (p+4<=bEnd) { h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; p+=4; } - while (p Date: Sat, 28 May 2016 15:30:01 +0200 Subject: [PATCH 053/208] Added : utility `roundTripCrash` which generates a crash (double-free) on detecting a round-trip corruption --- programs/.gitignore | 1 + programs/Makefile | 12 ++- programs/roundTripCrash.c | 185 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 programs/roundTripCrash.c diff --git a/programs/.gitignore b/programs/.gitignore index 886af777c..87e9e5301 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -9,6 +9,7 @@ zbufftest zbufftest32 datagen paramgrill +roundTripCrash # Object files *.o diff --git a/programs/Makefile b/programs/Makefile index c4e19229a..debabfde6 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -83,7 +83,7 @@ zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ - zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c + zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd_nolegacy : @@ -109,8 +109,8 @@ zstd-compress: $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd-small: clean - CFLAGS="-Os -s" $(MAKE) zstd-frugal +zstd-small: clean + CFLAGS="-Os -s" $(MAKE) zstd-frugal fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -141,12 +141,16 @@ paramgrill : $(ZSTD_FILES) \ datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) +roundTripCrash : $(ZSTD_FILES) \ + roundTripCrash.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + clean: @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ - datagen$(EXT) paramgrill$(EXT) + datagen$(EXT) paramgrill$(EXT) roundTripCrash($EXT) @echo Cleaning completed diff --git a/programs/roundTripCrash.c b/programs/roundTripCrash.c new file mode 100644 index 000000000..1b6e1d7b4 --- /dev/null +++ b/programs/roundTripCrash.c @@ -0,0 +1,185 @@ +/* + roundTripCrash + Copyright (C) Yann Collet 2013-2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net +*/ +/* + This program takes a file in input, + performs a zstd round-trip test (compression - decompress) + compares the result with original + and generates a crash (double free) on corruption detection. +*/ + +/*=========================================== +* Dependencies +*==========================================*/ +#include /* size_t */ +#include /* malloc, free, exit */ +#include /* fprintf */ +#include /* stat */ +#include /* stat */ +#include "zstd.h" + +/** roundTripTest() : +* Compresses `srcBuff` into `compressedBuff`, +* then decompresses `compressedBuff` into `resultBuff`. +* Compression level used is derived from first content byte. +* @return : result of decompression, which should be == `srcSize` +* or an error code if either compression or decompression fails. +* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)` +* for compression to be guaranteed to work */ +static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcBuffSize) +{ + static const int maxClevel = 19; + int const cLevel = (!srcBuffSize) ? 1 : (*(const unsigned char*)srcBuff) % maxClevel; + size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel); + if (ZSTD_isError(cSize)) { + fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize)); + return cSize; + } + return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize); +} + + +static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) +{ + const char* ip1 = (const char*)buff1; + const char* ip2 = (const char*)buff2; + size_t pos; + + for (pos=0; pos= `fileSize` */ +static void loadFile(void* buffer, const char* fileName, size_t fileSize) +{ + FILE* const f = fopen(fileName, "rb"); + if (isDirectory(fileName)) { + fprintf(stderr, "Ignoring %s directory \n", fileName); + exit(2); + } + if (f==NULL) { + fprintf(stderr, "Impossible to open %s \n", fileName); + exit(3); + } + { size_t const readSize = fread(buffer, 1, fileSize, f); + if (readSize != fileSize) { + fprintf(stderr, "Error reading %s \n", fileName); + exit(5); + } } + fclose(f); +} + + +static void fileCheck(const char* fileName) +{ + size_t const fileSize = getFileSize(fileName); + void* buffer = malloc(fileSize); + if (!buffer) { + fprintf(stderr, "not enough memory \n"); + exit(4); + } + loadFile(buffer, fileName, fileSize); + roundTripCheck(buffer, fileSize); + free (buffer); +} + +int main(int argCount, const char** argv) { + if (argCount < 2) { + fprintf(stderr, "Error : no argument : need input file \n"); + exit(9); + } + fileCheck(argv[1]); + fprintf(stderr, "no pb detected\n"); + return 0; +} From ab2ee0804274187f0ae63ec58fdca389bcbfc0b2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 28 May 2016 16:06:29 +0200 Subject: [PATCH 054/208] fixed makefile error in clean target --- programs/Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index debabfde6..78ffefaff 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -134,15 +134,13 @@ zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \ datagen.c xxhash.c zbufftest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -paramgrill : $(ZSTD_FILES) \ - datagen.c xxhash.c paramgrill.c +paramgrill : $(ZSTD_FILES) datagen.c xxhash.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) -roundTripCrash : $(ZSTD_FILES) \ - roundTripCrash.c +roundTripCrash : $(ZSTD_FILES) roundTripCrash.c $(CC) $(FLAGS) $^ -o $@$(EXT) clean: @@ -150,7 +148,7 @@ clean: zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ - datagen$(EXT) paramgrill$(EXT) roundTripCrash($EXT) + datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) @echo Cleaning completed From 0c5e8b17ad8b9b106c751a54e4cf2b65c7a7c42a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 May 2016 01:06:30 +0200 Subject: [PATCH 055/208] moved xxhash to lib/common --- lib/Makefile | 19 ++++++++++--------- {programs => lib/common}/xxhash.c | 0 {programs => lib/common}/xxhash.h | 0 lib/common/zstd.h | 4 ++-- programs/Makefile | 27 ++++++++++++++------------- 5 files changed, 26 insertions(+), 24 deletions(-) rename {programs => lib/common}/xxhash.c (100%) rename {programs => lib/common}/xxhash.h (100%) diff --git a/lib/Makefile b/lib/Makefile index 904366959..b4133a7b5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -2,19 +2,19 @@ # ZSTD library - Makefile # Copyright (C) Yann Collet 2015-2016 # All rights reserved. -# +# # BSD license # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. -# +# # * Redistributions in binary form must reproduce the above copyright notice, this # list of conditions and the following disclaimer in the documentation and/or # other materials provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -25,7 +25,7 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# +# # You can contact the author at : # - ZSTD homepage : http://www.zstd.net # ################################################################ @@ -43,7 +43,7 @@ VERSION?= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I./common +CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -51,10 +51,11 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include -ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c -ZSTDDECOMP_FILES := decompress/zstd_decompress.c common/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c +ZSTDCOMMON_FILES := common/fse_decompress.c common/zstd_common.c common/entropy_common.c common/xxhash.c +ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c +ZSTDDECOMP_FILES := decompress/zstd_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) common/entropy_common.c common/zstd_common.c $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES) +ZSTD_FILES := $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES) ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c ifeq ($(ZSTD_LEGACY_SUPPORT), 0) diff --git a/programs/xxhash.c b/lib/common/xxhash.c similarity index 100% rename from programs/xxhash.c rename to lib/common/xxhash.c diff --git a/programs/xxhash.h b/lib/common/xxhash.h similarity index 100% rename from programs/xxhash.h rename to lib/common/xxhash.h diff --git a/lib/common/zstd.h b/lib/common/zstd.h index dbcf62ba4..3ac4056d1 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -60,8 +60,8 @@ extern "C" { * Version ***************************************/ #define ZSTD_VERSION_MAJOR 0 -#define ZSTD_VERSION_MINOR 6 -#define ZSTD_VERSION_RELEASE 2 +#define ZSTD_VERSION_MINOR 7 +#define ZSTD_VERSION_RELEASE 0 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str diff --git a/programs/Makefile b/programs/Makefile index 78ffefaff..4ffb65c99 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,7 +33,7 @@ DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I../lib/common +CPPFLAGS= -I../lib/common -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -42,11 +42,12 @@ BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib -ZSTDCOMP_FILES := $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c +ZSTDCOMMON_FILES := $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c $(ZSTDDIR)/common/xxhash.c +ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c +ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) +ZSTD_FILES := $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) ifeq ($(ZSTD_LEGACY_SUPPORT), 0) CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 @@ -79,11 +80,11 @@ default: zstd all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32 zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ - zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c + zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ - zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c + zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd_nolegacy : @@ -103,10 +104,10 @@ zstd-pgo : clean zstd zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) -zstd-compress: $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c +zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c +zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) zstd-small: clean @@ -119,22 +120,22 @@ fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) fuzzer : $(ZSTD_FILES) \ - datagen.c xxhash.c fuzzer.c + datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) fuzzer32: $(ZSTD_FILES) \ - datagen.c xxhash.c fuzzer.c + datagen.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) \ - datagen.c xxhash.c zbufftest.c + datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \ - datagen.c xxhash.c zbufftest.c + datagen.c zbufftest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -paramgrill : $(ZSTD_FILES) datagen.c xxhash.c paramgrill.c +paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) datagen : datagen.c datagencli.c From f51e0660f4cbccf5d7df4c273992f0dc01e174ac Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 May 2016 01:39:19 +0200 Subject: [PATCH 056/208] Simplified list of `*.c` files --- lib/Makefile | 14 +++++-------- programs/Makefile | 50 +++++++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index b4133a7b5..75b09e5de 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -43,25 +43,21 @@ VERSION?= $(LIBVER) DESTDIR?= PREFIX ?= /usr/local +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR=$(PREFIX)/include + CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include -ZSTDCOMMON_FILES := common/fse_decompress.c common/zstd_common.c common/entropy_common.c common/xxhash.c -ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c -ZSTDDECOMP_FILES := decompress/zstd_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c -ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c -ZSTD_FILES := $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES) -ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c +ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c ifeq ($(ZSTD_LEGACY_SUPPORT), 0) CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 else -ZSTD_FILES+= $(ZSTD_LEGACY) +ZSTD_FILES+= legacy/*.c CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1 endif diff --git a/programs/Makefile b/programs/Makefile index 4ffb65c99..76367db45 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,30 +33,31 @@ DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I../lib/common -DXXH_NAMESPACE=ZSTD_ +BINDIR = $(PREFIX)/bin +MANDIR = $(PREFIX)/share/man/man1 + +ZSTDDIR = ../lib + +CPPFLAGS= -I$(ZSTDDIR)/common -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) -BINDIR = $(PREFIX)/bin -MANDIR = $(PREFIX)/share/man/man1 -ZSTDDIR = ../lib -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c $(ZSTDDIR)/common/xxhash.c +ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c -ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c +ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/decompress/zstd_decompress.c +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c -ZSTD_FILES := $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) +ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c ifeq ($(ZSTD_LEGACY_SUPPORT), 0) CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 -ZSTD_FILES_LEGACY:= +ZSTDLEGACY_FILES:= else ZSTD_LEGACY_SUPPORT:=1 -CPPFLAGS += -I../lib/legacy -I./legacy -ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \ - $(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c legacy/fileio_legacy.c +CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy +ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c endif @@ -79,11 +80,12 @@ default: zstd all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32 -zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ +zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c + @echo $(ZSTD_FILES) $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) -zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ +zstd32: $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) @@ -104,10 +106,12 @@ zstd-pgo : clean zstd zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) -zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c +zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ + $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c +zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \ + $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) zstd-small: clean @@ -119,20 +123,16 @@ fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -fuzzer : $(ZSTD_FILES) \ - datagen.c fuzzer.c +fuzzer : $(ZSTD_FILES) datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fuzzer32: $(ZSTD_FILES) \ - datagen.c fuzzer.c +fuzzer32: $(ZSTD_FILES) datagen.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) \ - datagen.c zbufftest.c +zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) -zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \ - datagen.c zbufftest.c +zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c @@ -141,7 +141,7 @@ paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) -roundTripCrash : $(ZSTD_FILES) roundTripCrash.c +roundTripCrash : $(ZSTD_FILES) roundTripCrash.c $(CC) $(FLAGS) $^ -o $@$(EXT) clean: From 0d348d46d8e2ba978097e276e2dab5000ab4b43b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 May 2016 02:02:24 +0200 Subject: [PATCH 057/208] fixed cmake fixed playTests --- programs/tests/playTests.sh | 11 +++++------ projects/cmake/.gitignore | 6 ++++++ projects/cmake/lib/CMakeLists.txt | 11 ++++++----- projects/cmake/programs/.gitignore | 8 ++++++++ projects/cmake/programs/CMakeLists.txt | 8 ++++---- 5 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 projects/cmake/.gitignore create mode 100644 projects/cmake/programs/.gitignore diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index dbeb916d0..479ffecae 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -39,13 +39,13 @@ $ZSTD -df tmp.zst # trivial decompression case (overwrite $ECHO "test : too large compression level (must fail)" $ZSTD -99 tmp && die "too large compression level undetected" $ECHO "test : compress to stdout" -$ZSTD tmp -c > tmpCompressed +$ZSTD tmp -c > tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format $ECHO "test : null-length file roundtrip" $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout $ECHO "test : decompress file with wrong suffix (must fail)" $ZSTD -d tmpCompressed && die "wrong suffix error not detected!" -$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout +$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout $ZSTD --decompress tmpCompressed -c > tmpResult $ZSTD --decompress tmpCompressed --stdout > tmpResult if [ "$isWindows" = false ] ; then @@ -124,9 +124,9 @@ $ECHO "\n**** dictionary tests **** " ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 diff -q tmp1 tmp2 $ZSTD --train *.c *.h -o tmpDict -$ZSTD xxhash.c -D tmpDict -of tmp +$ZSTD zstd_compress.c -D tmpDict -of tmp $ZSTD -d tmp -D tmpDict -of result -diff xxhash.c result +diff zstd_compress.c result $ECHO "\n**** multiple files tests **** " @@ -171,7 +171,7 @@ roundTripTest -g127K # TableID==2 roundTripTest -g255K # TableID==1 roundTripTest -g513K # TableID==0 roundTripTest -g512K 6 # greedy, hash chain -roundTripTest -g512K 16 # btlazy2 +roundTripTest -g512K 16 # btlazy2 roundTripTest -g512K 19 # btopt rm tmp* @@ -210,4 +210,3 @@ roundTripTest -g99000000 -P99 20 roundTripTest -g6000000000 -P99 1 rm tmp* - diff --git a/projects/cmake/.gitignore b/projects/cmake/.gitignore new file mode 100644 index 000000000..98f29c79d --- /dev/null +++ b/projects/cmake/.gitignore @@ -0,0 +1,6 @@ +# cmake producted +CMakeCache.txt +CMakeFiles +Makefile +cmake_install.cmake +cmake_uninstall.cmake diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index dae706cbe..834818bfe 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -2,19 +2,19 @@ # zstd - Makefile # Copyright (C) Yann Collet 2014-2016 # All rights reserved. -# +# # BSD license # # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. -# +# # * Redistributions in binary form must reproduce the above copyright notice, this # list of conditions and the following disclaimer in the documentation and/or # other materials provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -25,7 +25,7 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# +# # You can contact the author at : # - zstd homepage : http://www.zstd.net/ # ################################################################ @@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") SET(Sources ${LIBRARY_DIR}/common/entropy_common.c ${LIBRARY_DIR}/common/zstd_common.c + ${LIBRARY_DIR}/common/xxhash.c ${LIBRARY_DIR}/common/fse_decompress.c ${LIBRARY_DIR}/compress/fse_compress.c ${LIBRARY_DIR}/compress/huf_compress.c diff --git a/projects/cmake/programs/.gitignore b/projects/cmake/programs/.gitignore new file mode 100644 index 000000000..81eec349e --- /dev/null +++ b/projects/cmake/programs/.gitignore @@ -0,0 +1,8 @@ +# produced by make +datagen +fullbench +fuzzer +paramgrill +zbufftest +zstd +zstd-frugal diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index 3b4f51d79..f93f3f38c 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -58,13 +58,13 @@ IF (ZSTD_LEGACY_SUPPORT) SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c) ENDIF (ZSTD_LEGACY_SUPPORT) -ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY}) +ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY}) TARGET_LINK_LIBRARIES(zstd libzstd_static) ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c) TARGET_LINK_LIBRARIES(fullbench libzstd_static) -ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/fuzzer.c) +ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fuzzer.c) TARGET_LINK_LIBRARIES(fuzzer libzstd_static) IF (UNIX) @@ -72,10 +72,10 @@ IF (UNIX) TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static) SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT") - ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/zbufftest.c) + ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/zbufftest.c) TARGET_LINK_LIBRARIES(zbufftest libzstd_static) - ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/paramgrill.c) + ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/paramgrill.c) TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c) From c46fb924dfc433d448a1c1aea2d49c2c2195f692 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 May 2016 05:01:04 +0200 Subject: [PATCH 058/208] added dictionary ID (incomplete) --- lib/common/error_private.h | 1 + lib/common/error_public.h | 1 + lib/common/zstd_internal.h | 3 +- lib/common/zstd_static.h | 17 +- lib/compress/zstd_compress.c | 53 ++-- lib/decompress/zstd_decompress.c | 91 ++++-- lib/dictBuilder/zdict.c | 45 ++- programs/fileio.c | 1 + programs/tests/playTests.sh | 4 +- programs/zstdcli.c | 487 ------------------------------- 10 files changed, 136 insertions(+), 567 deletions(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 3f039ae67..6b243c077 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -106,6 +106,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(maxCode): default: return notErrorCode; } diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 6fcf802e0..660b2d3fe 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -58,6 +58,7 @@ typedef enum { ZSTD_error_maxSymbolValue_tooLarge, ZSTD_error_maxSymbolValue_tooSmall, ZSTD_error_dictionary_corrupted, + ZSTD_error_dictionary_wrong, ZSTD_error_maxCode } ZSTD_ErrorCode; diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 2eea5feee..e0a8a82d7 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -63,7 +63,7 @@ #endif #define ZSTD_OPT_NUM (1<<12) -#define ZSTD_DICT_MAGIC 0xEC30A436 +#define ZSTD_DICT_MAGIC 0xEC30A437 #define ZSTD_REP_NUM 3 #define ZSTD_REP_INIT ZSTD_REP_NUM @@ -82,6 +82,7 @@ #define ZSTD_WINDOWLOG_ABSOLUTEMIN 12 static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; +static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; #define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */ static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e0f50cf40..d9f619f26 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -51,7 +51,7 @@ extern "C" { /*-************************************* * Constants ***************************************/ -#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */ /*-************************************* @@ -87,6 +87,7 @@ typedef struct { typedef struct { U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ + U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -103,11 +104,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } * Advanced functions ***************************************/ /*! ZSTD_createCCtx_advanced() : - * Create a ZSTD compression context using external alloc and free functions */ + * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); /*! ZSTD_createDCtx_advanced() : - * Create a ZSTD decompression context using external alloc and free functions */ + * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API unsigned ZSTD_maxCLevel (void); @@ -191,10 +192,14 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci You can then reuse ZSTD_CCtx to compress some new frame. */ -typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams; +typedef struct { + U64 frameContentSize; + U32 windowLog; + U32 dictID; +} ZSTD_frameParams; -#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ -static const size_t ZSTD_frameHeaderSize_min = 5; +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_min = 6; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 91f6bb791..c7ba03053 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -99,6 +99,7 @@ struct ZSTD_CCtx_s U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 loadedDictEnd; U32 stage; /* 0: created; 1: init,dictLoad; 2:started */ + U32 dictID; ZSTD_parameters params; void* workSpace; size_t workSpaceSize; @@ -144,7 +145,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) if (!ctx) return NULL; memset(ctx, 0, sizeof(ZSTD_CCtx)); - ctx->customAlloc = customMem.customAlloc; + ctx->customAlloc = customMem.customAlloc; ctx->customFree = customMem.customFree; return ctx; } @@ -229,6 +230,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface { ZSTD_CCtx* zc = ZSTD_createCCtx(); ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); params.cParams = cParams; params.fParams.contentSizeFlag = 1; ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0); @@ -300,6 +302,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq; zc->stage = 1; + zc->dictID = 0; zc->loadedDictEnd = 0; return 0; @@ -328,7 +331,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); } - /* copy dictionary pointers */ + /* copy dictionary offsets */ dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; dstCCtx->nextSrc = srcCCtx->nextSrc; @@ -337,6 +340,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; + dstCCtx->dictID = srcCCtx->dictID; /* copy entropy tables */ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; @@ -2092,27 +2096,40 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, - ZSTD_parameters params, U64 pledgedSrcSize) + ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; U32 const fcsId = params.fParams.contentSizeFlag ? (pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */ 0; - BYTE const fdescriptor = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */ + BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */ | (fcsId << 6) ); - size_t const hSize = ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; - if (hSize > dstCapacity) return ERROR(dstSize_tooSmall); + U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + BYTE const fCheckByte = (BYTE)(dictIDSizeCode&3); + size_t pos; + + if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); MEM_writeLE32(dst, ZSTD_MAGICNUMBER); - op[4] = fdescriptor; + op[4] = fAllocByte; + op[5] = fCheckByte; + pos = 6; + switch(dictIDSizeCode) + { + default: /* impossible */ + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } switch(fcsId) { default: /* impossible */ case 0 : break; - case 1 : op[5] = (BYTE)(pledgedSrcSize); break; - case 2 : MEM_writeLE16(op+5, (U16)(pledgedSrcSize-256)); break; - case 3 : MEM_writeLE64(op+5, (U64)(pledgedSrcSize)); break; + case 1 : op[pos] = (BYTE)(pledgedSrcSize); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; } - return hSize; + return pos; } @@ -2126,7 +2143,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc, if (zc->stage==0) return ERROR(stage_wrong); if (frame && (zc->stage==1)) { /* copy saved header */ - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize, zc->dictID); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; dst = (char*)dst + fhSize; @@ -2284,13 +2301,14 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t * @return : 0, or an error code */ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { - if ((dict==NULL) || (dictSize<=4)) return 0; + if ((dict==NULL) || (dictSize<=8)) return 0; /* default : dict is pure content */ if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize); + zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4); /* known magic number : dict is parsed for entropy stats and content */ - { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4; + { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8) + 8; if (ZSTD_isError(eSize)) return eSize; return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); } @@ -2303,7 +2321,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - { U32 const hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); + { U32 const hashLog3 = (!pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; } { size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1); @@ -2330,8 +2348,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - params.fParams.contentSizeFlag = 0; ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); @@ -2358,7 +2376,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity) /* special case : empty frame */ if (zc->stage==1) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0, 0); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; op += fhSize; @@ -2434,6 +2452,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); params.fParams.contentSizeFlag = 1; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b670c548c..dfcc6e2d7 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -120,6 +120,7 @@ struct ZSTD_DCtx_s ZSTD_freeFunction customFree; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; + U32 dictID; U32 flagRepeatTable; const BYTE* litPtr; size_t litBufSize; @@ -140,6 +141,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->dictEnd = NULL; dctx->hufTableX4[0] = HufLog; dctx->flagRepeatTable = 0; + dctx->dictID = 0; return 0; } @@ -153,8 +155,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; - if (!customMem.customAlloc && !customMem.customFree) - { + if (!customMem.customAlloc && !customMem.customFree) { dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; dctx->customAlloc = malloc; @@ -169,7 +170,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; - dctx->customAlloc = customMem.customAlloc; + dctx->customAlloc = customMem.customAlloc; dctx->customFree = customMem.customFree; ZSTD_decompressBegin(dctx); @@ -211,12 +212,23 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) /* Frame descriptor - 1 byte, using : + 1 byte - Alloc : bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) - bit 4 : minmatch 4(0) or 3(1) + bit 4 : reserved for windowLog (must be zero) bit 5 : reserved (must be zero) bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes + 1 byte - checker : + bit 0-1 : dictID (0, 1, 2 or 4 bytes) + bit 2-7 : reserved (must be zero) + + Optional : dictID (0, 1, 2 or 4 bytes) + Automatic adaptation + 0 : no dictID + 1 : 1 - 255 + 2 : 256 - 65535 + 4 : all other values + Optional : content size (0, 1, 2 or 8 bytes) 0 : unknown 1 : 0-255 bytes @@ -297,8 +309,10 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); - { U32 const fcsId = (((const BYTE*)src)[4]) >> 6; - return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; } + { U32 const fcsId = (((const BYTE*)src)[4]) >> 6; + U32 const dictID =(((const BYTE*)src)[5]) & 3; + return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId] + ZSTD_did_fieldSize[dictID]; + } } @@ -319,16 +333,27 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (srcSize < fhsize) return fhsize; } memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - { BYTE const frameDesc = ip[4]; - fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ - switch(frameDesc >> 6) /* fcsId */ + { BYTE const allocByte = ip[4]; + BYTE const checkByte = ip[5]; + size_t pos = ZSTD_frameHeaderSize_min; + U32 const dictIDSizeCode = checkByte&3; + fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + switch(dictIDSizeCode) /* fcsId */ + { + default: /* impossible */ + case 0 : fparamsPtr->dictID = 0; break; + case 1 : fparamsPtr->dictID = ip[pos]; pos++; break; + case 2 : fparamsPtr->dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : fparamsPtr->dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(allocByte >> 6) /* fcsId */ { default: /* impossible */ case 0 : fparamsPtr->frameContentSize = 0; break; - case 1 : fparamsPtr->frameContentSize = ip[5]; break; - case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break; - case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break; + case 1 : fparamsPtr->frameContentSize = ip[pos]; break; + case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+pos); break; } } return 0; } @@ -1021,6 +1046,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; + if (dctx->dictID != dctx->fParams.dictID) return ERROR(dictionary_wrong); dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; return 0; @@ -1120,25 +1146,29 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { - size_t eSize; - U32 const magic = MEM_readLE32(dict); - if (magic != ZSTD_DICT_MAGIC) { - /* pure content mode */ + if (dictSize < 8) return ERROR(dictionary_corrupted); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_DICT_MAGIC) { + /* pure content mode */ + ZSTD_refDictContent(dctx, dict, dictSize); + return 0; + } + dctx->dictID = MEM_readLE32((const char*)dict + 4); + + /* load entropy tables */ + dict = (const char*)dict + 8; + dictSize -= 8; + { size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize); + if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + + /* reference dictionary content */ ZSTD_refDictContent(dctx, dict, dictSize); + return 0; } - /* load entropy tables */ - dict = (const char*)dict + 4; - dictSize -= 4; - eSize = ZSTD_loadEntropy(dctx, dict, dictSize); - if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); - - /* reference dictionary content */ - dict = (const char*)dict + eSize; - dictSize -= eSize; - ZSTD_refDictContent(dctx, dict, dictSize); - - return 0; } @@ -1154,4 +1184,3 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t return 0; } - diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 95d291f40..1d3732250 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -61,11 +61,11 @@ #include "fse.h" #include "huf_static.h" #include "zstd_internal.h" +#include "xxhash.h" #include "divsufsort.h" #include "zdict_static.h" - /*-************************************* * Constants ***************************************/ @@ -104,8 +104,7 @@ static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length) { const BYTE* const b = (const BYTE*)ptr; size_t u; - for (u=0; u126) c = '.'; /* non-printable char */ DISPLAYLEVEL(dlevel, "%c", c); @@ -198,8 +197,12 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch) { const char* const pStart = (const char*)pIn; for (;;) { - size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn); - if (!diff) { pIn = (const char*)pIn+sizeof(size_t); pMatch = (const char*)pMatch+sizeof(size_t); continue; } + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { + pIn = (const char*)pIn+sizeof(size_t); + pMatch = (const char*)pMatch+sizeof(size_t); + continue; + } pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); return (size_t)((const char*)pIn - pStart); } @@ -346,9 +349,8 @@ static dictItem ZDICT_analyzePos( maxLength = i; /* reduce maxLength in case of final into repetitive data */ - { - U32 l = (U32)maxLength; - BYTE c = b[pos + maxLength-1]; + { U32 l = (U32)maxLength; + BYTE const c = b[pos + maxLength-1]; while (b[pos+l-2]==c) l--; maxLength = l; } @@ -367,12 +369,10 @@ static dictItem ZDICT_analyzePos( solution.savings = savings[maxLength]; /* mark positions done */ - { - U32 id; - U32 testedPos; + { U32 id; for (id=start; idpos; + U32 const max = table->pos; U32 u; if (!id) return; /* protection, should never happen */ for (u=id; upos; if (nextElt >= maxSize) nextElt = maxSize-1; current = nextElt-1; @@ -530,8 +529,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, DISPLAYLEVEL(2, "finding patterns ... \n"); DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio); - { - U32 cursor; for (cursor=0; cursor < bufferSize; ) { + { U32 cursor; for (cursor=0; cursor < bufferSize; ) { dictItem solution; if (doneMarks[cursor]) { cursor++; continue; } solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio); @@ -542,8 +540,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, } } /* limit dictionary size */ - { - U32 max = dictList->pos; /* convention : nb of useful elts within dictList */ + { U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */ U32 currentSize = 0; U32 n; for (n=1; n>11)); + } + hSize = 8; /* entropic tables */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ DISPLAYLEVEL(2, "statistics ... \n"); - hSize += ZDICT_analyzeEntropy((char*)dictBuffer+4, maxDictSize-4, + hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, maxDictSize-hSize, compressionLevel, samplesBuffer, sampleSizes, nbSamples, (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize); @@ -946,4 +946,3 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, samplesBuffer, samplesSizes, nbSamples, params); } - diff --git a/programs/fileio.c b/programs/fileio.c index eba90d011..d436401d1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -308,6 +308,7 @@ static int FIO_compressFilename_internal(cRess_t ress, /* init */ { ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize); params.fParams.contentSizeFlag = 1; if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index 479ffecae..0c165dba9 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -124,9 +124,9 @@ $ECHO "\n**** dictionary tests **** " ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 diff -q tmp1 tmp2 $ZSTD --train *.c *.h -o tmpDict -$ZSTD zstd_compress.c -D tmpDict -of tmp +$ZSTD zstdcli.c -D tmpDict -of tmp $ZSTD -d tmp -D tmpDict -of result -diff zstd_compress.c result +diff zstdcli.c result $ECHO "\n**** multiple files tests **** " diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 68dd98cb0..e69de29bb 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1,487 +0,0 @@ -/* - zstdcli - Command Line Interface (cli) for zstd - Copyright (C) Yann Collet 2014-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ -/* - Note : this is a user program, not part of libzstd. - The license of libzstd is BSD. - The license of this command line program is GPLv2. -*/ - - -/*-************************************ -* Includes -**************************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ -#include /* strcmp, strlen */ -#include /* toupper */ -#include "fileio.h" -#ifndef ZSTD_NOBENCH -# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ -#endif -#include "zstd_static.h" /* ZSTD_maxCLevel, ZSTD version numbers */ -#ifndef ZSTD_NODICT -# include "dibio.h" -#endif - - - -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif -#endif - - -/*-************************************ -* Constants -**************************************/ -#define COMPRESSOR_NAME "zstd command line interface" -#ifndef ZSTD_VERSION -# define ZSTD_VERSION "v" ZSTD_VERSION_STRING -#endif -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR - -#define ZSTD_EXTENSION ".zst" -#define ZSTD_CAT "zstdcat" -#define ZSTD_UNZSTD "unzstd" - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -static const char* g_defaultDictName = "dictionary"; -static const unsigned g_defaultMaxDictSize = 110 KB; -static const unsigned g_defaultDictCLevel = 5; -static const unsigned g_defaultSelectivityLevel = 9; - - -/*-************************************ -* Display Macros -**************************************/ -#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static FILE* displayOut; -static unsigned displayLevel = 2; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ - - -/*-************************************ -* Command Line -**************************************/ -static int usage(const char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName); - DISPLAY( "\n"); - DISPLAY( "FILE : a filename\n"); - DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); - DISPLAY( "Arguments :\n"); -#ifndef ZSTD_NOCOMPRESS - DISPLAY( " -# : # compression level (1-%u, default:1) \n", ZSTD_maxCLevel()); -#endif -#ifndef ZSTD_NODECOMPRESS - DISPLAY( " -d : decompression \n"); -#endif - DISPLAY( " -D file: use `file` as Dictionary \n"); - DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); - DISPLAY( " -f : overwrite output without prompting \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); - return 0; -} - -static int usage_advanced(const char* programName) -{ - DISPLAY(WELCOME_MESSAGE); - usage(programName); - DISPLAY( "\n"); - DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -t : test compressed file integrity \n"); - DISPLAY( " -v : verbose mode\n"); - DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); - DISPLAY( " -c : force write to standard output, even if it is the console\n"); -#ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories\n"); -#endif -#ifndef ZSTD_NOCOMPRESS - DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); -#endif - DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); -#ifndef ZSTD_NODICT - DISPLAY( "\n"); - DISPLAY( "Dictionary builder :\n"); - DISPLAY( "--train : create a dictionary from a training set of files \n"); - DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName); - DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); - DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); -#endif -#ifndef ZSTD_NOBENCH - DISPLAY( "\n"); - DISPLAY( "Benchmark arguments :\n"); - DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); - DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); - DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); - DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); -#endif - return 0; -} - -static int badusage(const char* programName) -{ - DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (displayLevel >= 1) usage(programName); - return 1; -} - - -static void waitEnter(void) -{ - int unused; - DISPLAY("Press enter to continue...\n"); - unused = getchar(); - (void)unused; -} - - -#define CLEAN_RETURN(i) { operationResult = (i); goto _end; } - -int main(int argCount, const char** argv) -{ - int argNb, - bench=0, - decode=0, - forceStdout=0, - main_pause=0, - nextEntryIsDictionary=0, - operationResult=0, - dictBuild=0, - nextArgumentIsOutFileName=0, - nextArgumentIsMaxDict=0; - unsigned cLevel = 1; - unsigned cLevelLast = 1; - unsigned recursive = 0; - const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ - unsigned filenameIdx = 0; - const char* programName = argv[0]; - const char* outFileName = NULL; - const char* dictFileName = NULL; - char* dynNameSpace = NULL; - unsigned maxDictSize = g_defaultMaxDictSize; - unsigned dictCLevel = g_defaultDictCLevel; - unsigned dictSelect = g_defaultSelectivityLevel; -#ifdef UTIL_HAS_CREATEFILELIST - const char** fileNamesTable = NULL; - char* fileNamesBuf = NULL; - unsigned fileNamesNb; -#endif - - /* init */ - (void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ - (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ - if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } - filenameTable[0] = stdinmark; - displayOut = stderr; - /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ - { size_t pos; - for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } - programName += pos; - } - - /* preset behaviors */ - if (!strcmp(programName, ZSTD_UNZSTD)) decode=1; - if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } - - /* command switches */ - for(argNb=1; argNb='0') && (*argument<='9')) { - cLevel = 0; - while ((*argument >= '0') && (*argument <= '9')) { - cLevel *= 10; - cLevel += *argument - '0'; - argument++; - } - dictCLevel = cLevel; - if (dictCLevel > ZSTD_maxCLevel()) - CLEAN_RETURN(badusage(programName)); - continue; - } -#endif - - switch(argument[0]) - { - /* Display help */ - case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ - case 'H': - case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); - - /* Decoding */ - case 'd': decode=1; argument++; break; - - /* Force stdout, even if stdout==console */ - case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel=1; argument++; break; - - /* Use file content as dictionary */ - case 'D': nextEntryIsDictionary = 1; argument++; break; - - /* Overwrite */ - case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; - - /* Verbose mode */ - case 'v': displayLevel=4; argument++; break; - - /* Quiet mode */ - case 'q': displayLevel--; argument++; break; - - /* keep source file (default anyway, so useless; for gzip/xz compatibility) */ - case 'k': argument++; break; - - /* test compressed file */ - case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break; - - /* dictionary name */ - case 'o': nextArgumentIsOutFileName=1; argument++; break; - - /* recursive */ - case 'r': recursive=1; argument++; break; - -#ifndef ZSTD_NOBENCH - /* Benchmark */ - case 'b': bench=1; argument++; break; - - /* range bench (benchmark only) */ - case 'e': - /* compression Level */ - argument++; - if ((*argument>='0') && (*argument<='9')) { - cLevelLast = 0; - while ((*argument >= '0') && (*argument <= '9')) - cLevelLast *= 10, cLevelLast += *argument++ - '0'; - } - break; - - /* Modify Nb Iterations (benchmark only) */ - case 'i': - { U32 iters= 0; - argument++; - while ((*argument >='0') && (*argument <='9')) - iters *= 10, iters += *argument++ - '0'; - BMK_setNotificationLevel(displayLevel); - BMK_SetNbIterations(iters); - } - break; - - /* cut input into blocks (benchmark only) */ - case 'B': - { size_t bSize = 0; - argument++; - while ((*argument >='0') && (*argument <='9')) - bSize *= 10, bSize += *argument++ - '0'; - if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ - if (toupper(*argument)=='M') bSize<<=20, argument++; - if (toupper(*argument)=='B') argument++; - BMK_setNotificationLevel(displayLevel); - BMK_SetBlockSize(bSize); - } - break; -#endif /* ZSTD_NOBENCH */ - - /* Selection level */ - case 's': argument++; - dictSelect = 0; - while ((*argument >= '0') && (*argument <= '9')) - dictSelect *= 10, dictSelect += *argument++ - '0'; - break; - - /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ - case 'p': argument++; -#ifndef ZSTD_NOBENCH - if ((*argument>='0') && (*argument<='9')) { - int additionalParam = 0; - while ((*argument >= '0') && (*argument <= '9')) - additionalParam *= 10, additionalParam += *argument++ - '0'; - BMK_setAdditionalParam(additionalParam); - } else -#endif - main_pause=1; - break; - /* unknown command */ - default : CLEAN_RETURN(badusage(programName)); - } - } - continue; - } /* if (argument[0]=='-') */ - - if (nextEntryIsDictionary) { - nextEntryIsDictionary = 0; - dictFileName = argument; - continue; - } - - if (nextArgumentIsOutFileName) { - nextArgumentIsOutFileName = 0; - outFileName = argument; - if (!strcmp(outFileName, "-")) outFileName = stdoutmark; - continue; - } - - if (nextArgumentIsMaxDict) { - nextArgumentIsMaxDict = 0; - maxDictSize = 0; - while ((*argument>='0') && (*argument<='9')) - maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++; - if (toupper(*argument)=='K') maxDictSize <<= 10; - continue; - } - - /* add filename to list */ - filenameTable[filenameIdx++] = argument; - } - - /* Welcome message (if verbose) */ - DISPLAYLEVEL(3, WELCOME_MESSAGE); - -#ifdef UTIL_HAS_CREATEFILELIST - if (recursive) { - fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); - if (fileNamesTable) { - unsigned i; - for (i=0; i use stdin and stdout */ - filenameIdx += !filenameIdx; /*< default input is stdin */ - if (!strcmp(filenameTable[0], stdinmark) && !outFileName ) outFileName = stdoutmark; /*< when input is stdin, default output is stdout */ - - /* Check if input/output defined as console; trigger an error in this case */ - if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); - if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode)) - CLEAN_RETURN(badusage(programName)); - - /* user-selected output filename, only possible with a single file */ - if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) { - DISPLAY("Too many files (%u) on the command line. \n", filenameIdx); - CLEAN_RETURN(filenameIdx); - } - - /* No warning message in pipe mode (stdin + stdout) or multiple mode */ - if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; - if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1; - - /* IO Stream/File */ - FIO_setNotificationLevel(displayLevel); -#ifndef ZSTD_NOCOMPRESS - if (!decode) { - if (filenameIdx==1 && outFileName) - operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); - else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel); - } else -#endif - { /* decompression */ -#ifndef ZSTD_NODECOMPRESS - if (filenameIdx==1 && outFileName) - operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); - else - operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName); -#else - DISPLAY("Decompression not supported\n"); -#endif - } - -_end: - if (main_pause) waitEnter(); - free(dynNameSpace); -#ifdef UTIL_HAS_CREATEFILELIST - if (fileNamesTable) - UTIL_freeFileList(fileNamesTable, fileNamesBuf); - else -#endif - free((void*)filenameTable); - return operationResult; -} From fcf2087a15b4cf5151541141d003495803342546 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 May 2016 05:16:05 +0200 Subject: [PATCH 059/208] restored zstdcli.c --- programs/.gitignore | 2 + programs/zstdcli.c | 487 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 489 insertions(+) diff --git a/programs/.gitignore b/programs/.gitignore index 87e9e5301..f7061d3b3 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -38,6 +38,8 @@ roundTripCrash dictionary grillResults.txt _* +tmp* +*.zst # fuzzer afl diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e69de29bb..68dd98cb0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -0,0 +1,487 @@ +/* + zstdcli - Command Line Interface (cli) for zstd + Copyright (C) Yann Collet 2014-2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ +/* + Note : this is a user program, not part of libzstd. + The license of libzstd is BSD. + The license of this command line program is GPLv2. +*/ + + +/*-************************************ +* Includes +**************************************/ +#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include /* strcmp, strlen */ +#include /* toupper */ +#include "fileio.h" +#ifndef ZSTD_NOBENCH +# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ +#endif +#include "zstd_static.h" /* ZSTD_maxCLevel, ZSTD version numbers */ +#ifndef ZSTD_NODICT +# include "dibio.h" +#endif + + + +/*-************************************ +* OS-specific Includes +**************************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE +# include /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif +#endif + + +/*-************************************ +* Constants +**************************************/ +#define COMPRESSOR_NAME "zstd command line interface" +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "v" ZSTD_VERSION_STRING +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR + +#define ZSTD_EXTENSION ".zst" +#define ZSTD_CAT "zstdcat" +#define ZSTD_UNZSTD "unzstd" + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +static const char* g_defaultDictName = "dictionary"; +static const unsigned g_defaultMaxDictSize = 110 KB; +static const unsigned g_defaultDictCLevel = 5; +static const unsigned g_defaultSelectivityLevel = 9; + + +/*-************************************ +* Display Macros +**************************************/ +#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static FILE* displayOut; +static unsigned displayLevel = 2; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ + + +/*-************************************ +* Command Line +**************************************/ +static int usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName); + DISPLAY( "\n"); + DISPLAY( "FILE : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); + DISPLAY( "Arguments :\n"); +#ifndef ZSTD_NOCOMPRESS + DISPLAY( " -# : # compression level (1-%u, default:1) \n", ZSTD_maxCLevel()); +#endif +#ifndef ZSTD_NODECOMPRESS + DISPLAY( " -d : decompression \n"); +#endif + DISPLAY( " -D file: use `file` as Dictionary \n"); + DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); + DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); + return 0; +} + +static int usage_advanced(const char* programName) +{ + DISPLAY(WELCOME_MESSAGE); + usage(programName); + DISPLAY( "\n"); + DISPLAY( "Advanced arguments :\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -t : test compressed file integrity \n"); + DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); + DISPLAY( " -c : force write to standard output, even if it is the console\n"); +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories\n"); +#endif +#ifndef ZSTD_NOCOMPRESS + DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); +#endif + DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); +#ifndef ZSTD_NODICT + DISPLAY( "\n"); + DISPLAY( "Dictionary builder :\n"); + DISPLAY( "--train : create a dictionary from a training set of files \n"); + DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName); + DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); + DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); +#endif +#ifndef ZSTD_NOBENCH + DISPLAY( "\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); + DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); + DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); + DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); +#endif + return 0; +} + +static int badusage(const char* programName) +{ + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (displayLevel >= 1) usage(programName); + return 1; +} + + +static void waitEnter(void) +{ + int unused; + DISPLAY("Press enter to continue...\n"); + unused = getchar(); + (void)unused; +} + + +#define CLEAN_RETURN(i) { operationResult = (i); goto _end; } + +int main(int argCount, const char** argv) +{ + int argNb, + bench=0, + decode=0, + forceStdout=0, + main_pause=0, + nextEntryIsDictionary=0, + operationResult=0, + dictBuild=0, + nextArgumentIsOutFileName=0, + nextArgumentIsMaxDict=0; + unsigned cLevel = 1; + unsigned cLevelLast = 1; + unsigned recursive = 0; + const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + unsigned filenameIdx = 0; + const char* programName = argv[0]; + const char* outFileName = NULL; + const char* dictFileName = NULL; + char* dynNameSpace = NULL; + unsigned maxDictSize = g_defaultMaxDictSize; + unsigned dictCLevel = g_defaultDictCLevel; + unsigned dictSelect = g_defaultSelectivityLevel; +#ifdef UTIL_HAS_CREATEFILELIST + const char** fileNamesTable = NULL; + char* fileNamesBuf = NULL; + unsigned fileNamesNb; +#endif + + /* init */ + (void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ + if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } + filenameTable[0] = stdinmark; + displayOut = stderr; + /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ + { size_t pos; + for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } + programName += pos; + } + + /* preset behaviors */ + if (!strcmp(programName, ZSTD_UNZSTD)) decode=1; + if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } + + /* command switches */ + for(argNb=1; argNb='0') && (*argument<='9')) { + cLevel = 0; + while ((*argument >= '0') && (*argument <= '9')) { + cLevel *= 10; + cLevel += *argument - '0'; + argument++; + } + dictCLevel = cLevel; + if (dictCLevel > ZSTD_maxCLevel()) + CLEAN_RETURN(badusage(programName)); + continue; + } +#endif + + switch(argument[0]) + { + /* Display help */ + case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ + case 'H': + case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); + + /* Decoding */ + case 'd': decode=1; argument++; break; + + /* Force stdout, even if stdout==console */ + case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel=1; argument++; break; + + /* Use file content as dictionary */ + case 'D': nextEntryIsDictionary = 1; argument++; break; + + /* Overwrite */ + case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; + + /* Verbose mode */ + case 'v': displayLevel=4; argument++; break; + + /* Quiet mode */ + case 'q': displayLevel--; argument++; break; + + /* keep source file (default anyway, so useless; for gzip/xz compatibility) */ + case 'k': argument++; break; + + /* test compressed file */ + case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break; + + /* dictionary name */ + case 'o': nextArgumentIsOutFileName=1; argument++; break; + + /* recursive */ + case 'r': recursive=1; argument++; break; + +#ifndef ZSTD_NOBENCH + /* Benchmark */ + case 'b': bench=1; argument++; break; + + /* range bench (benchmark only) */ + case 'e': + /* compression Level */ + argument++; + if ((*argument>='0') && (*argument<='9')) { + cLevelLast = 0; + while ((*argument >= '0') && (*argument <= '9')) + cLevelLast *= 10, cLevelLast += *argument++ - '0'; + } + break; + + /* Modify Nb Iterations (benchmark only) */ + case 'i': + { U32 iters= 0; + argument++; + while ((*argument >='0') && (*argument <='9')) + iters *= 10, iters += *argument++ - '0'; + BMK_setNotificationLevel(displayLevel); + BMK_SetNbIterations(iters); + } + break; + + /* cut input into blocks (benchmark only) */ + case 'B': + { size_t bSize = 0; + argument++; + while ((*argument >='0') && (*argument <='9')) + bSize *= 10, bSize += *argument++ - '0'; + if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ + if (toupper(*argument)=='M') bSize<<=20, argument++; + if (toupper(*argument)=='B') argument++; + BMK_setNotificationLevel(displayLevel); + BMK_SetBlockSize(bSize); + } + break; +#endif /* ZSTD_NOBENCH */ + + /* Selection level */ + case 's': argument++; + dictSelect = 0; + while ((*argument >= '0') && (*argument <= '9')) + dictSelect *= 10, dictSelect += *argument++ - '0'; + break; + + /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ + case 'p': argument++; +#ifndef ZSTD_NOBENCH + if ((*argument>='0') && (*argument<='9')) { + int additionalParam = 0; + while ((*argument >= '0') && (*argument <= '9')) + additionalParam *= 10, additionalParam += *argument++ - '0'; + BMK_setAdditionalParam(additionalParam); + } else +#endif + main_pause=1; + break; + /* unknown command */ + default : CLEAN_RETURN(badusage(programName)); + } + } + continue; + } /* if (argument[0]=='-') */ + + if (nextEntryIsDictionary) { + nextEntryIsDictionary = 0; + dictFileName = argument; + continue; + } + + if (nextArgumentIsOutFileName) { + nextArgumentIsOutFileName = 0; + outFileName = argument; + if (!strcmp(outFileName, "-")) outFileName = stdoutmark; + continue; + } + + if (nextArgumentIsMaxDict) { + nextArgumentIsMaxDict = 0; + maxDictSize = 0; + while ((*argument>='0') && (*argument<='9')) + maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++; + if (toupper(*argument)=='K') maxDictSize <<= 10; + continue; + } + + /* add filename to list */ + filenameTable[filenameIdx++] = argument; + } + + /* Welcome message (if verbose) */ + DISPLAYLEVEL(3, WELCOME_MESSAGE); + +#ifdef UTIL_HAS_CREATEFILELIST + if (recursive) { + fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + if (fileNamesTable) { + unsigned i; + for (i=0; i use stdin and stdout */ + filenameIdx += !filenameIdx; /*< default input is stdin */ + if (!strcmp(filenameTable[0], stdinmark) && !outFileName ) outFileName = stdoutmark; /*< when input is stdin, default output is stdout */ + + /* Check if input/output defined as console; trigger an error in this case */ + if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); + if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode)) + CLEAN_RETURN(badusage(programName)); + + /* user-selected output filename, only possible with a single file */ + if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) { + DISPLAY("Too many files (%u) on the command line. \n", filenameIdx); + CLEAN_RETURN(filenameIdx); + } + + /* No warning message in pipe mode (stdin + stdout) or multiple mode */ + if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; + if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1; + + /* IO Stream/File */ + FIO_setNotificationLevel(displayLevel); +#ifndef ZSTD_NOCOMPRESS + if (!decode) { + if (filenameIdx==1 && outFileName) + operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); + else + operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel); + } else +#endif + { /* decompression */ +#ifndef ZSTD_NODECOMPRESS + if (filenameIdx==1 && outFileName) + operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); + else + operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName); +#else + DISPLAY("Decompression not supported\n"); +#endif + } + +_end: + if (main_pause) waitEnter(); + free(dynNameSpace); +#ifdef UTIL_HAS_CREATEFILELIST + if (fileNamesTable) + UTIL_freeFileList(fileNamesTable, fileNamesBuf); + else +#endif + free((void*)filenameTable); + return operationResult; +} From 33341de7d7e1797cd85625cd7138881a6cc50e4e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 May 2016 23:09:51 +0200 Subject: [PATCH 060/208] fixed fuzzer dictionary test --- programs/fuzzer.c | 60 ++++++++++++++++++------------------- programs/tests/playTests.sh | 6 ++-- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 4b5697c4b..b92c21f1b 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -87,7 +87,7 @@ static clock_t FUZ_clockSpan(clock_t cStart) # define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned int FUZ_rand(unsigned int* src) +unsigned FUZ_rand(unsigned* src) { static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; @@ -115,17 +115,14 @@ static unsigned FUZ_highbit32(U32 v32) static int basicUnitTests(U32 seed, double compressibility) { int testResult = 0; - void* CNBuffer; - void* compressedBuffer; - void* decodedBuffer; + void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + void* const compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); + void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); U32 randState = seed; size_t result, cSize; U32 testNb=0; /* Create compressible test buffer */ - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); - decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); if (!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("Not enough memory, aborting\n"); testResult = 1; @@ -227,19 +224,16 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3; - { ZSTD_parameters p; - p.cParams = ZSTD_getCParams(2, testSize, dictSize); - p.fParams.contentSizeFlag = 1; - { size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1); - if (ZSTD_isError(initResult)) goto _output_error; - } } + { ZSTD_parameters const p = (ZSTD_parameters) { ZSTD_getCParams(2, testSize, dictSize), { 1, 0 } }; + size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1); + if (ZSTD_isError(initResult)) goto _output_error; + } { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig); if (ZSTD_isError(copyResult)) goto _output_error; } cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); if (ZSTD_isError(cSize)) goto _output_error; { ZSTD_frameParams fp; - size_t const gfpResult = ZSTD_getFrameParams(&fp, compressedBuffer, cSize); - if (gfpResult!=0) goto _output_error; + if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error; if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error; } } DISPLAYLEVEL(4, "OK \n"); @@ -258,20 +252,20 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); ((char*)(CNBuffer))[0] = 1; - result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); - if (!ZSTD_isError(result)) goto _output_error; + { size_t const r = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); + if (!ZSTD_isError(r)) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - const size_t blockSize = 100 KB; - const size_t dictSize = 16 KB; + static const size_t blockSize = 100 KB; + static const size_t dictSize = 16 KB; /* basic block compression */ DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); - result = ZSTD_compressBegin(cctx, 5); - if (ZSTD_isError(result)) goto _output_error; + { size_t const r = ZSTD_compressBegin(cctx, 5); + if (ZSTD_isError(r)) goto _output_error; } cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); @@ -575,13 +569,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } if (pos <= cSize) break; /* add noise */ - { U32 nbBits = FUZ_rand(&lseed) % maxNbBits; - size_t mask, noiseStart, noiseLength; - if (nbBits>0) nbBits--; - mask = (1< cSize ) noiseLength = cSize-pos; - noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); + { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; + U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; + size_t const mask = (1<> tmpSparseRegenerated ls -ls tmpSparse* diff tmpSparse2M tmpSparseRegenerated -# rm tmpSparse* +rm tmpSparse* $ECHO "\n**** dictionary tests **** " @@ -123,10 +123,12 @@ $ECHO "\n**** dictionary tests **** " ./datagen -g1M | md5sum > tmp1 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 diff -q tmp1 tmp2 -$ZSTD --train *.c *.h -o tmpDict +$ZSTD --train *.c -o tmpDict $ZSTD zstdcli.c -D tmpDict -of tmp $ZSTD -d tmp -D tmpDict -of result diff zstdcli.c result +$ZSTD --train *.c *.h -o tmpDictC +$ZSTD -d tmp -D tmpDictC -of result && die "wrong dictionary not detected!" $ECHO "\n**** multiple files tests **** " From 7d64cb375c452f8b8fcc0e9b4f022ee366b15c31 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 01:12:13 +0200 Subject: [PATCH 061/208] commented versionsTest, which won't succeed while legacy 0.6 is missing --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index f583a803d..573a77e57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,24 +10,24 @@ before_install: env: - ZSTD_TRAVIS_CI_ENV=travis-install - ZSTD_TRAVIS_CI_ENV=cmaketest - - ZSTD_TRAVIS_CI_ENV=clangtest - - ZSTD_TRAVIS_CI_ENV=gpptest + - ZSTD_TRAVIS_CI_ENV=clangtest + - ZSTD_TRAVIS_CI_ENV=gpptest - ZSTD_TRAVIS_CI_ENV=gnu90test - ZSTD_TRAVIS_CI_ENV=c99test - ZSTD_TRAVIS_CI_ENV=gnu99test - - ZSTD_TRAVIS_CI_ENV=armtest-w-install - - ZSTD_TRAVIS_CI_ENV=test - - ZSTD_TRAVIS_CI_ENV="-C programs test32" + - ZSTD_TRAVIS_CI_ENV=armtest-w-install + - ZSTD_TRAVIS_CI_ENV=test + - ZSTD_TRAVIS_CI_ENV="-C programs test32" - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - - ZSTD_TRAVIS_CI_ENV="-C versionsTest" + #- ZSTD_TRAVIS_CI_ENV="-C versionsTest" - ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=asan - ZSTD_TRAVIS_CI_ENV=asan32 - - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" - + - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" + compiler: gcc -script: +script: - make $ZSTD_TRAVIS_CI_ENV matrix: From 954e2bc48ff4e8e50a4fc0e7b5b050ec864056a2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 01:42:05 +0200 Subject: [PATCH 062/208] added fuzzer test for `ZSTD_compressBegin_advanced()` --- programs/fuzzer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index b92c21f1b..450b5c8b6 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -607,7 +607,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } else { - ZSTD_parameters p = (ZSTD_parameters) { ZSTD_getCParams(cLevel, 0, dictSize), { 0, 0 } }; + ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */ + ZSTD_parameters p = (ZSTD_parameters) { ZSTD_getCParams(cLevel, 0, dictSize), fpar }; size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode)); } From c0a9bf3c2e72814c3832a891b38784fb6a3b629b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 01:56:08 +0200 Subject: [PATCH 063/208] minor code refactoring --- lib/common/zstd_common.c | 2 +- programs/fuzzer.c | 268 ++++++++++++++++++--------------------- 2 files changed, 124 insertions(+), 146 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 8812c0388..2a2c39d60 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -34,7 +34,7 @@ * Dependencies ***************************************/ #include "error_private.h" -#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */ +#include "zstd_static.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode */ #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 450b5c8b6..94cc1aeab 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -40,7 +40,7 @@ #include /* timeb */ #include /* strcmp */ #include /* clock_t */ -#include "zstd_static.h" /* ZSTD_VERSION_STRING */ +#include "zstd_static.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" #define XXH_STATIC_LINKING_ONLY @@ -86,8 +86,8 @@ static clock_t FUZ_clockSpan(clock_t cStart) } -# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned FUZ_rand(unsigned* src) +#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static unsigned FUZ_rand(unsigned* src) { static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; @@ -104,140 +104,137 @@ static unsigned FUZ_highbit32(U32 v32) { unsigned nbBits = 0; if (v32==0) return 0; - while (v32) { - v32 >>= 1; - nbBits ++; - } + while (v32) v32 >>= 1, nbBits++; return nbBits; } +#define CHECKTEST(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error +#define CHECK(fn) { CHECKTEST(err, fn); } +#define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; } static int basicUnitTests(U32 seed, double compressibility) { - int testResult = 0; void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); void* const compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - U32 randState = seed; - size_t result, cSize; + int testResult = 0; U32 testNb=0; + size_t cSize; - /* Create compressible test buffer */ + /* Create compressible noise */ if (!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("Not enough memory, aborting\n"); testResult = 1; goto _end; } - RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState); + RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., seed); /* Basic tests */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH); - result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (ZSTD_isError(result)) goto _output_error; - cSize = result; + /* + { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), + CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1) ); + cSize = r; } + */ + CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), + CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1), + cSize=r ); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH); - result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); - if (ZSTD_isError(result)) goto _output_error; - if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error; + CHECKPLUS( r , ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize), + if (r != COMPRESSIBLE_NOISE_LENGTH) goto _output_error); DISPLAYLEVEL(4, "OK \n"); - { size_t i; - DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); - for (i=0; i have same size */ } DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); - result = ZSTD_decompress_usingDict(dctx, + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize, - CNBuffer, dictSize); - if (ZSTD_isError(result)) goto _output_error; - if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; + CNBuffer, dictSize), + if (r != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error); DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3; { ZSTD_parameters const p = (ZSTD_parameters) { ZSTD_getCParams(2, testSize, dictSize), { 1, 0 } }; - size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1); - if (ZSTD_isError(initResult)) goto _output_error; + CHECK ( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); } - { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig); - if (ZSTD_isError(copyResult)) goto _output_error; } - cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); - if (ZSTD_isError(cSize)) goto _output_error; + CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) ); + + CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), + (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize), + cSize = r); { ZSTD_frameParams fp; if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error; if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error; } } DISPLAYLEVEL(4, "OK \n"); + ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); ZSTD_freeDCtx(dctx); @@ -245,9 +242,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); - result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); - if (!ZSTD_isError(result)) goto _output_error; - if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; + { size_t const r = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); + if (!ZSTD_isError(r)) goto _output_error; + if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); @@ -264,34 +261,28 @@ static int basicUnitTests(U32 seed, double compressibility) /* basic block compression */ DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); - { size_t const r = ZSTD_compressBegin(cctx, 5); - if (ZSTD_isError(r)) goto _output_error; } + CHECK( ZSTD_compressBegin(cctx, 5) ); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); - result = ZSTD_decompressBegin(dctx); - if (ZSTD_isError(result)) goto _output_error; - result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); - if (ZSTD_isError(result)) goto _output_error; - if (result != blockSize) goto _output_error; + CHECK( ZSTD_decompressBegin(dctx) ); + { CHECKPLUS(r, ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize), + if (r != blockSize) goto _output_error); } DISPLAYLEVEL(4, "OK \n"); /* dictionary block compression */ DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++); - result = ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5); - if (ZSTD_isError(result)) goto _output_error; + CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); - result = ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize); - if (ZSTD_isError(result)) goto _output_error; - result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); - if (ZSTD_isError(result)) goto _output_error; - if (result != blockSize) goto _output_error; + CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); + { CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize) ); + if (r != blockSize) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); ZSTD_freeCCtx(cctx); @@ -301,32 +292,29 @@ static int basicUnitTests(U32 seed, double compressibility) /* long rle test */ { size_t sampleSize = 0; DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); - RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState); + RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); sampleSize += 256 KB - 1; - RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState); + RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2); sampleSize += 96 KB; cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); if (ZSTD_isError(cSize)) goto _output_error; - result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize); - if (ZSTD_isError(result)) goto _output_error; - if (result!=sampleSize) goto _output_error; + { CHECKTEST(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); + if (regenSize!=sampleSize) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); } - /* All zeroes test (#137 verif) */ + /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); memset(CNBuffer, 0, ZEROESLENGTH); - result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1); - if (ZSTD_isError(result)) goto _output_error; - cSize = result; + { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); + cSize = r; } DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100); DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); - result = ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize); - if (ZSTD_isError(result)) goto _output_error; - if (result != ZEROESLENGTH) goto _output_error; + { CHECKTEST(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); + if (r != ZEROESLENGTH) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); /* nbSeq limit test */ @@ -354,15 +342,14 @@ static int basicUnitTests(U32 seed, double compressibility) i += 3; } }} DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); - result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19); - if (ZSTD_isError(result)) goto _output_error; - cSize = result; + { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), + CNBuffer, _3BYTESTESTLENGTH, 19) ); + cSize = r; } DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100); DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++); - result = ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize); - if (ZSTD_isError(result)) goto _output_error; - if (result != _3BYTESTESTLENGTH) goto _output_error; + { CHECKTEST(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); + if (r != _3BYTESTESTLENGTH) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); _end: @@ -382,11 +369,11 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max) { const BYTE* b1 = (const BYTE*)buf1; const BYTE* b2 = (const BYTE*)buf2; - size_t i; - for (i=0; i "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } @@ -409,35 +397,29 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD { static const U32 maxSrcLog = 23; static const U32 maxSampleLog = 22; + size_t const srcBufferSize = (size_t)1<= dstBufferSize) maxTestSize = dstBufferSize-1; - sampleSize = FUZ_randomLength(&lseed, maxSampleLog); - sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); - dict = srcBuffer + sampleStart; - dictSize = sampleSize; + dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */ + dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); if (FUZ_rand(&lseed) & 15) { size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } else { ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */ - ZSTD_parameters p = (ZSTD_parameters) { ZSTD_getCParams(cLevel, 0, dictSize), fpar }; + ZSTD_parameters const p = (ZSTD_parameters) { ZSTD_getCParams(cLevel, 0, dictSize), fpar }; size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode)); } @@ -618,21 +598,19 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD XXH64_reset(&xxhState, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; for (totalTestSize=0, cSize=0, n=0 ; n maxTestSize) break; + if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ + if (totalTestSize+segmentSize > maxTestSize) break; - { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize); + { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); cSize += compressResult; } - XXH64_update(&xxhState, srcBuffer+sampleStart, sampleSize); - memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); - totalTestSize += sampleSize; + XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); + memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); + totalTestSize += segmentSize; } { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); @@ -654,7 +632,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD totalCSize += inSize; } CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); - CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") + CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) { From 93fc13ebc8128c02bceb1c7548924d683bf8bd76 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 10:17:55 +0200 Subject: [PATCH 064/208] playTests.sh working with Linux, Windows, OS X --- appveyor.yml | 2 +- programs/Makefile | 2 +- programs/{tests => }/playTests.sh | 14 ++- programs/tests/playTests.bat | 157 ------------------------------ programs/tests/roundTripTest.bat | 8 -- programs/zstdcli.c | 2 +- 6 files changed, 13 insertions(+), 172 deletions(-) rename programs/{tests => }/playTests.sh (95%) mode change 100755 => 100644 delete mode 100644 programs/tests/playTests.bat delete mode 100644 programs/tests/roundTripTest.bat diff --git a/appveyor.yml b/appveyor.yml index 4721fa951..07c559649 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -93,7 +93,7 @@ test_script: CD programs && SET ZSTD=..\projects\zstd.exe && SET DATAGEN=..\projects\datagen.exe && - CALL tests\playTests.bat --test-large-data && + sh playTests.sh --test-large-data && CD .. && projects\fullbench.exe -i1 && projects\fullbench.exe -i1 -P0 && diff --git a/programs/Makefile b/programs/Makefile index c4e19229a..f236459e8 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -204,7 +204,7 @@ endif #------------------------------------------------------------------------ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) zstd-playTests: datagen - ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) + ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) test: test-zstd test-fullbench test-fuzzer test-zbuff diff --git a/programs/tests/playTests.sh b/programs/playTests.sh old mode 100755 new mode 100644 similarity index 95% rename from programs/tests/playTests.sh rename to programs/playTests.sh index 5ff4c4e07..da1fee968 --- a/programs/tests/playTests.sh +++ b/programs/playTests.sh @@ -15,8 +15,8 @@ roundTripTest() { rm -f tmp1 tmp2 $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" - ./datagen $1 $p | md5sum > tmp1 - ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 + ./datagen $1 $p | $MD5SUM > tmp1 + ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | $MD5SUM > tmp2 diff -q tmp1 tmp2 } @@ -29,7 +29,13 @@ case "$OS" in ;; esac +MD5SUM="md5sum" +if [ "$TRAVIS_OS_NAME" == "osx" ]; then + MD5SUM="md5 -r" +fi + $ECHO "\nStarting playTests.sh isWindows=$isWindows" +file $ZSTD [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" @@ -122,8 +128,8 @@ diff tmpSparse2M tmpSparseRegenerated $ECHO "\n**** dictionary tests **** " ./datagen > tmpDict -./datagen -g1M | md5sum > tmp1 -./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 +./datagen -g1M | $MD5SUM > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2 diff -q tmp1 tmp2 $ZSTD --train *.c *.h -o tmpDict $ZSTD xxhash.c -D tmpDict -of tmp diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat deleted file mode 100644 index 5f7ebd685..000000000 --- a/programs/tests/playTests.bat +++ /dev/null @@ -1,157 +0,0 @@ -@echo off -ECHO Start playTests.bat -ECHO ZSTD^: %ZSTD% -ECHO DATAGEN^: %DATAGEN% -if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1 -if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1 - -SET ROUNDTRIPTEST=tests\roundTripTest.bat - -echo. && echo **** simple tests **** -%DATAGEN% > tmp -%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst -%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) -echo test : too large compression level (must fail) -%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1) -echo test : compress to stdout -%ZSTD% tmp -c > tmpCompressed -%ZSTD% tmp --stdout > tmpCompressed && REM long command format -echo test : null-length file roundtrip -echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1) -echo test : decompress file with wrong suffix (must fail) -%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1) -%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout -%ZSTD% --decompress tmpCompressed -c > tmpResult -%ZSTD% --decompress tmpCompressed --stdout > tmpResult -REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout -REM %ZSTD% -d - < tmp.zst > NUL -%ZSTD% -dc < tmp.zst > NUL -%ZSTD% -dc - < tmp.zst > NUL -%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1) -%ZSTD% -q -f tmp -%ZSTD% -q --force tmp -%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) - -echo. && echo **** Pass-Through mode **** -echo "Hello world !" | %ZSTD% -df -echo "Hello world !" | %ZSTD% -dcf - - -echo. && echo **** frame concatenation **** - -echo hello > hello.tmp -echo world! > world.tmp -cat hello.tmp world.tmp > helloworld.tmp -%ZSTD% -c hello.tmp > hello.zstd -%ZSTD% -c world.tmp > world.zstd -cat hello.zstd world.zstd > helloworld.zstd -%ZSTD% -dc helloworld.zstd > result.tmp -cat result.tmp -fc /b helloworld.tmp result.tmp -rm *.tmp *.zstd - -echo frame concatenation tests completed - - -REM echo. && echo **** flush write error test **** - -REM echo echo foo ^| %ZSTD% ^> v:\full -REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1) -REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full" -REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1) - - -echo. && echo **** dictionary tests **** - -%DATAGEN% > tmpDict -%DATAGEN% -g1M | md5sum > tmp1 -%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 -fc tmp1 tmp2 -%ZSTD% --train *.c *.h -o tmpDict -%ZSTD% xxhash.c -D tmpDict -of tmp -%ZSTD% -d tmp -D tmpDict -of result -fc xxhash.c result - - -echo. && echo **** multiple files tests **** - -%DATAGEN% -s1 > tmp1 2> NUL -%DATAGEN% -s2 -g100K > tmp2 2> NUL -%DATAGEN% -s3 -g1M > tmp3 2> NUL -%ZSTD% -f tmp* -echo compress tmp* : -ls -ls tmp* -rm tmp1 tmp2 tmp3 -echo decompress tmp* : -%ZSTD% -df *.zst -ls -ls tmp* - -echo compress tmp* into stdout ^> tmpall : -%ZSTD% -c tmp1 tmp2 tmp3 > tmpall -ls -ls tmp* -echo decompress tmpall* into stdout ^> tmpdec : -cp tmpall tmpall2 -%ZSTD% -dc tmpall* > tmpdec -ls -ls tmp* -echo compress multiple files including a missing one (notHere) : -%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1) - - -echo. && echo **** integrity tests **** -echo test one file (tmp1.zst) -%ZSTD% -t tmp1.zst -%ZSTD% --test tmp1.zst -echo test multiple files (*.zst) -%ZSTD% -t *.zst -echo test good and bad files (*) -%ZSTD% -t * && (echo bad files not detected! && exit /b 1) - - -echo. && echo **** zstd round-trip tests **** - -CALL %ROUNDTRIPTEST% -CALL %ROUNDTRIPTEST% -g15K && REM TableID==3 -CALL %ROUNDTRIPTEST% -g127K && REM TableID==2 -CALL %ROUNDTRIPTEST% -g255K && REM TableID==1 -CALL %ROUNDTRIPTEST% -g513K && REM TableID==0 -CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain -CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2 -CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt - -rm tmp* -echo Param = %1 -if NOT "%1"=="--test-large-data" ( - echo skipping large data tests - exit /b 0 -) - -CALL %ROUNDTRIPTEST% -g270000000 1 -CALL %ROUNDTRIPTEST% -g270000000 2 -CALL %ROUNDTRIPTEST% -g270000000 3 - -CALL %ROUNDTRIPTEST% -g140000000 -P60 4 -CALL %ROUNDTRIPTEST% -g140000000 -P60 5 -CALL %ROUNDTRIPTEST% -g140000000 -P60 6 - -CALL %ROUNDTRIPTEST% -g70000000 -P70 7 -CALL %ROUNDTRIPTEST% -g70000000 -P70 8 -CALL %ROUNDTRIPTEST% -g70000000 -P70 9 - -CALL %ROUNDTRIPTEST% -g35000000 -P75 10 -CALL %ROUNDTRIPTEST% -g35000000 -P75 11 -CALL %ROUNDTRIPTEST% -g35000000 -P75 12 - -CALL %ROUNDTRIPTEST% -g18000000 -P80 13 -CALL %ROUNDTRIPTEST% -g18000000 -P80 14 -CALL %ROUNDTRIPTEST% -g18000000 -P80 15 -CALL %ROUNDTRIPTEST% -g18000000 -P80 16 -CALL %ROUNDTRIPTEST% -g18000000 -P80 17 - -CALL %ROUNDTRIPTEST% -g50000000 -P94 18 -CALL %ROUNDTRIPTEST% -g50000000 -P94 19 - -CALL %ROUNDTRIPTEST% -g99000000 -P99 20 -CALL %ROUNDTRIPTEST% -g6000000000 -P99 1 - -rm tmp* -exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat deleted file mode 100644 index 759891f0b..000000000 --- a/programs/tests/roundTripTest.bat +++ /dev/null @@ -1,8 +0,0 @@ -@echo off -if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) -rm -f tmp1 tmp2 -echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d -%DATAGEN% %1 %P% | md5sum > tmp1 -%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 -fc tmp1 tmp2 -EXIT /B %ERRORLEVEL% diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 68dd98cb0..18a81d661 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -52,7 +52,7 @@ # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE +#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else From 1981d5a7306d845df18708054560f8d1664170cd Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 10:18:56 +0200 Subject: [PATCH 065/208] Travis tests for PowerPC and OS X --- .travis.yml | 123 +++++++++++++++++++++++++++++++++++++++------------- Makefile | 27 +++++++----- 2 files changed, 108 insertions(+), 42 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70f4becee..f130978b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,96 @@ language: c - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq clang - - sudo apt-get install -qq g++-multilib - - sudo apt-get install -qq gcc-multilib - - sudo apt-get install -qq valgrind - -env: - - ZSTD_TRAVIS_CI_ENV=travis-install - - ZSTD_TRAVIS_CI_ENV=cmaketest - - ZSTD_TRAVIS_CI_ENV=clangtest - - ZSTD_TRAVIS_CI_ENV=gpptest - - ZSTD_TRAVIS_CI_ENV=gnu90test - - ZSTD_TRAVIS_CI_ENV=c99test - - ZSTD_TRAVIS_CI_ENV=gnu99test - - ZSTD_TRAVIS_CI_ENV=armtest-w-install - - ZSTD_TRAVIS_CI_ENV=test - - ZSTD_TRAVIS_CI_ENV="-C programs test32" - - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - - ZSTD_TRAVIS_CI_ENV="-C versionsTest" - - ZSTD_TRAVIS_CI_ENV=usan - - ZSTD_TRAVIS_CI_ENV=asan - - ZSTD_TRAVIS_CI_ENV=asan32 - - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" - - ZSTD_TRAVIS_CI_ENV=zlibwrapper - compiler: gcc - -script: - - make $ZSTD_TRAVIS_CI_ENV - matrix: fast_finish: true + include: + # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy" + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + # Standard Ubuntu 12.04 LTS Server Edition 64 bit + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32 + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test + #- os: linux + # sudo: required + # env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=zlibwrapper + # Ubuntu 14.04 LTS Server Edition 64 bit + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper + # OS X Mavericks + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=test + exclude: + - compiler: gcc + +before_install: + - set -e + - | + if [ "$TRAVIS_OS_NAME" == "linux" ]; then + CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l) + echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n" + if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then + sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind + fi + fi + +script: + - make $MAKE_PARAM diff --git a/Makefile b/Makefile index 6be618bc1..3a585da3f 100644 --- a/Makefile +++ b/Makefile @@ -120,11 +120,8 @@ armtest: clean $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" # for Travis CI -arminstall: clean - sudo apt-get install -q qemu - sudo apt-get install -q binfmt-support - sudo apt-get install -q qemu-user-static - sudo apt-get install -q gcc-arm-linux-gnueabi +arminstall: clean + sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi # for Travis CI armtest-w-install: clean arminstall armtest @@ -134,16 +131,24 @@ ppctest: clean $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" # for Travis CI -ppcinstall: clean - sudo apt-get install -q qemu - sudo apt-get install -q binfmt-support - sudo apt-get install -q qemu-user-static - sudo apt-get update -q - sudo apt-get install -q gcc-powerpc-linux-gnu # unfortunately, doesn't work on Travis CI (package not available) +ppcinstall: clean + sudo apt-get update -y -q + sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04 # for Travis CI ppctest-w-install: clean ppcinstall ppctest +ppc64test: clean + $(MAKE) -C $(PRGDIR) datagen # use native, faster + $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" + +ppc64install: clean + sudo apt-get update -y -q + sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu + update-binfmts --displ + +ppc64test-w-install: clean ppc64install ppc64test + usan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" From 957d67803170585df6fc0b1aefa0bc3f91b40b7b Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 10:23:52 +0200 Subject: [PATCH 066/208] update playTests.sh perimissions --- programs/playTests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 programs/playTests.sh diff --git a/programs/playTests.sh b/programs/playTests.sh old mode 100644 new mode 100755 From 1dcb2ff23f6485c7ccfba49c6b164e79a61ae99e Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 11:33:52 +0200 Subject: [PATCH 067/208] improved scripts compatibility --- appveyor.yml | 8 ++++---- programs/playTests.sh | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 07c559649..724d0136e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\ + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\ ) test_script: @@ -91,9 +91,9 @@ test_script: ) - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( CD programs && - SET ZSTD=..\projects\zstd.exe && - SET DATAGEN=..\projects\datagen.exe && - sh playTests.sh --test-large-data && + SET ZSTD=./zstd.exe && + SET DATAGEN=./datagen.exe && + sh -e playTests.sh --test-large-data && CD .. && projects\fullbench.exe -i1 && projects\fullbench.exe -i1 -P0 && diff --git a/programs/playTests.sh b/programs/playTests.sh index da1fee968..189d169f5 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -30,15 +30,15 @@ case "$OS" in esac MD5SUM="md5sum" -if [ "$TRAVIS_OS_NAME" == "osx" ]; then +if [ "$TRAVIS_OS_NAME" = "osx" ]; then MD5SUM="md5 -r" fi -$ECHO "\nStarting playTests.sh isWindows=$isWindows" -file $ZSTD +$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" +file $ZSTD $ECHO "\n**** simple tests **** " ./datagen > tmp From dd97e241a8824b2a0ea44160b3ad666e6bc698c0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 14:05:44 +0200 Subject: [PATCH 068/208] Updated NEWS with @inikep's work --- NEWS | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index c100700aa..db8f9bad6 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ v0.6.2 +New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski New : Support for Sparse File-systems (do not use space for zero-filled sectors) New : Support pass-through mode, when using `-df` +New : Support for custom malloc/free functions v0.6.1 New : zlib wrapper API, thanks to Przemyslaw Skibinski @@ -63,7 +65,7 @@ Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) removed `zstd.c` v0.4.0 -Command line utility compatible with high compression levels +Command line utility compatible with high compression levels Removed zstdhc => merged into zstd Added : ZBUFF API (see zstd_buffered.h) Rolling buffer support @@ -93,7 +95,7 @@ v0.2.2 Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier v0.2.1 -Fix : Read errors, advanced fuzzer tests, by Hanno Böck +Fix : Read errors, advanced fuzzer tests, by Hanno Böck v0.2.0 **Breaking format change** @@ -112,4 +114,3 @@ detects write-flush errors git@github.com:Cyan4973/zstd.git v0.1.0 first release - From d2858e93cbf8a928c72730df0331199f466edeaf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 15:10:09 +0200 Subject: [PATCH 069/208] minor refactoring --- programs/fuzzer.c | 135 ++++++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 72 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 94cc1aeab..27d4bcf10 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -78,7 +78,6 @@ static clock_t g_displayClock = 0; * Fuzzer functions *********************************************************/ #define MIN(a,b) ((a)<(b)?(a):(b)) -#define MAX(a,b) ((a)>(b)?(a):(b)) static clock_t FUZ_clockSpan(clock_t cStart) { @@ -114,9 +113,10 @@ static unsigned FUZ_highbit32(U32 v32) #define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; } static int basicUnitTests(U32 seed, double compressibility) { - void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - void* const compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); - void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + size_t const CNBuffSize = COMPRESSIBLE_NOISE_LENGTH; + void* const CNBuffer = malloc(CNBuffSize); + void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize)); + void* const decodedBuffer = malloc(CNBuffSize); int testResult = 0; U32 testNb=0; size_t cSize; @@ -127,40 +127,35 @@ static int basicUnitTests(U32 seed, double compressibility) testResult = 1; goto _end; } - RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., seed); + RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); /* Basic tests */ - DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH); - /* - { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), - CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1) ); - cSize = r; } - */ - CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), - CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1), + DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize); + CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize), + CNBuffer, CNBuffSize, 1), cSize=r ); - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH); - CHECKPLUS( r , ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize), - if (r != COMPRESSIBLE_NOISE_LENGTH) goto _output_error); + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); + CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize), + if (r != CNBuffSize) goto _output_error); DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); { size_t u; - for (u=0; u have same size */ } - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, + decodedBuffer, CNBuffSize, compressedBuffer, cSize, CNBuffer, dictSize), - if (r != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error); + if (r != CNBuffSize - dictSize) goto _output_error); DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); - { size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3; + { size_t const testSize = CNBuffSize / 3; { ZSTD_parameters const p = (ZSTD_parameters) { ZSTD_getCParams(2, testSize, dictSize), { 1, 0 } }; - CHECK ( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); + CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); } CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) ); CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), - (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize), + (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), cSize = r); { ZSTD_frameParams fp; if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error; @@ -234,7 +229,6 @@ static int basicUnitTests(U32 seed, double compressibility) } } DISPLAYLEVEL(4, "OK \n"); - ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); ZSTD_freeDCtx(dctx); @@ -242,14 +236,14 @@ static int basicUnitTests(U32 seed, double compressibility) /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); - { size_t const r = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); if (!ZSTD_isError(r)) goto _output_error; if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); ((char*)(CNBuffer))[0] = 1; - { size_t const r = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4); if (!ZSTD_isError(r)) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); @@ -268,8 +262,8 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); CHECK( ZSTD_decompressBegin(dctx) ); - { CHECKPLUS(r, ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize), - if (r != blockSize) goto _output_error); } + { CHECKTEST(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + if (r != blockSize) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); /* dictionary block compression */ @@ -281,7 +275,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); - { CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize) ); + { CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); if (r != blockSize) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); @@ -334,12 +328,11 @@ static int basicUnitTests(U32 seed, double compressibility) }} /* randomly fills CNBuffer with prepared 3-bytes sequences */ - { int i; for (i=0; i < _3BYTESTESTLENGTH; ) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ - U32 id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; + { int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ + U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; - i += 3; } }} DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), @@ -408,10 +401,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; - ZSTD_CCtx* refCtx = ZSTD_createCCtx(); - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - clock_t startClock = clock(); + ZSTD_CCtx* const refCtx = ZSTD_createCCtx(); + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + clock_t const startClock = clock(); clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC; /* allocation */ @@ -439,7 +432,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) { size_t sampleSize, maxTestSize, totalTestSize; size_t cSize, totalCSize, totalGenSize; - U32 sampleSizeLog, nbChunks, n; XXH64_state_t xxhState; U64 crcOrig; BYTE* sampleBuffer; @@ -469,18 +461,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } /* select src segment */ - sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; - sampleSize = FUZ_rLogLength(&lseed, sampleSizeLog); + sampleSize = FUZ_randomLength(&lseed, maxSampleLog); /* create sample buffer (to catch read error with valgrind & sanitizers) */ sampleBuffer = (BYTE*)malloc(sampleSize); - CHECK (sampleBuffer==NULL, "not enough memory for sample buffer"); + CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression tests */ - { int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (sampleSizeLog/3))) + 1; + { unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); @@ -494,9 +485,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); } { U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } - } - } - + } } /* frame header decompression test */ { ZSTD_frameParams dParams; @@ -518,7 +507,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD /* truncated src decompression test */ { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ size_t const tooSmallSize = cSize - missing; - void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ + void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ CHECK(cBufferTooSmall == NULL, "not enough memory !"); memcpy(cBufferTooSmall, cBuffer, tooSmallSize); { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); @@ -596,22 +585,24 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); } } XXH64_reset(&xxhState, 0); - nbChunks = (FUZ_rand(&lseed) & 127) + 2; - for (totalTestSize=0, cSize=0, n=0 ; n maxTestSize) break; + if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ + if (totalTestSize+segmentSize > maxTestSize) break; + + { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); + CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); + cSize += compressResult; + } + XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); + memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); + totalTestSize += segmentSize; + } } - { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); - CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); - cSize += compressResult; - } - XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); - memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); - totalTestSize += segmentSize; - } { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); cSize += flushResult; @@ -637,7 +628,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) { size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize); - CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", + CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", (U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]); } } } /* for ( ; (testNb <= nbTests) */ From 30009521d7f988cb78da49934a9a3d6542e831f1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 16:17:33 +0200 Subject: [PATCH 070/208] fuzzer tests dictBuilder. Added : ability to not store dictID during compression; decompression doesn't check dictID then --- lib/decompress/zstd_decompress.c | 8 ++-- programs/Makefile | 7 +-- programs/fuzzer.c | 78 ++++++++++++++++++++++++++++---- 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index dfcc6e2d7..b0831617c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -362,10 +362,11 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t /** ZSTD_decodeFrameHeader() : * `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ -static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize) +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize) { - size_t const result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize); - if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); + size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); + if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); return result; } @@ -1046,7 +1047,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; - if (dctx->dictID != dctx->fParams.dictID) return ERROR(dictionary_wrong); dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; return 0; diff --git a/programs/Makefile b/programs/Makefile index 76367db45..377e49475 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -82,7 +82,6 @@ all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzze zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c - @echo $(ZSTD_FILES) $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd32: $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ @@ -123,10 +122,12 @@ fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -fuzzer : $(ZSTD_FILES) datagen.c fuzzer.c +fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder +fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fuzzer32: $(ZSTD_FILES) datagen.c fuzzer.c +fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder +fuzzer32: $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 27d4bcf10..c5d5f46a7 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -41,6 +41,7 @@ #include /* strcmp */ #include /* clock_t */ #include "zstd_static.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */ +#include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" #define XXH_STATIC_LINKING_ONLY @@ -54,7 +55,6 @@ #define MB *(1U<<20) #define GB *(1U<<30) -static const size_t COMPRESSIBLE_NOISE_LENGTH = 10 MB; /* capital, used to be a macro */ static const U32 FUZ_compressibility_default = 50; static const U32 nbTestsDefault = 30000; @@ -113,7 +113,7 @@ static unsigned FUZ_highbit32(U32 v32) #define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; } static int basicUnitTests(U32 seed, double compressibility) { - size_t const CNBuffSize = COMPRESSIBLE_NOISE_LENGTH; + size_t const CNBuffSize = 5 MB; void* const CNBuffer = malloc(CNBuffSize); void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize)); void* const decodedBuffer = malloc(CNBuffSize); @@ -176,7 +176,7 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) ); DISPLAYLEVEL(4, "OK \n"); - DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); + DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++); cSize = 0; CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize), (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), @@ -185,7 +185,7 @@ static int basicUnitTests(U32 seed, double compressibility) cSize += r); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); + DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, @@ -234,6 +234,68 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeDCtx(dctx); } + /* Dictionary and dictBuilder tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t dictSize = 16 KB; + void* dictBuffer = malloc(dictSize); + size_t const totalSampleSize = 1 MB; + size_t const sampleUnitSize = 8 KB; + U32 const nbSamples = totalSampleSize / sampleUnitSize; + size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); + + if (dictBuffer==NULL || samplesSizes==NULL) { + free(dictBuffer); + free(samplesSizes); + goto _output_error; + } + + DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++); + { U32 u; for (u=0; u Date: Mon, 30 May 2016 18:20:46 +0200 Subject: [PATCH 071/208] Fixed visual projects Signed-off-by: Yann Collet --- .gitattributes | 1 + programs/fuzzer.c | 18 ++- projects/VS2008/fuzzer/fuzzer.vcproj | 180 ++++++++++++++---------- projects/VS2008/zstd/zstd.vcproj | 156 ++++++++++---------- projects/VS2008/zstdlib/zstdlib.vcproj | 156 ++++++++++---------- projects/VS2010/fuzzer/fuzzer.vcxproj | 19 ++- projects/VS2010/zstd/zstd.vcxproj | 4 +- projects/VS2010/zstdlib/zstdlib.vcxproj | 2 + projects/cmake/build/.keep | 0 projects/cmake/programs/CMakeLists.txt | 2 +- 10 files changed, 295 insertions(+), 243 deletions(-) delete mode 100644 projects/cmake/build/.keep diff --git a/.gitattributes b/.gitattributes index f314af9ca..da0f7a530 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,6 +12,7 @@ # Visual Studio *.sln text eol=crlf *.vcxproj* text eol=crlf +*.vcproj* text eol=crlf *.suo binary *.rc binary diff --git a/programs/fuzzer.c b/programs/fuzzer.c index c5d5f46a7..fef2edaa9 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -28,7 +28,7 @@ #ifdef _MSC_VER /* Visual Studio */ # define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif @@ -215,7 +215,10 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = CNBuffSize / 3; - { ZSTD_parameters const p = (ZSTD_parameters) { ZSTD_getCParams(2, testSize, dictSize), { 1, 0 } }; + { ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize); + ZSTD_frameParameters const fPar = { 1 , 0 }; + ZSTD_parameters p; + p.cParams = cPar; p.fParams = fPar; CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); } CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) ); @@ -274,7 +277,9 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++); { ZSTD_frameParameters const fParams = { 0, 1 /*NoDictID*/ }; - ZSTD_parameters const p = { ZSTD_getCParams(3, CNBuffSize, dictSize), fParams }; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize); + ZSTD_parameters p; + p.cParams = cParams; p.fParams = fParams; cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), CNBuffer, CNBuffSize, dictBuffer, dictSize, p); @@ -638,9 +643,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } else { + ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */ - ZSTD_parameters const p = (ZSTD_parameters) { ZSTD_getCParams(cLevel, 0, dictSize), fpar }; - size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0); + ZSTD_parameters p; + size_t errorCode; + p.cParams = cPar; p.fParams = fpar; + errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode)); } { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx); diff --git a/projects/VS2008/fuzzer/fuzzer.vcproj b/projects/VS2008/fuzzer/fuzzer.vcproj index 9e572da26..ab0bab2c6 100644 --- a/projects/VS2008/fuzzer/fuzzer.vcproj +++ b/projects/VS2008/fuzzer/fuzzer.vcproj @@ -44,7 +44,7 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + @@ -357,7 +361,11 @@ > + + + + @@ -410,10 +422,22 @@ RelativePath="..\..\..\lib\common\mem.h" > + + + + + + diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj index 9e4626500..64949899c 100644 --- a/projects/VS2008/zstd/zstd.vcproj +++ b/projects/VS2008/zstd/zstd.vcproj @@ -93,81 +93,6 @@ Name="VCPostBuildEventTool" /> - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2008/zstdlib/zstdlib.vcproj b/projects/VS2008/zstdlib/zstdlib.vcproj index 2a4779aac..1bb1c3b99 100644 --- a/projects/VS2008/zstdlib/zstdlib.vcproj +++ b/projects/VS2008/zstdlib/zstdlib.vcproj @@ -92,80 +92,6 @@ Name="VCPostBuildEventTool" /> - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + @@ -418,6 +422,10 @@ RelativePath="..\..\..\lib\common\mem.h" > + + diff --git a/projects/VS2010/fuzzer/fuzzer.vcxproj b/projects/VS2010/fuzzer/fuzzer.vcxproj index 047d5fd55..bdda5f3fc 100644 --- a/projects/VS2010/fuzzer/fuzzer.vcxproj +++ b/projects/VS2010/fuzzer/fuzzer.vcxproj @@ -1,4 +1,4 @@ - + @@ -66,24 +66,24 @@ true false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); $(Platform)\$(Configuration)\ true false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); $(Platform)\$(Configuration)\ false false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); @@ -158,30 +158,35 @@ + + + - + + + + - diff --git a/projects/VS2010/zstd/zstd.vcxproj b/projects/VS2010/zstd/zstd.vcxproj index 8fff6320d..9e51cd56a 100644 --- a/projects/VS2010/zstd/zstd.vcxproj +++ b/projects/VS2010/zstd/zstd.vcxproj @@ -20,6 +20,7 @@ + @@ -41,10 +42,10 @@ - + @@ -69,7 +70,6 @@ - diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj index e8625a501..25d3ca1d7 100644 --- a/projects/VS2010/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -20,6 +20,7 @@ + @@ -39,6 +40,7 @@ + diff --git a/projects/cmake/build/.keep b/projects/cmake/build/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index f93f3f38c..bc8a51674 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -44,7 +44,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR}) IF (WORKAROUND_OUTDATED_CODE_STYLE) # Define library directory, where sources and header files are located SET(LIBRARY_DIR ${ROOT_DIR}/lib) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common) + INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder) ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) IF (ZSTD_LEGACY_SUPPORT) From 118bfc4ccb1dd91fcff514e4753d57f68d5d6f8e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 18:50:28 +0200 Subject: [PATCH 072/208] fixed cmaketest --- Makefile | 24 +++++++++++++----------- projects/.gitignore | 1 + 2 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 projects/.gitignore diff --git a/Makefile b/Makefile index 0894f75d7..65af4c907 100644 --- a/Makefile +++ b/Makefile @@ -2,19 +2,19 @@ # zstd - Makefile # Copyright (C) Yann Collet 2014-2016 # All rights reserved. -# +# # BSD license # # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. -# +# # * Redistributions in binary form must reproduce the above copyright notice, this # list of conditions and the following disclaimer in the documentation and/or # other materials provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -25,7 +25,7 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# +# # You can contact the author at : # - zstd homepage : http://www.zstd.net/ # ################################################################ @@ -44,7 +44,7 @@ endif default: zstdprogram -all: +all: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ @@ -76,6 +76,8 @@ travis-install: $(MAKE) install PREFIX=~/install_test_dir cmaketest: + rm -rf projects/cmake/build + mkdir projects/cmake/build cd projects/cmake/build ; cmake .. ; $(MAKE) clangtest: clean @@ -114,8 +116,8 @@ armtest: clean $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" # for Travis CI -arminstall: clean - sudo apt-get install -q qemu +arminstall: clean + sudo apt-get install -q qemu sudo apt-get install -q binfmt-support sudo apt-get install -q qemu-user-static sudo apt-get install -q gcc-arm-linux-gnueabi @@ -125,11 +127,11 @@ armtest-w-install: clean arminstall armtest ppctest: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" # for Travis CI -ppcinstall: clean - sudo apt-get install -q qemu +ppcinstall: clean + sudo apt-get install -q qemu sudo apt-get install -q binfmt-support sudo apt-get install -q qemu-user-static sudo apt-get update -q diff --git a/projects/.gitignore b/projects/.gitignore new file mode 100644 index 000000000..378eac25d --- /dev/null +++ b/projects/.gitignore @@ -0,0 +1 @@ +build From 815580a53864a17d5f486db6b9e4565ce59f5fa4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 18:57:39 +0200 Subject: [PATCH 073/208] added `cmake --version` to check cmake version on Travis --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 65af4c907..8d7ee5a5a 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,7 @@ travis-install: $(MAKE) install PREFIX=~/install_test_dir cmaketest: + cmake --version rm -rf projects/cmake/build mkdir projects/cmake/build cd projects/cmake/build ; cmake .. ; $(MAKE) From 290aaa75212334b18b0ef36c679d4d4a0704acca Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 21:18:52 +0200 Subject: [PATCH 074/208] Added : ability to manually select the dictionary ID of a newly created dictionary --- lib/dictBuilder/zdict.c | 24 +++++------ lib/dictBuilder/zdict_static.h | 5 ++- programs/dibio.c | 78 +++++++++++++++++----------------- programs/tests/playTests.sh | 2 + programs/zstd.1 | 14 ++++-- programs/zstdcli.c | 15 ++++++- 6 files changed, 81 insertions(+), 57 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 1d3732250..be141ce10 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -819,10 +819,10 @@ size_t ZDICT_trainFromBuffer_unsafe( ZDICT_params_t params) { U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16)); - dictItem* dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); + dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); unsigned selectivity = params.selectivityLevel; unsigned compressionLevel = params.compressionLevel; - size_t targetDictSize = maxDictSize; + size_t const targetDictSize = maxDictSize; size_t sBuffSize; size_t dictSize = 0; @@ -865,17 +865,16 @@ size_t ZDICT_trainFromBuffer_unsafe( /* create dictionary */ { U32 dictContentSize = ZDICT_dictSize(dictList); size_t hSize; - BYTE* ptr; - U32 u; /* build dict content */ - ptr = (BYTE*)dictBuffer + maxDictSize; - for (u=1; upos; u++) { - U32 l = dictList[u].length; - ptr -= l; - if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */ - memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); - } + { U32 u; + BYTE* ptr = (BYTE*)dictBuffer + maxDictSize; + for (u=1; upos; u++) { + U32 l = dictList[u].length; + ptr -= l; + if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */ + memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); + } } /* fast mode dict content */ if (selectivity==1) { /* note could also be used to complete a dictionary, but not necessarily better */ @@ -888,7 +887,8 @@ size_t ZDICT_trainFromBuffer_unsafe( /* dictionary header */ MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); { U64 const randomID = XXH64((char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize, 0); - MEM_writeLE32((char*)dictBuffer+4, (U32)(randomID>>11)); + U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11); + MEM_writeLE32((char*)dictBuffer+4, dictID); } hSize = 8; diff --git a/lib/dictBuilder/zdict_static.h b/lib/dictBuilder/zdict_static.h index e5f909ac7..e34e6c07d 100644 --- a/lib/dictBuilder/zdict_static.h +++ b/lib/dictBuilder/zdict_static.h @@ -54,7 +54,8 @@ typedef struct { unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */ unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */ unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned reserved[3]; /* space for future parameters */ + unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ + unsigned reserved[2]; /* space for future parameters */ } ZDICT_params_t; @@ -65,7 +66,7 @@ typedef struct { Same as ZDICT_trainFromBuffer() with control over more parameters. `parameters` is optional and can be provided with values set to 0 to mean "default". @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`) - or an error code, which can be tested by DiB_isError(). + or an error code, which can be tested by ZDICT_isError(). note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel() */ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, diff --git a/programs/dibio.c b/programs/dibio.c index 23f3c8176..d23476e31 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -101,27 +101,30 @@ const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCo /* ******************************************************** * File related operations **********************************************************/ -static void DiB_loadFiles(void* buffer, size_t bufferSize, - size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) +/** DiB_loadFiles() : +* @return : nb of files effectively loaded into `buffer` */ +static unsigned DiB_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, + const char** fileNamesTable, unsigned nbFiles) { - char* buff = (char*)buffer; + char* const buff = (char*)buffer; size_t pos = 0; unsigned n; for (n=0; n bufferSize-pos ? 0 : fs64); + FILE* const f = fopen(fileNamesTable[n], "rb"); if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); - if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */ - readSize = fread(buff+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); - pos += readSize; - fileSizes[n] = (size_t)fileSize; + { size_t const readSize = fread(buff+pos, 1, fileSize, f); + if (readSize != fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + pos += readSize; } + fileSizes[n] = fileSize; fclose(f); + if (fileSize == 0) break; /* stop there, not enough memory to load all files */ } + return n; } @@ -130,7 +133,7 @@ static void DiB_loadFiles(void* buffer, size_t bufferSize, **********************************************************/ static size_t DiB_findMaxMem(unsigned long long requiredMem) { - size_t step = 8 MB; + size_t const step = 8 MB; void* testmem = NULL; requiredMem = (((requiredMem >> 23) + 1) << 23); @@ -162,7 +165,7 @@ static void DiB_fillNoise(void* buffer, size_t length) static void DiB_saveDict(const char* dictFileName, const void* buff, size_t buffSize) { - FILE* f = fopen(dictFileName, "wb"); + FILE* const f = fopen(dictFileName, "wb"); if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); { size_t const n = fwrite(buff, 1, buffSize, f); @@ -185,47 +188,44 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity, ZDICT_params_t parameters); +#define MIN(a,b) ((a)<(b)?(a):(b)) int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, ZDICT_params_t params) { - void* srcBuffer; - size_t benchedSize; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - unsigned long long totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - void* dictBuffer = malloc(maxDictSize); - size_t dictSize; + void* const dictBuffer = malloc(maxDictSize); + size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT; + size_t const benchedSize = MIN (maxMem, (size_t)totalSizeToLoad); + void* const srcBuffer = malloc(benchedSize+NOISELENGTH); int result = 0; + /* Checks */ + if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ + /* init */ g_displayLevel = params.notificationLevel; - benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT; - if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize < totalSizeToLoad) DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20)); - /* Memory allocation & restrictions */ - srcBuffer = malloc(benchedSize+NOISELENGTH); /* + noise */ - if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ - /* Load input buffer */ - DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); + nbFiles = DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ - /* call buffer version */ - dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize, - srcBuffer, fileSizes, nbFiles, - params); - if (ZDICT_isError(dictSize)) { - DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ - result = 1; - goto _cleanup; + { size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize, + srcBuffer, fileSizes, nbFiles, + params); + if (ZDICT_isError(dictSize)) { + DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ + result = 1; + goto _cleanup; + } + /* save dict */ + DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); + DiB_saveDict(dictFileName, dictBuffer, dictSize); } - /* save dict */ - DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); - DiB_saveDict(dictFileName, dictBuffer, dictSize); - /* clean up */ _cleanup: free(srcBuffer); diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index 172905239..ae7823033 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -129,6 +129,8 @@ $ZSTD -d tmp -D tmpDict -of result diff zstdcli.c result $ZSTD --train *.c *.h -o tmpDictC $ZSTD -d tmp -D tmpDictC -of result && die "wrong dictionary not detected!" +$ZSTD --train *.c --dictID 1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "\n**** multiple files tests **** " diff --git a/programs/zstd.1 b/programs/zstd.1 index 27d607f5c..1bab57abb 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -18,11 +18,11 @@ .PP .B unzstd is equivalent to -.BR "zstd \-d" +.BR "zstd \-d" .br .B zstdcat is equivalent to -.BR "zstd \-dc" +.BR "zstd \-dc" .br .SH DESCRIPTION @@ -90,7 +90,15 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB). dictionary saved into `file` (default: dictionary) .TP .B \--maxdict # - limit dictionary to specified size (default : 112640) + limit dictionary to specified size (default : 112640) +.TP +.B \--dictID # + A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary. + By default, zstd will create a 4-bytes random number ID. + It's possible to give a precise number instead. + Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, + and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default. + However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries. .TP .B \-s# dictionary selectivity level (default: 9) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 68dd98cb0..74f3878be 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -143,6 +143,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName); DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); + DISPLAY( "--dictID: force dictionary ID to specified value (default: random)\n"); #endif #ifndef ZSTD_NOBENCH DISPLAY( "\n"); @@ -185,7 +186,8 @@ int main(int argCount, const char** argv) operationResult=0, dictBuild=0, nextArgumentIsOutFileName=0, - nextArgumentIsMaxDict=0; + nextArgumentIsMaxDict=0, + nextArgumentIsDictID=0; unsigned cLevel = 1; unsigned cLevelLast = 1; unsigned recursive = 0; @@ -196,6 +198,7 @@ int main(int argCount, const char** argv) const char* dictFileName = NULL; char* dynNameSpace = NULL; unsigned maxDictSize = g_defaultMaxDictSize; + unsigned dictID = 0; unsigned dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; #ifdef UTIL_HAS_CREATEFILELIST @@ -236,6 +239,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; } if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } + if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } @@ -393,6 +397,14 @@ int main(int argCount, const char** argv) continue; } + if (nextArgumentIsDictID) { + nextArgumentIsDictID = 0; + dictID = 0; + while ((*argument>='0') && (*argument<='9')) + dictID = dictID * 10 + (*argument - '0'), argument++; + continue; + } + /* add filename to list */ filenameTable[filenameIdx++] = argument; } @@ -429,6 +441,7 @@ int main(int argCount, const char** argv) dictParams.compressionLevel = dictCLevel; dictParams.selectivityLevel = dictSelect; dictParams.notificationLevel = displayLevel; + dictParams.dictID = dictID; DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams); #endif goto _end; From b81cbbade1c38836ff6ab39ea263680a6edcec34 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 May 2016 22:29:45 +0200 Subject: [PATCH 075/208] fixed minor conversion warning --- programs/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index fef2edaa9..7fbf906ce 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -244,7 +244,7 @@ static int basicUnitTests(U32 seed, double compressibility) void* dictBuffer = malloc(dictSize); size_t const totalSampleSize = 1 MB; size_t const sampleUnitSize = 8 KB; - U32 const nbSamples = totalSampleSize / sampleUnitSize; + U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); if (dictBuffer==NULL || samplesSizes==NULL) { From 6381e99fb21624887ba5224ed10a82f78173fc86 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 31 May 2016 02:29:45 +0200 Subject: [PATCH 076/208] Added : ability to create compressed frames without dictID --- programs/bench.h | 6 ++++-- programs/fileio.c | 15 +++++++-------- programs/fileio.h | 1 + programs/tests/playTests.sh | 15 ++++++++++++--- programs/zstdcli.c | 14 +++++++++----- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/programs/bench.h b/programs/bench.h index 74ac20f9a..1a3156404 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -21,10 +21,11 @@ You can contact the author at : - ZSTD homepage : http://www.zstd.net/ */ -#pragma once +#ifndef BENCH_H_121279284357 +#define BENCH_H_121279284357 +#include -/* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast); @@ -34,3 +35,4 @@ void BMK_SetBlockSize(size_t blockSize); void BMK_setAdditionalParam(int additionalParam); void BMK_setNotificationLevel(unsigned level); + #endif /* BENCH_H_121279284357 */ diff --git a/programs/fileio.c b/programs/fileio.c index d436401d1..fc3022080 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -133,6 +133,8 @@ static U32 g_maxWLog = 23; void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } +static U32 g_dictIDFlag = 1; +void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } /*-************************************* @@ -186,7 +188,7 @@ static FILE* FIO_openDstFile(const char* dstFileName) DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); } } else { - if (!g_overwrite) { /* Check if destination file already exists */ + if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ f = fopen( dstFileName, "rb" ); if (f != 0) { /* dest file exists, prompt for overwrite authorization */ fclose(f); @@ -311,6 +313,7 @@ static int FIO_compressFilename_internal(cRess_t ress, memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize); params.fParams.contentSizeFlag = 1; + params.fParams.noDictIDFlag = !g_dictIDFlag; if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); } @@ -410,14 +413,10 @@ static int FIO_compressFilename_extRess(cRess_t ress, int FIO_compressFilename(const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel) { - clock_t start; - cRess_t ress; + clock_t const start = clock(); + cRess_t const ress = FIO_createCResources(dictFileName); int issueWithSrcFile = 0; - /* Init */ - start = clock(); - ress = FIO_createCResources(dictFileName); - issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel); FIO_freeCResources(ress); @@ -702,7 +701,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); + DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize); /* Close */ fclose(srcFile); diff --git a/programs/fileio.h b/programs/fileio.h index 6e7912380..5a9cdc1ce 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -48,6 +48,7 @@ void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ +void FIO_setDictIDFlag(unsigned dictIDFlag); /*-************************************* diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index ae7823033..42abc3a81 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -123,14 +123,23 @@ $ECHO "\n**** dictionary tests **** " ./datagen -g1M | md5sum > tmp1 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 diff -q tmp1 tmp2 +$ECHO "Create first dictionary" $ZSTD --train *.c -o tmpDict -$ZSTD zstdcli.c -D tmpDict -of tmp -$ZSTD -d tmp -D tmpDict -of result +cp zstdcli.c tmp +$ZSTD -f tmp -D tmpDict +$ZSTD -d tmp.zst -D tmpDict -of result diff zstdcli.c result +$ECHO "Create second (different) dictionary" $ZSTD --train *.c *.h -o tmpDictC -$ZSTD -d tmp -D tmpDictC -of result && die "wrong dictionary not detected!" +$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!" +$ECHO "Create dictionary with short dictID" $ZSTD --train *.c --dictID 1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +$ECHO "Compress without dictID" +$ZSTD -f tmp -D tmpDict1 --no-dictID +$ZSTD -d tmp.zst -D tmpDict -of result +diff zstdcli.c result +rm tmp* $ECHO "\n**** multiple files tests **** " diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 74f3878be..e9fa0526c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -125,7 +125,6 @@ static int usage_advanced(const char* programName) DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -t : test compressed file integrity \n"); DISPLAY( " -v : verbose mode\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); @@ -134,8 +133,12 @@ static int usage_advanced(const char* programName) #endif #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); + DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n"); #endif +#ifndef ZSTD_NODECOMPRESS + DISPLAY( " -t : test compressed file integrity \n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); +#endif #ifndef ZSTD_NODICT DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); @@ -236,14 +239,15 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; } if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; } + if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } + if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } + if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } + if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; } if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ - if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } - if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } - if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){ @@ -300,7 +304,7 @@ int main(int argCount, const char** argv) case 'k': argument++; break; /* test compressed file */ - case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break; + case 't': decode=1; outFileName=nulmark; argument++; break; /* dictionary name */ case 'o': nextArgumentIsOutFileName=1; argument++; break; From 6fca9e7545c21291f48b694dbee56fac5aa7ff3d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 31 May 2016 02:40:42 +0200 Subject: [PATCH 077/208] Changed : max dictionary file size increased to 8 MB --- programs/fileio.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index fc3022080..767d4c05f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -97,7 +97,7 @@ #define CACHELINE 64 -#define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */ +#define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */ #define FNSPACE 30 @@ -214,12 +214,12 @@ static FILE* FIO_openDstFile(const char* dstFileName) /*! FIO_loadFile() : * creates a buffer, pointed by `*bufferPtr`, * loads `filename` content into it, -* up to MAX_DICT_SIZE bytes +* up to MAX_DICT_SIZE bytes. +* @return : loaded size */ static size_t FIO_loadFile(void** bufferPtr, const char* fileName) { FILE* fileHandle; - size_t readSize; U64 fileSize; *bufferPtr = NULL; @@ -239,8 +239,8 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName) } *bufferPtr = (BYTE*)malloc((size_t)fileSize); if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer"); - readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); - if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); + { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); + if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); } fclose(fileHandle); return (size_t)fileSize; } @@ -315,9 +315,9 @@ static int FIO_compressFilename_internal(cRess_t ress, params.fParams.contentSizeFlag = 1; params.fParams.noDictIDFlag = !g_dictIDFlag; if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; - { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); - if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); } - } + { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); + if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); + } } /* Main compression loop */ readsize = 0; From 370b751e2424024f719f95e75b7f3d18f5f0499c Mon Sep 17 00:00:00 2001 From: Giuseppe Ottaviano Date: Mon, 30 May 2016 18:49:58 -0700 Subject: [PATCH 078/208] Expose function to add entropy tables to pre-built dictionary. In some cases a custom dictionary building algorithm tailored for a specific input can be more effective than the one produced by `ZDICT_trainFromBuffer`, but with the current API it's not possible encode the entropy tables into the custom-built dictionary. This commit extracts the logic to add entropy tables to a dictionary from `ZDICT_trainFromBuffer` and exposes it as a function `ZDICT_addEntropyTablesFromBuffer`. --- lib/dictBuilder/zdict.c | 72 +++++++++++++++++++++++++---------------- lib/dictBuilder/zdict.h | 15 +++++++++ 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index be141ce10..09249e9ea 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -807,6 +807,33 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize, return nbSegments * DIB_FASTSEGMENTSIZE; } +size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) +{ + size_t hSize; + unsigned const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel; + + /* dictionary header */ + MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); + { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); + U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11); + MEM_writeLE32((char*)dictBuffer+4, dictID); + } + hSize = 8; + + /* entropy tables */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + DISPLAYLEVEL(2, "statistics ... \n"); + hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, + compressionLevel, + samplesBuffer, samplesSizes, nbSamples, + (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); + + if (hSize + dictContentSize < dictBufferCapacity) + memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); + return MIN(dictBufferCapacity, hSize+dictContentSize); +} #define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3) /*! ZDICT_trainFromBuffer_unsafe() : @@ -815,13 +842,12 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize, */ size_t ZDICT_trainFromBuffer_unsafe( void* dictBuffer, size_t maxDictSize, - const void* samplesBuffer, const size_t* sampleSizes, unsigned nbSamples, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t params) { U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16)); dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); unsigned selectivity = params.selectivityLevel; - unsigned compressionLevel = params.compressionLevel; size_t const targetDictSize = maxDictSize; size_t sBuffSize; size_t dictSize = 0; @@ -831,18 +857,17 @@ size_t ZDICT_trainFromBuffer_unsafe( if (!dictList) return ERROR(memory_allocation); /* init */ - { unsigned u; for (u=0, sBuffSize=0; u1) { /* selectivity == 1 => fast mode */ ZDICT_trainBuffer(dictList, dictListSize, samplesBuffer, sBuffSize, - sampleSizes, nbSamples, + samplesSizes, nbSamples, selectivity, (U32)targetDictSize); /* display best matches */ @@ -864,7 +889,6 @@ size_t ZDICT_trainFromBuffer_unsafe( /* create dictionary */ { U32 dictContentSize = ZDICT_dictSize(dictList); - size_t hSize; /* build dict content */ { U32 u; @@ -884,25 +908,9 @@ size_t ZDICT_trainFromBuffer_unsafe( samplesBuffer, sBuffSize); } - /* dictionary header */ - MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); - { U64 const randomID = XXH64((char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize, 0); - U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11); - MEM_writeLE32((char*)dictBuffer+4, dictID); - } - hSize = 8; - - /* entropic tables */ - DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ - DISPLAYLEVEL(2, "statistics ... \n"); - hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, maxDictSize-hSize, - compressionLevel, - samplesBuffer, sampleSizes, nbSamples, - (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize); - - if (hSize + dictContentSize < maxDictSize) - memmove((char*)dictBuffer + hSize, (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize); - dictSize = MIN(maxDictSize, hSize+dictContentSize); + dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize, + samplesBuffer, samplesSizes, nbSamples, + params); } /* clean up */ @@ -914,8 +922,8 @@ size_t ZDICT_trainFromBuffer_unsafe( /* issue : samplesBuffer need to be followed by a noisy guard band. * work around : duplicate the buffer, and add the noise */ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, - ZDICT_params_t params) + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) { void* newBuff; size_t sBuffSize; @@ -946,3 +954,13 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, samplesBuffer, samplesSizes, nbSamples, params); } + +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_params_t params; + memset(¶ms, 0, sizeof(params)); + return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + params); +} diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 2ca190ce3..3a724d0be 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -52,6 +52,21 @@ extern "C" { size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); +/*! ZDICT_addEntropyTablesFromBuffer() : + + Given a content-only dictionary (built for example from common strings in + the input), add entropy tables computed from the memory buffer + `samplesBuffer`, where `nbSamples` samples have been stored concatenated. + Each sample size is provided into an orderly table `samplesSizes`. + + The input dictionary is the last `dictContentSize` bytes of `dictBuffer`. The + resulting dictionary with added entropy tables will written back to + `dictBuffer`. + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`). +*/ +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + /*-************************************* * Helper functions From e3f4e6cbda7962fd8717f8ce3d8c6557052d6670 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 31 May 2016 10:12:04 +0200 Subject: [PATCH 079/208] Updated news --- NEWS | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index db8f9bad6..beb6ae3db 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,10 @@ -v0.6.2 +v0.7.0 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski New : Support for Sparse File-systems (do not use space for zero-filled sectors) -New : Support pass-through mode, when using `-df` -New : Support for custom malloc/free functions +New : Support pass-through mode (when using `-df`) +New : API : dictionary files from custom content, by Giuseppe Ottaviano +New : API support for custom malloc/free functions +New : controllable Dictionary ID v0.6.1 New : zlib wrapper API, thanks to Przemyslaw Skibinski From d57b418214f47cd8ef1e9e44476555e3a676f114 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 11:46:16 +0200 Subject: [PATCH 080/208] updated appveyor.yml --- appveyor.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 724d0136e..3d46ddc19 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -92,11 +92,10 @@ test_script: - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( CD programs && SET ZSTD=./zstd.exe && - SET DATAGEN=./datagen.exe && sh -e playTests.sh --test-large-data && + fullbench.exe -i1 && + fullbench.exe -i1 -P0 && CD .. && - projects\fullbench.exe -i1 && - projects\fullbench.exe -i1 -P0 && projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && From f772bf54a5649d6bcc741b2d78f11b7f37d9de65 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 12:43:46 +0200 Subject: [PATCH 081/208] support for skippable frames --- lib/common/zstd_internal.h | 2 +- lib/common/zstd_static.h | 4 +++- lib/decompress/zbuff_decompress.c | 24 +++++++++++++++++---- lib/decompress/zstd_decompress.c | 35 +++++++++++++++++++++++++++++-- programs/fileio.c | 2 +- programs/zbufftest.c | 11 +++++++--- 6 files changed, 66 insertions(+), 12 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 2eea5feee..b2fcd322d 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -252,6 +252,6 @@ typedef struct { const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); - +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e0f50cf40..126d6cf5c 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -51,7 +51,8 @@ extern "C" { /*-************************************* * Constants ***************************************/ -#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U /*-************************************* @@ -196,6 +197,7 @@ typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams; #define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ static const size_t ZSTD_frameHeaderSize_min = 5; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index d4ca550a7..e785e9ec3 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -218,12 +218,19 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; - if (!decodedSize) break; /* this was just a header */ + if (!decodedSize) { + if (isSkipFrame) { + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = 0; + } + break; /* this was just a header */ + } zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; break; @@ -243,12 +250,21 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + if (!decodedSize) { + if (isSkipFrame) { + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = 0; + break; + } + zbd->stage = ZBUFFds_read; /* this was just a header */ + break; + } zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; // break; /* ZBUFFds_flush follows */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b670c548c..64be8fb56 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -101,7 +101,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } * Context management ***************************************************************/ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage; + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; struct ZSTD_DCtx_s { @@ -312,7 +313,15 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t const BYTE* ip = (const BYTE*)src; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; - if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ + fparamsPtr->frameContentSize = 0; + fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; + return 0; + } + return ERROR(prefix_unknown); + } /* ensure there is enough `srcSize` to fully read/decode frame header */ { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); @@ -995,6 +1004,11 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) +{ + return dctx->stage == ZSTDds_skipFrame; +} + size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ @@ -1006,6 +1020,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case ZSTDds_getFrameHeaderSize : if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); @@ -1063,6 +1083,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c dctx->previousDstEnd = (char*)dst + rSize; return rSize; } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = ZSTD_frameHeaderSize_min; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } default: return ERROR(GENERIC); /* impossible */ } diff --git a/programs/fileio.c b/programs/fileio.c index eba90d011..b333ded24 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) continue; } #endif - if (magic != ZSTD_MAGICNUMBER) { + if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) { if (g_overwrite) /* -df : pass-through mode */ return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); else { diff --git a/programs/zbufftest.c b/programs/zbufftest.c index fce0ab275..8d32bbcc8 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -146,7 +146,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo int testResult = 0; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* CNBuffer = malloc(CNBufferSize); - size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); @@ -162,15 +163,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(compressedBuffer+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); readSize = CNBufferSize; genSize = compressedBufferSize; - result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); + result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ - cSize = genSize; + cSize += genSize; genSize = compressedBufferSize - cSize; result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); if (result != 0) goto _output_error; /* error, or some data not flushed */ From 5bee32e34956e4eaa726d81fd185f62d385227b9 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 13:36:14 +0200 Subject: [PATCH 082/208] cmake project works with MSYS --- .gitignore | 2 +- programs/Makefile | 2 +- programs/dibio.h | 2 +- programs/zbufftest.c | 2 +- projects/cmake/lib/CMakeLists.txt | 23 ++++++++--------------- projects/cmake/programs/CMakeLists.txt | 8 ++------ 6 files changed, 14 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 2d3b6917a..f0e2c9f12 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,4 @@ _zstdbench/ zlib_wrapper/ # CMake -contrib/cmake/ +projects/cmake/ diff --git a/programs/Makefile b/programs/Makefile index f236459e8..c57d8ec16 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,7 +33,7 @@ DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I../lib/common +CPPFLAGS= -I../lib/common -I../lib/dictBuilder CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) diff --git a/programs/dibio.h b/programs/dibio.h index fd1f21df0..0ccec4135 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,7 @@ /*-************************************* * Dependencies ***************************************/ -#include "../lib/dictBuilder/zdict_static.h" /* ZDICT_params_t */ +#include "zdict_static.h" /* ZDICT_params_t */ /*-************************************* diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 8d32bbcc8..6e5b74394 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -165,7 +165,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* generate skippable frame */ MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); - MEM_writeLE32(compressedBuffer+4, (U32)skippableFrameSize); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); cSize = skippableFrameSize + 8; /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index dae706cbe..c5557ff74 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -2,19 +2,19 @@ # zstd - Makefile # Copyright (C) Yann Collet 2014-2016 # All rights reserved. -# +# # BSD license # # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. -# +# # * Redistributions in binary form must reproduce the above copyright notice, this # list of conditions and the following disclaimer in the documentation and/or # other materials provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -25,7 +25,7 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# +# # You can contact the author at : # - zstd homepage : http://www.zstd.net/ # ################################################################ @@ -105,11 +105,6 @@ IF (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v04.h) ENDIF (ZSTD_LEGACY_SUPPORT) -IF (MSVC) - SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/visual/2013/zstdlib) - SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/resource.h ${MSVC_RESOURCE_DIR}/zstdlib.rc) -ENDIF (MSVC) - # Split project to static and shared libraries build ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources}) ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) @@ -131,13 +126,11 @@ IF (NOT WORKAROUND_OUTDATED_CODE_STYLE) ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE) # Define library base name -IF (UNIX) - SET(LIBRARY_BASE_NAME libzstd) -ELSEIF (MSVC) +IF (MSVC) SET(LIBRARY_BASE_NAME zstdlib) ELSE () - MESSAGE(FATAL_ERROR "Unsupported build type") -ENDIF (UNIX) + SET(LIBRARY_BASE_NAME libzstd) +ENDIF (MSVC) # Define static and shared library names SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}) diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index 3b4f51d79..2670fc6a8 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -38,14 +38,10 @@ SET(CMAKE_INCLUDE_CURRENT_DIR TRUE) SET(ROOT_DIR ../../..) # Define programs directory, where sources and header files are located +SET(LIBRARY_DIR ${ROOT_DIR}/lib) SET(PROGRAMS_DIR ${ROOT_DIR}/programs) -INCLUDE_DIRECTORIES(${PROGRAMS_DIR}) +INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder) -IF (WORKAROUND_OUTDATED_CODE_STYLE) - # Define library directory, where sources and header files are located - SET(LIBRARY_DIR ${ROOT_DIR}/lib) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common) -ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) IF (ZSTD_LEGACY_SUPPORT) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) From 5d500b60e7eb63106dcf8e1848c1d396f320a9d3 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 13:39:36 +0200 Subject: [PATCH 083/208] updated CMakeLists.txt --- projects/cmake/lib/CMakeLists.txt | 13 +++---------- projects/cmake/programs/CMakeLists.txt | 9 ++------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index 834818bfe..540f20747 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -106,11 +106,6 @@ IF (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v04.h) ENDIF (ZSTD_LEGACY_SUPPORT) -IF (MSVC) - SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/visual/2013/zstdlib) - SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/resource.h ${MSVC_RESOURCE_DIR}/zstdlib.rc) -ENDIF (MSVC) - # Split project to static and shared libraries build ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources}) ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) @@ -132,13 +127,11 @@ IF (NOT WORKAROUND_OUTDATED_CODE_STYLE) ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE) # Define library base name -IF (UNIX) - SET(LIBRARY_BASE_NAME libzstd) -ELSEIF (MSVC) +IF (MSVC) SET(LIBRARY_BASE_NAME zstdlib) ELSE () - MESSAGE(FATAL_ERROR "Unsupported build type") -ENDIF (UNIX) + SET(LIBRARY_BASE_NAME libzstd) +ENDIF (MSVC) # Define static and shared library names SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}) diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index bc8a51674..ef5f0eafd 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -38,14 +38,9 @@ SET(CMAKE_INCLUDE_CURRENT_DIR TRUE) SET(ROOT_DIR ../../..) # Define programs directory, where sources and header files are located +SET(LIBRARY_DIR ${ROOT_DIR}/lib) SET(PROGRAMS_DIR ${ROOT_DIR}/programs) -INCLUDE_DIRECTORIES(${PROGRAMS_DIR}) - -IF (WORKAROUND_OUTDATED_CODE_STYLE) - # Define library directory, where sources and header files are located - SET(LIBRARY_DIR ${ROOT_DIR}/lib) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder) -ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) +INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder) IF (ZSTD_LEGACY_SUPPORT) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) From 8243b27f442e7d454338cc4a3eaa160b05849913 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 17:19:05 +0200 Subject: [PATCH 084/208] cmake project: removed WORKAROUND_OUTDATED_CODE_STYLE --- projects/cmake/CMakeLists.txt | 7 ------- projects/cmake/lib/CMakeLists.txt | 21 +++++++++------------ projects/cmake/programs/CMakeLists.txt | 7 +------ 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/projects/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt index 418833735..ae8f7be6e 100644 --- a/projects/cmake/CMakeLists.txt +++ b/projects/cmake/CMakeLists.txt @@ -44,13 +44,6 @@ ELSE (ZSTD_LEGACY_SUPPORT) ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0) ENDIF (ZSTD_LEGACY_SUPPORT) -# TARGET_INCLUDE_DIRECTORIES can use in version 2.8.11 and greate -IF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11)) - SET(WORKAROUND_OUTDATED_CODE_STYLE TRUE) -ELSE () - SET(WORKAROUND_OUTDATED_CODE_STYLE FALSE) -ENDIF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11)) - ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(programs) diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index c5557ff74..405888746 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -95,16 +95,23 @@ IF (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v01.c ${LIBRARY_LEGACY_DIR}/zstd_v02.c ${LIBRARY_LEGACY_DIR}/zstd_v03.c - ${LIBRARY_LEGACY_DIR}/zstd_v04.c) + ${LIBRARY_LEGACY_DIR}/zstd_v04.c + ${LIBRARY_LEGACY_DIR}/zstd_v05.c) SET(Headers ${Headers} ${LIBRARY_LEGACY_DIR}/zstd_legacy.h ${LIBRARY_LEGACY_DIR}/zstd_v01.h ${LIBRARY_LEGACY_DIR}/zstd_v02.h ${LIBRARY_LEGACY_DIR}/zstd_v03.h - ${LIBRARY_LEGACY_DIR}/zstd_v04.h) + ${LIBRARY_LEGACY_DIR}/zstd_v04.h + ${LIBRARY_LEGACY_DIR}/zstd_v05.h) ENDIF (ZSTD_LEGACY_SUPPORT) +IF (MSVC) + SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib) + SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc) +ENDIF (MSVC) + # Split project to static and shared libraries build ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources}) ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) @@ -115,16 +122,6 @@ IF (MSVC) SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS") ENDIF (MSVC) -# Define include directories -IF (NOT WORKAROUND_OUTDATED_CODE_STYLE) - TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common) - TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common) - IF (ZSTD_LEGACY_SUPPORT) - TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR}) - TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR}) - ENDIF (ZSTD_LEGACY_SUPPORT) -ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE) - # Define library base name IF (MSVC) SET(LIBRARY_BASE_NAME zstdlib) diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index 2670fc6a8..1cf1b69e3 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -45,12 +45,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBui IF (ZSTD_LEGACY_SUPPORT) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) - INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR}) - - IF (WORKAROUND_OUTDATED_CODE_STYLE) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}/legacy) - ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) - + INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy) SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c) ENDIF (ZSTD_LEGACY_SUPPORT) From 019b83330646a50eec928166633cdece852e6d65 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 17:34:37 +0200 Subject: [PATCH 085/208] VS project: dictBuilder added to include directories --- projects/VS2008/zstd/zstd.vcproj | 8 ++++---- projects/VS2008/zstdlib/zstdlib.vcproj | 8 ++++---- projects/VS2010/zstd/zstd.vcxproj | 10 +++++----- projects/VS2010/zstdlib/zstdlib.vcxproj | 10 +++++----- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj index 9e4626500..f84fbc93f 100644 --- a/projects/VS2008/zstd/zstd.vcproj +++ b/projects/VS2008/zstd/zstd.vcproj @@ -44,7 +44,7 @@ + @@ -119,27 +119,27 @@ true - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ true - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj index e8625a501..9e4978111 100644 --- a/projects/VS2010/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -1,4 +1,4 @@ - + @@ -97,28 +97,28 @@ true zstdlib_x86 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false true zstdlib_x64 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false zstdlib_x86 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false zstdlib_x64 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false From 43aa9fe8b3f6b48e1ab5ad4da7d8b8a28b47d181 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 19:36:51 +0200 Subject: [PATCH 086/208] fixed skippable frame --- lib/decompress/zbuff_decompress.c | 20 +++------------- lib/decompress/zstd_decompress.c | 2 +- programs/zbufftest.c | 38 +++++++++++++++++++------------ 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index e785e9ec3..72456fd5f 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -224,13 +224,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; - if (!decodedSize) { - if (isSkipFrame) { - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = 0; - } - break; /* this was just a header */ - } + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; break; @@ -256,15 +250,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, zbd->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { - if (isSkipFrame) { - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = 0; - break; - } - zbd->stage = ZBUFFds_read; /* this was just a header */ - break; - } + if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; // break; /* ZBUFFds_flush follows */ @@ -292,7 +278,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, *srcSizePtr = ip-istart; *dstCapacityPtr = op-ostart; { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ +// if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ nextSrcSizeHint -= zbd->inPos; /* already loaded*/ return nextSrcSizeHint; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 64be8fb56..75e91e81c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1090,7 +1090,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c return 0; } case ZSTDds_skipFrame: - { dctx->expected = ZSTD_frameHeaderSize_min; + { dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; return 0; } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 6e5b74394..5b172d23e 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -151,7 +151,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); - size_t result, cSize, readSize, genSize; + size_t result, cSize, readSize, readSkipSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); @@ -185,12 +185,17 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - readSize = cSize; + readSkipSize = cSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); + if (genSize != 0) goto _output_error; /* skippable frame */ + ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + readSize = cSize - readSkipSize; + genSize = CNBufferSize; + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer+readSkipSize, &readSize); if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if (readSize != cSize) goto _output_error; /* should have read the entire frame */ + if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ @@ -204,17 +209,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); - ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - { size_t r = 1, pIn=0, pOut=0; - while (r) { - size_t inS = 1; - size_t outS = 1; - r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); - pIn += inS; - pOut += outS; - } - readSize = pIn; - genSize = pOut; + { size_t r, pIn=0, pOut=0; + do + { ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + r = 1; + while (r) { + size_t inS = 1; + size_t outS = 1; + r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); + pIn += inS; + pOut += outS; + } + readSize = pIn; + genSize = pOut; + } while (genSize==0); } if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize != cSize) goto _output_error; /* should have read the entire frame */ From cf6edcd0a13fd7f2dc85384e3883886535960d23 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 20:21:43 +0200 Subject: [PATCH 087/208] zbufftest.c: fixed g++ warning --- programs/zbufftest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 5b172d23e..552d09287 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -192,7 +192,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSize = cSize - readSkipSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer+readSkipSize, &readSize); + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize); if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */ From f2a3b6e7b4d4a98975c1b0b7687034ba6b9d01e8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 31 May 2016 18:13:56 +0200 Subject: [PATCH 088/208] added : frame content checksum --- .gitignore | 4 +- lib/common/zstd_static.h | 6 +- lib/compress/zstd_compress.c | 124 +++++++++++++++++-------------- lib/decompress/zstd_decompress.c | 9 +++ programs/.clang_complete | 3 - programs/.gitignore | 1 + programs/fuzzer.c | 10 ++- 7 files changed, 91 insertions(+), 66 deletions(-) delete mode 100644 programs/.clang_complete diff --git a/.gitignore b/.gitignore index 2d3b6917a..c90212490 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,4 @@ projects/VS2015 _codelite/ _zstdbench/ zlib_wrapper/ - -# CMake -contrib/cmake/ +.clang_complete diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index d9f619f26..6b79b3669 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -86,8 +86,9 @@ typedef struct { } ZSTD_compressionParameters; typedef struct { - U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ - U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ + U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ + U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -196,6 +197,7 @@ typedef struct { U64 frameContentSize; U32 windowLog; U32 dictID; + U32 checksumFlag; } ZSTD_frameParams; #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c7ba03053..a9a2dc79d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,9 +51,11 @@ /*-************************************* * Dependencies ***************************************/ -#include /* malloc */ -#include /* memset */ +#include /* malloc */ +#include /* memset */ #include "mem.h" +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH_reset, update, digest */ #include "fse_static.h" #include "huf_static.h" #include "zstd_internal.h" @@ -104,6 +106,7 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; + XXH64_state_t xxhState; ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; @@ -266,6 +269,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, } } if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ + XXH64_reset(&zc->xxhState, 0); zc->hashTable3 = (U32*)(zc->workSpace); zc->hashTable = zc->hashTable3 + h3Size; zc->chainTable = zc->hashTable + hSize; @@ -1938,7 +1942,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* catch up */ if (offset) { - U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ @@ -2043,19 +2047,22 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa -static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { - size_t blockSize = zc->blockSize; + size_t blockSize = cctx->blockSize; size_t remaining = srcSize; const BYTE* ip = (const BYTE*)src; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = &zc->seqStore.stats; + const U32 maxDist = 1 << cctx->params.cParams.windowLog; + ZSTD_stats_t* stats = &cctx->seqStore.stats; ZSTD_statsInit(stats); + if (cctx->params.fParams.checksumFlag) + XXH64_update(&cctx->xxhState, src, srcSize); + while (remaining) { size_t cSize; ZSTD_statsResetFreqs(stats); @@ -2063,14 +2070,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; - if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { + if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { /* enforce maxDist */ - U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist; - if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit; - if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit; + U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; + if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; + if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; } - cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) { /* block is not compressible */ @@ -2090,7 +2097,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, op += cSize; } - ZSTD_statsPrint(stats, zc->params.cParams.searchLength); + ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); return op-ostart; } @@ -2104,7 +2111,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */ | (fcsId << 6) ); U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - BYTE const fCheckByte = (BYTE)(dictIDSizeCode&3); + BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4)); size_t pos; if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); @@ -2261,40 +2268,45 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { /* note : magic number already checked */ - size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode; - short offcodeNCount[MaxOff+1]; - unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; - short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; - short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + size_t const dictSizeStart = dictSize; - size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize); - if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); - zc->flagStaticTables = 1; - dict = (const char*)dict + hufHeaderSize; - dictSize -= hufHeaderSize; + { size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize); + if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); + zc->flagStaticTables = 1; + dict = (const char*)dict + hufHeaderSize; + dictSize -= hufHeaderSize; + } - offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); - if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + offcodeHeaderSize; - dictSize -= offcodeHeaderSize; + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dict = (const char*)dict + offcodeHeaderSize; + dictSize -= offcodeHeaderSize; + } - matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); - if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + matchlengthHeaderSize; - dictSize -= matchlengthHeaderSize; + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dict = (const char*)dict + matchlengthHeaderSize; + dictSize -= matchlengthHeaderSize; + } - litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); - if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictSize -= litlengthHeaderSize; + } - return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; + return (dictSizeStart-dictSize); } /** ZSTD_compress_insertDictionary() : @@ -2366,30 +2378,34 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) /*! ZSTD_compressEnd() : * Write frame epilogue. * @return : nb of bytes written into dst (or an error code) */ -size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity) +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) { BYTE* op = (BYTE*)dst; size_t fhSize = 0; /* not even init ! */ - if (zc->stage==0) return ERROR(stage_wrong); + if (cctx->stage==0) return ERROR(stage_wrong); /* special case : empty frame */ - if (zc->stage==1) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0, 0); + if (cctx->stage==1) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; op += fhSize; - zc->stage = 2; + cctx->stage = 2; } /* frame epilogue */ if (dstCapacity < 3) return ERROR(dstSize_tooSmall); - op[0] = (BYTE)(bt_end << 6); - op[1] = 0; - op[2] = 0; + { U32 const checksum = cctx->params.fParams.checksumFlag ? + (U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) : + 0; + op[0] = (BYTE)((bt_end<<6) + (checksum>>16)); + op[1] = (BYTE)(checksum>>8); + op[2] = (BYTE)checksum; + } - zc->stage = 0; /* return to "created by not init" status */ + cctx->stage = 0; /* return to "created but not init" status */ return 3+fhSize; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b0831617c..6ade38711 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -57,6 +57,8 @@ #include /* memcpy, memmove */ #include /* debug only : printf */ #include "mem.h" /* low level memory routines */ +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64_* */ #include "zstd_internal.h" #include "fse_static.h" #include "huf_static.h" @@ -116,6 +118,7 @@ struct ZSTD_DCtx_s size_t expected; size_t headerSize; ZSTD_frameParams fParams; + XXH64_state_t xxhState; ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ @@ -339,6 +342,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t U32 const dictIDSizeCode = checkByte&3; fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + fparamsPtr->checksumFlag = checkByte & 0x10; switch(dictIDSizeCode) /* fcsId */ { default: /* impossible */ @@ -367,6 +372,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t sr size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); return result; } @@ -1021,6 +1027,9 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } +/** ZSTD_decompressContinue() : +* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) +* or an error code, which can be tested using ZSTD_isError() */ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ diff --git a/programs/.clang_complete b/programs/.clang_complete deleted file mode 100644 index 658aa00b0..000000000 --- a/programs/.clang_complete +++ /dev/null @@ -1,3 +0,0 @@ --I../lib/common --I../lib/legacy --I./legacy diff --git a/programs/.gitignore b/programs/.gitignore index f7061d3b3..0f391d2da 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -40,6 +40,7 @@ grillResults.txt _* tmp* *.zst +result # fuzzer afl diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 7fbf906ce..0c3389282 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -216,7 +216,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = CNBuffSize / 3; { ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize); - ZSTD_frameParameters const fPar = { 1 , 0 }; + ZSTD_frameParameters const fPar = { 1 , 0 , 0 }; ZSTD_parameters p; p.cParams = cPar; p.fParams = fPar; CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); @@ -276,7 +276,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++); - { ZSTD_frameParameters const fParams = { 0, 1 /*NoDictID*/ }; + { ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ }; ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize); ZSTD_parameters p; p.cParams = cParams; p.fParams = fParams; @@ -639,12 +639,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */ dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); - if (FUZ_rand(&lseed) & 15) { + if (FUZ_rand(&lseed) & 0xF) { size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } else { ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); - ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */ + ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, + !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, + 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ ZSTD_parameters p; size_t errorCode; p.cParams = cPar; p.fParams = fpar; From 00fa1ae0c02df0a9527f8b6e3b9dfc90c39b0f70 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 00:13:02 +0200 Subject: [PATCH 089/208] update playTests.sh permission --- programs/playTests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 programs/playTests.sh diff --git a/programs/playTests.sh b/programs/playTests.sh old mode 100644 new mode 100755 From 8e3a36a6dba3a35220a70fd572ace95947651b52 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 00:18:28 +0200 Subject: [PATCH 090/208] decompression validates frame content checksum --- lib/common/error_private.h | 1 + lib/common/error_public.h | 1 + lib/decompress/zstd_decompress.c | 45 ++++++++++++++++++++------------ 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 6b243c077..889061496 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -102,6 +102,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size incorrect"; case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 660b2d3fe..e8cfcc918 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -54,6 +54,7 @@ typedef enum { ZSTD_error_dstSize_tooSmall, ZSTD_error_srcSize_wrong, ZSTD_error_corruption_detected, + ZSTD_error_checksum_wrong, ZSTD_error_tableLog_tooLarge, ZSTD_error_maxSymbolValue_tooLarge, ZSTD_error_maxSymbolValue_tooSmall, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6ade38711..ace0ad30e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -148,12 +148,6 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) return 0; } -ZSTD_DCtx* ZSTD_createDCtx(void) -{ - ZSTD_customMem customMem = { NULL, NULL }; - return ZSTD_createDCtx_advanced(customMem); -} - ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; @@ -180,6 +174,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) return dctx; } +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + ZSTD_customMem const customMem = { NULL, NULL }; + return ZSTD_createDCtx_advanced(customMem); +} + size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { @@ -340,9 +340,9 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t BYTE const checkByte = ip[5]; size_t pos = ZSTD_frameHeaderSize_min; U32 const dictIDSizeCode = checkByte&3; - fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; fparamsPtr->checksumFlag = checkByte & 0x10; switch(dictIDSizeCode) /* fcsId */ { @@ -961,6 +961,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, if (cBlockSize == 0) break; /* bt_end */ if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); op += decodedSize; ip += cBlockSize; remainingSize -= cBlockSize; @@ -1065,6 +1066,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(cBlockSize)) return cBlockSize; if (bp.blockType == bt_end) { + if (dctx->fParams.checksumFlag) { + U64 const h64 = XXH64_digest(&dctx->xxhState); + U32 const h32 = (U32)(h64>>11) & ((1<<22)-1); + const BYTE* const ip = (const BYTE*)src; + U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16); + if (check32 != h32) return ERROR(checksum_wrong); + } dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; } else { @@ -1096,6 +1104,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c dctx->stage = ZSTDds_decodeBlockHeader; dctx->expected = ZSTD_blockHeaderSize; dctx->previousDstEnd = (char*)dst + rSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); return rSize; } default: @@ -1112,18 +1121,19 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi dctx->previousDstEnd = (const char*)dict + dictSize; } -static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const dictSizeStart) { - size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize; + size_t dictSize = dictSizeStart; - hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); - if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + hSize; - dictSize -= hSize; + { size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); + if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + hSize; + dictSize -= hSize; + } { short offcodeNCount[MaxOff+1]; U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; - offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } @@ -1133,7 +1143,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; - matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } @@ -1143,14 +1153,15 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; - litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictSize -= litlengthHeaderSize; } dctx->flagRepeatTable = 1; - return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; + return dictSizeStart - dictSize; } static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) From 202844ebd01b157350ead2bfadcf302a3b1b27f7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 00:44:36 +0200 Subject: [PATCH 091/208] fixed zbufftest : --- lib/compress/zbuff_compress.c | 3 ++- lib/decompress/zstd_decompress.c | 1 + programs/zbufftest.c | 25 +++++++++++++------------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index e078d7eb2..66deb495f 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -125,7 +125,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; memset(zbc, 0, sizeof(ZBUFF_CCtx)); - zbc->customAlloc = customMem.customAlloc; + zbc->customAlloc = customMem.customAlloc; zbc->customFree = customMem.customFree; zbc->zc = ZSTD_createCCtx_advanced(customMem); return zbc; @@ -180,6 +180,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); params.fParams.contentSizeFlag = 0; ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index ace0ad30e..4ef8de30f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1104,6 +1104,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c dctx->stage = ZSTDds_decodeBlockHeader; dctx->expected = ZSTD_blockHeaderSize; dctx->previousDstEnd = (char*)dst + rSize; + if (ZSTD_isError(rSize)) return rSize; if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); return rSize; } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index d0c0503a8..c7951eeff 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -97,7 +97,7 @@ static U32 FUZ_GetMilliStart(void) static U32 FUZ_GetMilliSpan(U32 nTimeStart) { - U32 nCurrent = FUZ_GetMilliStart(); + U32 const nCurrent = FUZ_GetMilliStart(); U32 nSpan = nCurrent - nTimeStart; if (nTimeStart > nCurrent) nSpan += 0x100000 * 1000; @@ -148,10 +148,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* CNBuffer = malloc(CNBufferSize); size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); - void* compressedBuffer = malloc(compressedBufferSize); + void* const compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; - void* decodedBuffer = malloc(decodedBufferSize); - size_t result, cSize, readSize, genSize; + void* const decodedBuffer = malloc(decodedBufferSize); + size_t cSize, readSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); @@ -168,13 +168,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); readSize = CNBufferSize; genSize = compressedBufferSize; - result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); - if (ZBUFF_isError(result)) goto _output_error; + { size_t const r = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); + if (ZBUFF_isError(r)) goto _output_error; } if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ cSize = genSize; genSize = compressedBufferSize - cSize; - result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); - if (result != 0) goto _output_error; /* error, or some data not flushed */ + { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); + if (r != 0) goto _output_error; } /*< error, or some data not flushed */ cSize += genSize; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); @@ -183,8 +183,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSize = cSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); - if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ + { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize != cSize) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); @@ -600,9 +600,10 @@ int main(int argc, const char** argv) if (testNb==0) { result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */ - if (!result) + if (!result) { + DISPLAYLEVEL(4, "Unit tests using customMem :\n") result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */ - } + } } if (!result) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); From c30d8d50da018fadd1120c183505304470a8997a Mon Sep 17 00:00:00 2001 From: cyan Date: Wed, 1 Jun 2016 09:31:34 +0200 Subject: [PATCH 092/208] fixed fullbench visual project --- projects/.gitignore | 1 + projects/VS2008/fullbench/fullbench.vcproj | 156 ++++++++++---------- projects/VS2010/fullbench/fullbench.vcxproj | 2 + 3 files changed, 85 insertions(+), 74 deletions(-) diff --git a/projects/.gitignore b/projects/.gitignore index 378eac25d..dc2035158 100644 --- a/projects/.gitignore +++ b/projects/.gitignore @@ -1 +1,2 @@ build +*Copy diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/projects/VS2008/fullbench/fullbench.vcproj index 6ab878cc9..f55602022 100644 --- a/projects/VS2008/fullbench/fullbench.vcproj +++ b/projects/VS2008/fullbench/fullbench.vcproj @@ -92,80 +92,6 @@ Name="VCPostBuildEventTool" /> - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + @@ -414,6 +418,10 @@ RelativePath="..\..\..\lib\common\mem.h" > + + diff --git a/projects/VS2010/fullbench/fullbench.vcxproj b/projects/VS2010/fullbench/fullbench.vcxproj index 2d9b78ce1..150faa8d6 100644 --- a/projects/VS2010/fullbench/fullbench.vcxproj +++ b/projects/VS2010/fullbench/fullbench.vcxproj @@ -157,6 +157,7 @@ + @@ -174,6 +175,7 @@ + From 043b3c01bb0d0f20cf8761d1a6bf57a52a5149f3 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 10:15:25 +0200 Subject: [PATCH 093/208] zlibWrapped: removed dependency from ZSTD_FRAMEHEADERSIZE_MIN --- zlibWrapper/Makefile | 11 ++++++++++- zlibWrapper/zstd_zlibwrapper.c | 22 +++++++--------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index c74107d40..3f293cd90 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -22,7 +22,7 @@ LDFLAGS = $(LOC) RM = rm -f -all: clean test testdll +all: clean test testdll testzstd test: example ./example @@ -30,6 +30,9 @@ test: example testdll: example_d ./example_d +testzstd: example_zstd + ./example_zstd + .c.o: $(CC) $(CFLAGS) -c -o $@ $< @@ -39,12 +42,18 @@ example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB) +example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) + $(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c $(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c +$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h + $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c + clean: -$(RM) $(ZLIBWRAPPER_PATH)/*.o -$(RM) $(EXAMPLE_PATH)/*.o diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 5df1215e5..0d18cbb27 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -35,17 +35,17 @@ #include "zstd_zlibwrapper.h" #include "zstd.h" #include "zstd_static.h" /* ZSTD_MAGICNUMBER */ -#include "zstd_internal.h" /* MIN */ #include "zbuff.h" #define Z_INFLATE_SYNC 8 #define ZWRAP_HEADERSIZE 4 -#define ZSTD_FRAMEHEADERSIZE_MIN 5 #define LOG_WRAPPER(...) // printf(__VA_ARGS__) +#define MIN(a,b) ((a)<(b)?(a):(b)) + #define FINISH_WITH_ERR(msg) { \ fprintf(stderr, "ERROR: %s\n", msg); \ return Z_MEM_ERROR; \ @@ -253,9 +253,8 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm, typedef struct { ZBUFF_DCtx* zbd; - char headerBuf[ZSTD_FRAMEHEADERSIZE_MIN]; + char headerBuf[ZWRAP_HEADERSIZE]; int errorCount; - int requiredHeaderSize; /* zlib params */ int stream_size; @@ -298,7 +297,6 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, zwd->zalloc = strm->zalloc; zwd->zfree = strm->zfree; zwd->opaque = strm->opaque; - zwd->requiredHeaderSize = ZWRAP_HEADERSIZE; strm->state = (struct internal_state*) zwd; strm->total_in = 0; @@ -360,14 +358,14 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) size_t errorCode, dstCapacity, srcSize; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); - while (strm->total_in < ZSTD_FRAMEHEADERSIZE_MIN) + if (strm->total_in < ZWRAP_HEADERSIZE) { - srcSize = MIN(strm->avail_in, zwd->requiredHeaderSize - strm->total_in); + srcSize = MIN(strm->avail_in, ZWRAP_HEADERSIZE - strm->total_in); memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); strm->total_in += srcSize; strm->next_in += srcSize; strm->avail_in -= srcSize; - if (strm->total_in < zwd->requiredHeaderSize) return Z_OK; + if (strm->total_in < ZWRAP_HEADERSIZE) return Z_OK; if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { z_stream strm2; @@ -401,16 +399,11 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); } - if (zwd->requiredHeaderSize < ZSTD_FRAMEHEADERSIZE_MIN) { - zwd->requiredHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN; - continue; - } - zwd->zbd = ZBUFF_createDCtx(); { size_t const errorCode = ZBUFF_decompressInit(zwd->zbd); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } - srcSize = ZSTD_FRAMEHEADERSIZE_MIN; + srcSize = ZWRAP_HEADERSIZE; dstCapacity = 0; errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize); LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity); @@ -419,7 +412,6 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return Z_MEM_ERROR; } if (strm->avail_in == 0) return Z_OK; - break; } srcSize = strm->avail_in; From b37b79e86865cb0af658ec11b50dca30bf979770 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 10:24:57 +0200 Subject: [PATCH 094/208] VS projects: fixed xxhash dependency in fullbench --- projects/VS2008/fullbench/fullbench.vcproj | 8 ++++++++ projects/VS2010/fullbench/fullbench.vcxproj | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/projects/VS2008/fullbench/fullbench.vcproj index 6ab878cc9..85a37c0f3 100644 --- a/projects/VS2008/fullbench/fullbench.vcproj +++ b/projects/VS2008/fullbench/fullbench.vcproj @@ -344,6 +344,10 @@ RelativePath="..\..\..\lib\common\fse_decompress.c" > + + @@ -411,6 +415,10 @@ > + + diff --git a/projects/VS2010/fullbench/fullbench.vcxproj b/projects/VS2010/fullbench/fullbench.vcxproj index 2d9b78ce1..475765350 100644 --- a/projects/VS2010/fullbench/fullbench.vcxproj +++ b/projects/VS2010/fullbench/fullbench.vcxproj @@ -1,4 +1,4 @@ - + @@ -158,6 +158,7 @@ + @@ -174,6 +175,7 @@ + From 8b452453ce7dadad8249907eb845f92e107c6c2f Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 10:50:17 +0200 Subject: [PATCH 095/208] zlibWrapper: Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd --- appveyor.yml | 24 ++++++++++++------------ zlibWrapper/zstd_zlibwrapper.c | 6 +++++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 3d46ddc19..10da235e6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,24 +1,24 @@ version: 1.0.{build} environment: matrix: + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw64" - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw32" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 0d18cbb27..7ec68a4c5 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -40,6 +40,7 @@ #define Z_INFLATE_SYNC 8 #define ZWRAP_HEADERSIZE 4 +#define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ #define LOG_WRAPPER(...) // printf(__VA_ARGS__) @@ -116,6 +117,9 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, zwc = ZWRAP_createCCtx(); if (zwc == NULL) return Z_MEM_ERROR; + if (level == Z_DEFAULT_COMPRESSION) + level = ZWRAP_DEFAULT_CLEVEL; + { size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } @@ -127,7 +131,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, } -ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, +ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy, const char *version, int stream_size)) From 13f42d9085832d5988551dcacac6d4f25dd0ba06 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 14:44:31 +0200 Subject: [PATCH 096/208] VS2010 project: reverted zstdlib.rc --- Makefile | 79 ++++++++++++++----------- lib/decompress/zstd_decompress.c | 3 +- projects/VS2010/zstdlib/zstdlib.rc | 51 ++++++++++++++++ projects/VS2010/zstdlib/zstdlib.vcxproj | 4 +- 4 files changed, 102 insertions(+), 35 deletions(-) create mode 100644 projects/VS2010/zstdlib/zstdlib.rc diff --git a/Makefile b/Makefile index 3c600f54a..db58d430c 100644 --- a/Makefile +++ b/Makefile @@ -68,8 +68,9 @@ clean: #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) - +HOST_OS = POSIX install: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ @@ -81,43 +82,13 @@ uninstall: travis-install: $(MAKE) install PREFIX=~/install_test_dir -cmaketest: - cmake --version - rm -rf projects/cmake/build - mkdir projects/cmake/build - cd projects/cmake/build ; cmake .. ; $(MAKE) +gpptest: clean + $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" clangtest: clean clang -v $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" -gpptest: clean - $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - -c90test: clean - CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long - -gnu90test: clean - CFLAGS="-std=gnu90" $(MAKE) all - -c99test: clean - CFLAGS="-std=c99" $(MAKE) all - -gnu99test: clean - CFLAGS="-std=gnu99" $(MAKE) all - -c11test: clean - CFLAGS="-std=c11" $(MAKE) all - -bmix64test: clean - CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test - -bmix32test: clean - CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test - -bmi32test: clean - CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test - armtest: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" @@ -168,3 +139,45 @@ uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +CMAKE_PARAMS = -G"MSYS Makefiles" +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +cmaketest: + cmake --version + rm -rf projects/cmake/build + mkdir projects/cmake/build + cd projects/cmake/build ; cmake $(CMAKE_PARAMS) .. ; $(MAKE) + +c90test: clean + CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long + +gnu90test: clean + CFLAGS="-std=gnu90" $(MAKE) all + +c99test: clean + CFLAGS="-std=c99" $(MAKE) all + +gnu99test: clean + CFLAGS="-std=gnu99" $(MAKE) all + +c11test: clean + CFLAGS="-std=c11" $(MAKE) all + +bmix64test: clean + CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test + +bmix32test: clean + CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test + +bmi32test: clean + CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test +endif diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6d79e678a..e88b873c8 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -333,7 +333,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ - fparamsPtr->frameContentSize = 0; + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; return 0; } diff --git a/projects/VS2010/zstdlib/zstdlib.rc b/projects/VS2010/zstdlib/zstdlib.rc new file mode 100644 index 000000000..6c4dde486 --- /dev/null +++ b/projects/VS2010/zstdlib/zstdlib.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "zstd.h" /* ZSTD_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION ZSTD_LIB_VERSION + PRODUCTVERSION ZSTD_LIB_VERSION + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Yann Collet" + VALUE "FileDescription", "Fast and efficient compression algorithm" + VALUE "FileVersion", ZSTD_VERSION_STRING + VALUE "InternalName", "zstdlib.dll" + VALUE "LegalCopyright", "Copyright (C) 2013-2015, Yann Collet" + VALUE "OriginalFilename", "zstdlib.dll" + VALUE "ProductName", "Zstandard" + VALUE "ProductVersion", ZSTD_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj index 79900501e..e0e3dbb6a 100644 --- a/projects/VS2010/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -48,7 +48,9 @@ - + + + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} From 83c3f4427c4bb42188c6abc29cd0dc2b415161b3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 17:44:53 +0200 Subject: [PATCH 097/208] upgraded zbufftest to also test advanced frame parameters no/checksum no/dictID --- lib/common/zbuff_static.h | 15 +++++++++------ programs/zbufftest.c | 27 +++++++++++++++++---------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/common/zbuff_static.h b/lib/common/zbuff_static.h index c50a3e166..44a0321d1 100644 --- a/lib/common/zbuff_static.h +++ b/lib/common/zbuff_static.h @@ -32,9 +32,9 @@ #ifndef ZSTD_BUFFERED_STATIC_H #define ZSTD_BUFFERED_STATIC_H -/* The objects defined into this file should be considered experimental. - * They are not labelled stable, as their prototype may change in the future. - * You can use them for tests, provide feedback, or if you can endure risk of future changes. +/* The objects defined into this file must be considered experimental. + * Their prototype may change in future versions. + * Never use them with a dynamic library. */ #if defined (__cplusplus) @@ -54,21 +54,24 @@ extern "C" { * Advanced functions ***************************************/ /*! ZBUFF_createCCtx_advanced() : - * Create a ZBUFF compression context using external alloc and free functions */ + * Create a ZBUFF compression context using external alloc and free functions */ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); /*! ZBUFF_createDCtx_advanced() : - * Create a ZBUFF decompression context using external alloc and free functions */ + * Create a ZBUFF decompression context using external alloc and free functions */ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); /* ************************************* * Advanced Streaming functions ***************************************/ -ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, +ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); + +/* internal util function */ + MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { size_t length = ZBUFF_MIN(dstCapacity, srcSize); diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 19a385b42..166922970 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -212,7 +212,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); { size_t r, pIn=0, pOut=0; - do + do { ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); r = 1; while (r) { @@ -346,8 +346,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; - /* state total reset */ - /* some problems only happen when states are re-used in a specific order */ + /* states full reset (unsynchronized) */ + /* some issues only happen when reusing states in a specific sequence of parameters */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); } if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); } @@ -370,15 +370,22 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); + dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; /* random dictionary selection */ - { size_t dictStart; - dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; - dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; } - { size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel); - CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError)); - } } + { ZSTD_compressionParameters cPar = ZSTD_getCParams(cLevel, 0, dictSize); + U32 const checksum = FUZ_rand(&lseed) & 1; + U32 const noDictIDFlag = FUZ_rand(&lseed) & 1; + ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag }; + ZSTD_parameters params; + ZSTD_adjustCParams(&cPar, 0, dictSize); + params.cParams = cPar; + params.fParams = fPar; + { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0); + CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError)); + } } } /* multi-segments compression test */ XXH64_reset(&xxhState, 0); From 70d1301d6e84ce4285623b10e462e259b9bee312 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 18:45:34 +0200 Subject: [PATCH 098/208] Changed `ZSTD_adjustCParams()` prototype `ZSTD_adjustCParams()` is now automatically invoked at the end of `ZSTD_getCParams()` --- lib/common/zstd_static.h | 2 +- lib/compress/zbuff_compress.c | 2 -- lib/compress/zstd_compress.c | 29 +++++++++++++++-------------- programs/bench.c | 2 -- programs/zbufftest.c | 3 +-- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e4c992bee..01aaecf6d 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -127,7 +127,7 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustParams() : * optimize params for a given `srcSize` and `dictSize`. * both values are optional, select `0` if unknown. */ -ZSTDLIB_API void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : * Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 66deb495f..ee33df541 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -182,8 +182,6 @@ size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t di ZSTD_parameters params; memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - params.fParams.contentSizeFlag = 0; - ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a9a2dc79d..7c7349b54 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -202,30 +202,32 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz } -/** ZSTD_adjustParams() : - optimize params for q given input (`srcSize` and `dictSize`). +/** ZSTD_adjustCParams() : + optimize cPar for a given input (`srcSize` and `dictSize`). mostly downsizing to reduce memory consumption and initialization. Both `srcSize` and `dictSize` are optional (use 0 if unknown), but if both are 0, no optimization can be done. - Note : params is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize) + Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize) { - if (srcSize+dictSize == 0) return; /* no size information available : no adjustment */ + if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ /* resize params, to use less memory when necessary */ { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1<windowLog > srcLog) params->windowLog = srcLog; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; } } - if (params->hashLog > params->windowLog) params->hashLog = params->windowLog; - { U32 const btPlus = (params->strategy == ZSTD_btlazy2) || (params->strategy == ZSTD_btopt); - U32 const maxChainLog = params->windowLog+btPlus; - if (params->chainLog > maxChainLog) params->chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ + if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; + { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt); + U32 const maxChainLog = cPar.windowLog+btPlus; + if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ - if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ - if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ + + return cPar; } @@ -2362,7 +2364,6 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict ZSTD_parameters params; memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); } @@ -2472,7 +2473,6 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); params.fParams.contentSizeFlag = 1; - ZSTD_adjustCParams(¶ms.cParams, srcSize, dictSize); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } @@ -2625,5 +2625,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; } + cp = ZSTD_adjustCParams(cp, srcSize, dictSize); return cp; } diff --git a/programs/bench.c b/programs/bench.c index 52c64d525..97f71162b 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -220,7 +220,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { ZSTD_parameters params; params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); params.fParams.contentSizeFlag = 1; - ZSTD_adjustCParams(¶ms.cParams, blockSize, dictBufferSize); { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); if (ZSTD_isError(initResult)) break; } } @@ -505,4 +504,3 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); return 0; } - diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 166922970..fd9269bd6 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -375,12 +375,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); dict = srcBuffer + dictStart; } - { ZSTD_compressionParameters cPar = ZSTD_getCParams(cLevel, 0, dictSize); + { ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); U32 const checksum = FUZ_rand(&lseed) & 1; U32 const noDictIDFlag = FUZ_rand(&lseed) & 1; ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag }; ZSTD_parameters params; - ZSTD_adjustCParams(&cPar, 0, dictSize); params.cParams = cPar; params.fParams = fPar; { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0); From 9242816b5640f99547dd731f0e947b67eafa113e Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 18:47:04 +0200 Subject: [PATCH 099/208] fparamsPtr->windowLog==0 means that a frame is skippable --- lib/common/zstd_static.h | 9 +++++++++ lib/decompress/zbuff_decompress.c | 2 ++ lib/decompress/zstd_decompress.c | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e4c992bee..19118f464 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -246,6 +246,15 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. Context can then be reset to start a new decompression. + + Skippable frames allow the integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following: + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_decompressContinue() always returns 0. + For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. + It also returns Frame Size as fparamsPtr->frameContentSize. */ diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 72456fd5f..69898fdb9 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -192,6 +192,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (ZSTD_isError(h2Result)) return h2Result; } } + if (zbd->fParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) zbd->fParams.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for buffer allocation */ + /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index e88b873c8..27dd780ad 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -335,7 +335,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ memset(fparamsPtr, 0, sizeof(*fparamsPtr)); fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); - fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; + fparamsPtr->windowLog = 0; /* windowLog==0 means a frame is skippable */ return 0; } return ERROR(prefix_unknown); From eb70043e5937cf144ba9a85eeb5e5f21e231b612 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 18:59:55 +0200 Subject: [PATCH 100/208] Fixed paramgrill --- programs/paramgrill.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 6dc90b219..cb9a2ea49 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -600,22 +600,22 @@ static void playAround(FILE* f, winnerInfo_t* winners, } -static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance) +static ZSTD_compressionParameters randomParams(void) { - U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1)); + ZSTD_compressionParameters p; U32 validated = 0; - if (!chance) while (!validated) { /* totally random entry */ - p->chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN; - p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; - p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN; - p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN; - p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; - p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; - p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(*p)); + p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN; + p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; + p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN; + p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN; + p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; + p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; + p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); } + return p; } static void BMK_selectRandomStart( @@ -623,12 +623,10 @@ static void BMK_selectRandomStart( const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx) { - U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); + U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); if ((id==0) || (winners[id].params.windowLog==0)) { /* totally random entry */ - ZSTD_compressionParameters p; - potentialRandomParams(&p, 1); - ZSTD_adjustCParams(&p, srcSize, 0); + ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); playAround(f, winners, p, srcBuffer, srcSize, ctx); } else @@ -649,7 +647,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) if (g_singleRun) { BMK_result_t testResult; - ZSTD_adjustCParams(&g_params, srcSize, 0); + g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); DISPLAY("\n"); return; @@ -861,7 +859,7 @@ int optimizeForSize(char* inFileName) do { params = winner.params; paramVariation(¶ms); - potentialRandomParams(¶ms, 16); + if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams(); /* exclude faster if already played set of params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; From 87cfbe3e5fede455209457120b6292d7e389f20e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 19:22:15 +0200 Subject: [PATCH 101/208] checksum generation can be selected from command line (`--checksum`) --- programs/fileio.c | 3 +++ programs/fileio.h | 1 + programs/zstdcli.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index 55e36cc55..48555424c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -135,6 +135,8 @@ static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file ye void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } static U32 g_dictIDFlag = 1; void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } +static U32 g_checksumFlag = 0; +void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } /*-************************************* @@ -313,6 +315,7 @@ static int FIO_compressFilename_internal(cRess_t ress, memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize); params.fParams.contentSizeFlag = 1; + params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); diff --git a/programs/fileio.h b/programs/fileio.h index 5a9cdc1ce..01e308340 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -49,6 +49,7 @@ void FIO_setNotificationLevel(unsigned level); void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setDictIDFlag(unsigned dictIDFlag); +void FIO_setChecksumFlag(unsigned checksumFlag); /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 24bba89db..f31b6e1f8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -134,6 +134,7 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n"); + DISPLAY( "--checksum:compute checksum of content, for decompressor validation\n"); #endif #ifndef ZSTD_NODECOMPRESS DISPLAY( " -t : test compressed file integrity \n"); @@ -240,6 +241,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } + if (!strcmp(argument, "--checksum")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } From bf25d7ac574bac5da3e1fbdc4dc6273392d4daf2 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 10:19:35 +0200 Subject: [PATCH 102/208] zlibWrapper: support for zlib versions from 1.2.3 to 1.2.8 --- .travis.yml | 12 ++--- zlibWrapper/Makefile | 9 ++-- zlibWrapper/zstd_zlibwrapper.c | 84 +++++++++++++++++++++------------- zlibWrapper/zstd_zlibwrapper.h | 6 ++- 4 files changed, 64 insertions(+), 47 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6a32c255..41f67c77a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ matrix: - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required @@ -53,15 +56,6 @@ matrix: - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test - #- os: linux - # sudo: required - # env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=zlibwrapper # Ubuntu 14.04 LTS Server Edition 64 bit - os: linux dist: trusty diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 3f293cd90..6c00622b0 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -7,8 +7,9 @@ # Paths to static and dynamic zlib and zstd libraries ifneq (,$(filter Windows%,$(OS))) -STATICLIB = ../../zlib/libz.a ../lib/libzstd.a -IMPLIB = ../../zlib/libz.dll.a ../lib/libzstd.a +ZLIBDIR = ../../zlib1.2.8 +STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a +IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a else STATICLIB = -static -lz ../lib/libzstd.a IMPLIB = -lz ../lib/libzstd.a @@ -17,12 +18,12 @@ endif ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples CC = gcc -CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 +CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 LDFLAGS = $(LOC) RM = rm -f -all: clean test testdll testzstd +all: clean test testzstd test: example ./example diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 7ec68a4c5..c9a8b1d13 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -31,6 +31,7 @@ #include /* fprintf */ #include /* malloc */ +#include /* va_list */ #include #include "zstd_zlibwrapper.h" #include "zstd.h" @@ -492,6 +493,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, } +#if ZLIB_VERNUM >= 0x1260 ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, unsigned *pending, int *bits)) @@ -500,6 +502,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, return deflatePending(strm, pending, bits); FINISH_WITH_ERR("deflatePending is not supported!"); } +#endif ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, @@ -524,6 +527,7 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, /* Advanced compression functions */ +#if ZLIB_VERNUM >= 0x1280 ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, Bytef *dictionary, uInt *dictLength)) @@ -532,7 +536,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, return inflateGetDictionary(strm, dictionary, dictLength); FINISH_WITH_ERR("inflateGetDictionary is not supported!"); } - +#endif ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, @@ -552,6 +556,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) } +#if ZLIB_VERNUM >= 0x1240 ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, int windowBits)) { @@ -559,6 +564,17 @@ ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, return inflateReset2(strm, windowBits); FINISH_WITH_ERR("inflateReset2 is not supported!"); } +#endif + + +#if ZLIB_VERNUM >= 0x1240 +ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) +{ + if (!strm->reserved) + return inflateMark(strm); + FINISH_WITH_ERR("inflateMark is not supported!"); +} +#endif ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, @@ -571,14 +587,6 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, } -ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) -{ - if (!strm->reserved) - return inflateMark(strm); - FINISH_WITH_ERR("inflateMark is not supported!"); -} - - ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, gz_headerp head)) { @@ -699,6 +707,7 @@ ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode)) } +#if ZLIB_VERNUM >= 0x1240 ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) { if (!g_useZSTD) @@ -707,6 +716,31 @@ ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) } +ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) +{ + if (!g_useZSTD) + return gzoffset(file); + FINISH_WITH_ERR("gzoffset is not supported!"); +} + + +ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) +{ + if (!g_useZSTD) + return gzclose_r(file); + FINISH_WITH_ERR("gzclose_r is not supported!"); +} + + +ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) +{ + if (!g_useZSTD) + return gzclose_w(file); + FINISH_WITH_ERR("gzclose_w is not supported!"); +} +#endif + + ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) { if (!g_useZSTD) @@ -732,7 +766,11 @@ ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file, } +#if ZLIB_VERNUM >= 0x1260 ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...)) +#else +ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) +#endif { if (!g_useZSTD) { int ret; @@ -774,7 +812,11 @@ ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c)) } +#if ZLIB_VERNUM == 0x1260 +ZEXTERN int ZEXPORT z_gzgetc_ OF((gzFile file)) +#else ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) +#endif { if (!g_useZSTD) return gzgetc(file); @@ -822,14 +864,6 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) } -ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) -{ - if (!g_useZSTD) - return gzoffset(file); - FINISH_WITH_ERR("gzoffset is not supported!"); -} - - ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) { if (!g_useZSTD) @@ -854,22 +888,6 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) } -ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) -{ - if (!g_useZSTD) - return gzclose_r(file); - FINISH_WITH_ERR("gzclose_r is not supported!"); -} - - -ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) -{ - if (!g_useZSTD) - return gzclose_w(file); - FINISH_WITH_ERR("gzclose_w is not supported!"); -} - - ZEXTERN const char * ZEXPORT z_gzerror OF((gzFile file, int *errnum)) { if (!g_useZSTD) diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index c2c908ddb..e438789ac 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -41,7 +41,11 @@ extern "C" { #include #if !defined(z_const) -# define z_const const +#if ZLIB_VERNUM >= 0x1260 + #define z_const const +#else + #define z_const +#endif #endif void useZSTD(int turn_on); From e02bf99fa4ec7f93b27204a33c545e1a7f18250b Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 12:00:32 +0200 Subject: [PATCH 103/208] zlibWrapper: fixed memory deallocation bug with zstd decompression --- zlibWrapper/zstd_zlibwrapper.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index c9a8b1d13..ff85a8613 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -125,7 +125,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } zwc->compressionLevel = level; - strm->opaque = (voidpf) zwc; + strm->state = (struct internal_state*) zwc; /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; return Z_OK; @@ -151,7 +151,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, if (!g_useZSTD) return deflateSetDictionary(strm, dictionary, dictLength); - { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque; + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type); { size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } @@ -171,7 +171,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) return res; } - zwc = (ZWRAP_CCtx*) strm->opaque; + zwc = (ZWRAP_CCtx*) strm->state; LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (strm->avail_in > 0) { @@ -220,7 +220,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) return deflateEnd(strm); } LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); - { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque; + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; size_t const errorCode = ZWRAP_freeCCtx(zwc); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } @@ -265,9 +265,6 @@ typedef struct { int stream_size; char *version; int windowBits; - alloc_func zalloc; /* used to allocate the internal state */ - free_func zfree; /* used to free the internal state */ - voidpf opaque; /* private data object passed to zalloc and zfree */ } ZWRAP_DCtx; @@ -299,11 +296,8 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, zwd->stream_size = stream_size; zwd->version = strdup(version); - zwd->zalloc = strm->zalloc; - zwd->zfree = strm->zfree; - zwd->opaque = strm->opaque; - strm->state = (struct internal_state*) zwd; + strm->state = (struct internal_state*) zwd; /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; strm->reserved = 1; /* mark as unknown steam */ @@ -399,7 +393,11 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_in = strm2.avail_in; strm->next_out = strm2.next_out; strm->avail_out = strm2.avail_out; + strm->reserved = 0; /* mark as zlib stream */ + { size_t const errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + if (flush == Z_INFLATE_SYNC) return inflateSync(strm); return inflate(strm, flush); } @@ -444,7 +442,7 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) { int ret = Z_OK; if (!strm->reserved) - ret = inflateEnd(strm); + return inflateEnd(strm); LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; From 2866951558d8038dc872d4811f22cd0756c0d506 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 13:04:18 +0200 Subject: [PATCH 104/208] opaque parameter for custom memory allocation functions --- lib/common/zstd_common.c | 18 ++++++++++++++++++ lib/common/zstd_static.h | 11 ++++++++--- lib/compress/zbuff_compress.c | 27 ++++++++++++--------------- lib/compress/zstd_compress.c | 31 ++++++++++++------------------- lib/decompress/zbuff_decompress.c | 27 ++++++++++++--------------- lib/decompress/zstd_decompress.c | 17 ++++++----------- programs/zbufftest.c | 14 ++++++++------ 7 files changed, 76 insertions(+), 69 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 2a2c39d60..c4ec46fb9 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -33,6 +33,7 @@ /*-************************************* * Dependencies ***************************************/ +#include /* malloc */ #include "error_private.h" #include "zstd_static.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode */ #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ @@ -70,3 +71,20 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(c unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + + + +void* ZSTD_defaultAllocFunction(void* opaque, size_t size) +{ + (void)opaque; + void* address = malloc(size); + /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ + return address; +} + +void ZSTD_defaultFreeFunction(void* opaque, void* address) +{ + (void)opaque; + /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ + free(address); +} diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index 19118f464..b36e92c87 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -97,9 +97,14 @@ typedef struct { ZSTD_frameParameters fParams; } ZSTD_parameters; -typedef void* (*ZSTD_allocFunction) (size_t size); -typedef void (*ZSTD_freeFunction) (void* address); -typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem; +/* custom memory allocation functions */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; + +void* ZSTD_defaultAllocFunction(void* opaque, size_t size); +void ZSTD_defaultFreeFunction(void* opaque, void* address); +static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; /*-************************************* diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 66deb495f..97d87b992 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -95,13 +95,12 @@ struct ZBUFF_CCtx_s { size_t outBuffContentSize; size_t outBuffFlushedSize; ZBUFF_cStage stage; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; }; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ ZBUFF_CCtx* ZBUFF_createCCtx(void) { - ZSTD_customMem customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZBUFF_createCCtx_advanced(customMem); } @@ -113,8 +112,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) { zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; - zbc->customAlloc = malloc; - zbc->customFree = free; + memcpy(&zbc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); zbc->zc = ZSTD_createCCtx(); return zbc; } @@ -122,11 +120,10 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc || !customMem.customFree) return NULL; - zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); + zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; memset(zbc, 0, sizeof(ZBUFF_CCtx)); - zbc->customAlloc = customMem.customAlloc; - zbc->customFree = customMem.customFree; + memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem)); zbc->zc = ZSTD_createCCtx_advanced(customMem); return zbc; } @@ -135,9 +132,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) { if (zbc==NULL) return 0; /* support free on NULL */ ZSTD_freeCCtx(zbc->zc); - zbc->customFree(zbc->inBuff); - zbc->customFree(zbc->outBuff); - zbc->customFree(zbc); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); + zbc->customMem.customFree(zbc->customMem.opaque, zbc); return 0; } @@ -152,16 +149,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zbc->inBuffSize < neededInBuffSize) { zbc->inBuffSize = neededInBuffSize; - zbc->customFree(zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); } if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; - zbc->customFree(zbc->outBuff); /* should not be necessary */ - zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize); if (zbc->outBuff == NULL) return ERROR(memory_allocation); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a9a2dc79d..eb2df3b42 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -107,8 +107,7 @@ struct ZSTD_CCtx_s size_t workSpaceSize; size_t blockSize; XXH64_state_t xxhState; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -123,7 +122,7 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - ZSTD_customMem customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZSTD_createCCtx_advanced(customMem); } @@ -135,28 +134,24 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); if (!ctx) return NULL; - - ctx->customAlloc = malloc; - ctx->customFree = free; + memcpy(&ctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); return ctx; } if (!customMem.customAlloc || !customMem.customFree) return NULL; - ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + ctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx)); if (!ctx) return NULL; - memset(ctx, 0, sizeof(ZSTD_CCtx)); - ctx->customAlloc = customMem.customAlloc; - ctx->customFree = customMem.customFree; + memcpy(&ctx->customMem, &customMem, sizeof(ZSTD_customMem)); return ctx; } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - cctx->customFree(cctx->workSpace); - cctx->customFree(cctx); + cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace); + cctx->customMem.customFree(cctx->customMem.opaque, cctx); return 0; /* reserved as a potential error code in the future */ } @@ -262,8 +257,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); if (zc->workSpaceSize < neededSpace) { - zc->customFree(zc->workSpace); - zc->workSpace = zc->customAlloc(neededSpace); + zc->customMem.customFree(zc->customMem.opaque, zc->workSpace); + zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; } } @@ -322,8 +317,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) if (srcCCtx->stage!=1) return ERROR(stage_wrong); dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ - dstCCtx->customAlloc = srcCCtx->customAlloc; - dstCCtx->customFree = srcCCtx->customFree; + memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ @@ -2487,10 +2481,9 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); - ctxBody.customAlloc = malloc; - ctxBody.customFree = free; + memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); - ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ + ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 69898fdb9..c4321498b 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -82,14 +82,13 @@ struct ZBUFF_DCtx_s { size_t blockSize; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ ZBUFF_DCtx* ZBUFF_createDCtx(void) { - ZSTD_customMem customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZBUFF_createDCtx_advanced(customMem); } @@ -101,8 +100,7 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) { zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx)); if (zbd==NULL) return NULL; - zbd->customAlloc = malloc; - zbd->customFree = free; + memcpy(&zbd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); zbd->zd = ZSTD_createDCtx(); zbd->stage = ZBUFFds_init; return zbd; @@ -111,11 +109,10 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc || !customMem.customFree) return NULL; - zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx)); + zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx)); if (zbd==NULL) return NULL; memset(zbd, 0, sizeof(ZBUFF_DCtx)); - zbd->customAlloc = customMem.customAlloc; - zbd->customFree = customMem.customFree; + memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem)); zbd->zd = ZSTD_createDCtx_advanced(customMem); zbd->stage = ZBUFFds_init; return zbd; @@ -125,9 +122,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) { if (zbd==NULL) return 0; /* support free on null */ ZSTD_freeDCtx(zbd->zd); - zbd->customFree(zbd->inBuff); - zbd->customFree(zbd->outBuff); - zbd->customFree(zbd); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd); return 0; } @@ -198,16 +195,16 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; if (zbd->inBuffSize < blockSize) { - zbd->customFree(zbd->inBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)zbd->customAlloc(blockSize); + zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; if (zbd->outBuffSize < neededOutSize) { - zbd->customFree(zbd->outBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)zbd->customAlloc(neededOutSize); + zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); if (zbd->outBuff == NULL) return ERROR(memory_allocation); } } } zbd->stage = ZBUFFds_read; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 27dd780ad..461741d9a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -120,8 +120,7 @@ struct ZSTD_DCtx_s size_t headerSize; ZSTD_frameParams fParams; XXH64_state_t xxhState; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 dictID; @@ -156,9 +155,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc && !customMem.customFree) { dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; - dctx->customAlloc = malloc; - dctx->customFree = free; - + memcpy(&dctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); ZSTD_decompressBegin(dctx); return dctx; } @@ -166,25 +163,23 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc || !customMem.customFree) return NULL; - dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); + dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx)); if (!dctx) return NULL; - dctx->customAlloc = customMem.customAlloc; - dctx->customFree = customMem.customFree; - + memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem)); ZSTD_decompressBegin(dctx); return dctx; } ZSTD_DCtx* ZSTD_createDCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZSTD_createDCtx_advanced(customMem); } size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { - dctx->customFree(dctx); + dctx->customMem.customFree(dctx->customMem.opaque, dctx); return 0; /* reserved as a potential error code in the future */ } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 19a385b42..f57c4cdf9 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -129,16 +129,18 @@ static unsigned FUZ_highbit32(U32 v32) } */ -void* ZBUFF_allocFunction(size_t size) +void* ZBUFF_allocFunction(void* opaque, size_t size) { + (void)opaque; void* address = malloc(size); - /* DISPLAYLEVEL(4, "alloc %p, %d \n", address, (int)size); */ + /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ return address; } -void ZBUFF_freeFunction(void* address) +void ZBUFF_freeFunction(void* opaque, void* address) { - /* if (address) DISPLAYLEVEL(4, "free %p \n", address); */ + (void)opaque; + /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ free(address); } @@ -512,8 +514,8 @@ int main(int argc, const char** argv) int result=0; U32 mainPause = 0; const char* programName = argv[0]; - ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction }; - ZSTD_customMem customNULL = { NULL, NULL }; + ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, (void*)777 }; + ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Thu, 2 Jun 2016 15:11:39 +0200 Subject: [PATCH 105/208] default custom allocation functions moved to zstd_internal.h --- lib/common/zstd_common.c | 2 +- lib/common/zstd_internal.h | 5 +++++ lib/common/zstd_static.h | 4 ---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index c4ec46fb9..262eeb232 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -76,8 +76,8 @@ const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(error void* ZSTD_defaultAllocFunction(void* opaque, size_t size) { - (void)opaque; void* address = malloc(size); + (void)opaque; /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ return address; } diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 63e56aaf0..8c647b511 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -255,4 +255,9 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); +/* custom memory allocation functions */ +void* ZSTD_defaultAllocFunction(void* opaque, size_t size); +void ZSTD_defaultFreeFunction(void* opaque, void* address); +MEM_STATIC ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index b36e92c87..f452e6962 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -102,10 +102,6 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -void* ZSTD_defaultAllocFunction(void* opaque, size_t size); -void ZSTD_defaultFreeFunction(void* opaque, void* address); -static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; - /*-************************************* * Advanced functions From ff9114aee3123a4e80effe848776b746c3e4cc91 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 16:52:36 +0200 Subject: [PATCH 106/208] zlibWrapper: added support for custom memory allocation functions --- lib/common/zstd_common.c | 4 ++-- lib/common/zstd_internal.h | 2 +- programs/zbufftest.c | 4 ++-- zlibWrapper/examples/example.c | 5 +++- zlibWrapper/zstd_zlibwrapper.c | 43 +++++++++++++++++++++++++++++----- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 262eeb232..5d3db0ea0 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -78,13 +78,13 @@ void* ZSTD_defaultAllocFunction(void* opaque, size_t size) { void* address = malloc(size); (void)opaque; - /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ + /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ return address; } void ZSTD_defaultFreeFunction(void* opaque, void* address) { (void)opaque; - /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ + /* if (address) printf("free %p opaque=%p \n", address, opaque); */ free(address); } diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8c647b511..230ce79fc 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -258,6 +258,6 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); /* custom memory allocation functions */ void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); -MEM_STATIC ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; +static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f57c4cdf9..18382abf9 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -133,14 +133,14 @@ void* ZBUFF_allocFunction(void* opaque, size_t size) { (void)opaque; void* address = malloc(size); - /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ + /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ return address; } void ZBUFF_freeFunction(void* opaque, void* address) { (void)opaque; - /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ + /* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */ free(address); } diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index fe6c80ac1..3fea9d2ad 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -62,11 +62,14 @@ void *myalloc(q, n, m) unsigned n, m; { q = Z_NULL; - return calloc(n, m); + void *buf = calloc(n, m); + /* printf("myalloc %p n=%d m=%d\n", buf, n, m); */ + return buf; } void myfree(void *q, void *p) { + /* printf("myfree %p\n", p); */ q = Z_NULL; free(p); } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index ff85a8613..26f173b13 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -37,6 +37,7 @@ #include "zstd.h" #include "zstd_static.h" /* ZSTD_MAGICNUMBER */ #include "zbuff.h" +#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ #define Z_INFLATE_SYNC 8 @@ -75,6 +76,22 @@ const char * zstdVersion() { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } +void* ZWRAP_allocFunction(void* opaque, size_t size) +{ + z_streamp strm = (z_streamp) opaque; + void* address = strm->zalloc(strm->opaque, 1, size); + /* printf("ZWRAP alloc %p, %d \n", address, (int)size); */ + return address; +} + +void ZWRAP_freeFunction(void* opaque, void* address) +{ + z_streamp strm = (z_streamp) opaque; + strm->zfree(strm->opaque, address); + /* if (address) printf("ZWRAP free %p \n", address); */ +} + + /* *** Compression *** */ @@ -82,15 +99,22 @@ typedef struct { ZBUFF_CCtx* zbc; size_t bytesLeft; int compressionLevel; + z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_CCtx; -ZWRAP_CCtx* ZWRAP_createCCtx() +ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*)malloc(sizeof(ZWRAP_CCtx)); if (zwc==NULL) return NULL; memset(zwc, 0, sizeof(*zwc)); - zwc->zbc = ZBUFF_createCCtx(); + if (strm->zalloc && strm->zfree) { + ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; + memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); + zwc->zbc = ZBUFF_createCCtx_advanced(ZWRAP_customMem); + } + else + zwc->zbc = ZBUFF_createCCtx(); return zwc; } @@ -115,7 +139,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, } LOG_WRAPPER("- deflateInit level=%d\n", level); - zwc = ZWRAP_createCCtx(); + zwc = ZWRAP_createCCtx(strm); if (zwc == NULL) return Z_MEM_ERROR; if (level == Z_DEFAULT_COMPRESSION) @@ -265,14 +289,16 @@ typedef struct { int stream_size; char *version; int windowBits; + z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_DCtx; -ZWRAP_DCtx* ZWRAP_createDCtx(void) +ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)malloc(sizeof(ZWRAP_DCtx)); if (zwd==NULL) return NULL; memset(zwd, 0, sizeof(*zwd)); + memcpy(&zwd->allocFunc, strm, sizeof(z_stream)); return zwd; } @@ -290,7 +316,7 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, const char *version, int stream_size)) { - ZWRAP_DCtx* zwd = ZWRAP_createDCtx(); + ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm); LOG_WRAPPER("- inflateInit\n"); if (zwd == NULL) return Z_MEM_ERROR; @@ -402,7 +428,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); } - zwd->zbd = ZBUFF_createDCtx(); + if (zwd->allocFunc.zalloc && zwd->allocFunc.zfree) { + ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; + zwd->zbd = ZBUFF_createDCtx_advanced(ZWRAP_customMem); + } else + zwd->zbd = ZBUFF_createDCtx(); + { size_t const errorCode = ZBUFF_decompressInit(zwd->zbd); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } From 2c4acdabdc85b86e957f753e10d12ddb9cc5dcde Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 2 Jun 2016 17:05:50 +0200 Subject: [PATCH 107/208] Changed command name to `--check` , `-C`, following `xz` convention --- programs/zstd.1 | 9 ++++++++- programs/zstdcli.c | 9 ++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index 1bab57abb..cc62eb30f 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -52,7 +52,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core. use `file` as Dictionary to compress or decompress FILE(s) .TP .B \-o file - save result into `file` (only possible with a single input FILE) + save result into `file` (only possible with a single INPUT-FILE) .TP .BR \-f ", " --force overwrite output without prompting @@ -71,6 +71,13 @@ It also features a very fast decoder, with speed > 500 MB/s per core. .TP .BR \-c ", " --stdout force write to standard output, even if it is the console +.TP +.BR \-C ", " --check + add integrity check computed from uncompressed data +.TP +.BR \-t ", " --test + Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR. + No files are created or removed. .SH DICTIONARY .PP diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f31b6e1f8..df7a33208 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -134,10 +134,10 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n"); - DISPLAY( "--checksum:compute checksum of content, for decompressor validation\n"); + DISPLAY( "--check : enable integrity check\n"); #endif #ifndef ZSTD_NODECOMPRESS - DISPLAY( " -t : test compressed file integrity \n"); + DISPLAY( "--test : test compressed file integrity \n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #endif #ifndef ZSTD_NODICT @@ -241,7 +241,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } - if (!strcmp(argument, "--checksum")) { FIO_setChecksumFlag(2); continue; } + if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } @@ -305,6 +305,9 @@ int main(int argCount, const char** argv) /* keep source file (default anyway, so useless; for gzip/xz compatibility) */ case 'k': argument++; break; + /* Checksum */ + case 'C': argument++; FIO_setChecksumFlag(2); break; + /* test compressed file */ case 't': decode=1; outFileName=nulmark; argument++; break; From 923938edde18f6aeaf3dd705cf8ed73e4ad67a6a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 2 Jun 2016 17:56:00 +0200 Subject: [PATCH 108/208] Added `-Wdeclaration-after-statement` compilation flag --- lib/Makefile | 2 +- programs/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 75b09e5de..76731abc1 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -48,7 +48,7 @@ INCLUDEDIR=$(PREFIX)/include CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) diff --git a/programs/Makefile b/programs/Makefile index 1e8a7f268..52a7ca076 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -40,7 +40,7 @@ ZSTDDIR = ../lib CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) From 7cab86f6470184ca72dbfdeba6a9cbaad6d7b04d Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 18:24:07 +0200 Subject: [PATCH 109/208] ZWRAP_allocFunction and ZWRAP_freeFunction are static --- Makefile | 2 +- programs/zbufftest.c | 6 +++--- zlibWrapper/zstd_zlibwrapper.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index db58d430c..c8b5831ba 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ ppctest: clean # for Travis CI ppcinstall: clean - sudo apt-get update -y -q + # sudo apt-get update -y -q sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04 # for Travis CI diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 18382abf9..b1d5b47c5 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -129,7 +129,7 @@ static unsigned FUZ_highbit32(U32 v32) } */ -void* ZBUFF_allocFunction(void* opaque, size_t size) +static void* ZBUFF_allocFunction(void* opaque, size_t size) { (void)opaque; void* address = malloc(size); @@ -137,7 +137,7 @@ void* ZBUFF_allocFunction(void* opaque, size_t size) return address; } -void ZBUFF_freeFunction(void* opaque, void* address) +static void ZBUFF_freeFunction(void* opaque, void* address) { (void)opaque; /* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */ @@ -514,7 +514,7 @@ int main(int argc, const char** argv) int result=0; U32 mainPause = 0; const char* programName = argv[0]; - ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, (void*)777 }; + ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL }; ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 26f173b13..b55b760af 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -76,7 +76,7 @@ const char * zstdVersion() { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } -void* ZWRAP_allocFunction(void* opaque, size_t size) +static void* ZWRAP_allocFunction(void* opaque, size_t size) { z_streamp strm = (z_streamp) opaque; void* address = strm->zalloc(strm->opaque, 1, size); @@ -84,7 +84,7 @@ void* ZWRAP_allocFunction(void* opaque, size_t size) return address; } -void ZWRAP_freeFunction(void* opaque, void* address) +static void ZWRAP_freeFunction(void* opaque, void* address) { z_streamp strm = (z_streamp) opaque; strm->zfree(strm->opaque, address); From 614fddee6a71d460844de3f31057f5dcc9f9ea81 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 18:40:41 +0200 Subject: [PATCH 110/208] zlibWrapper: support for stricter gcc warnings --- zlibWrapper/Makefile | 3 ++- zlibWrapper/examples/example.c | 2 +- zlibWrapper/zstd_zlibwrapper.c | 12 ++++++------ zlibWrapper/zstd_zlibwrapper.h | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 6c00622b0..962f92941 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -18,7 +18,8 @@ endif ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples CC = gcc -CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 +CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index 3fea9d2ad..a2836cdbf 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -61,8 +61,8 @@ void *myalloc(q, n, m) void *q; unsigned n, m; { - q = Z_NULL; void *buf = calloc(n, m); + q = Z_NULL; /* printf("myalloc %p n=%d m=%d\n", buf, n, m); */ return buf; } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index b55b760af..4740e0dbd 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -69,9 +69,9 @@ static int g_useZSTD = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ void useZSTD(int turn_on) { g_useZSTD = turn_on; } -int isUsingZSTD() { return g_useZSTD; } +int isUsingZSTD(void) { return g_useZSTD; } -const char * zstdVersion() { return ZSTD_VERSION_STRING; } +const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } @@ -421,8 +421,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_out = strm2.avail_out; strm->reserved = 0; /* mark as zlib stream */ - { size_t const errorCode = ZWRAP_freeDCtx(zwd); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; if (flush == Z_INFLATE_SYNC) return inflateSync(strm); return inflate(strm, flush); @@ -434,8 +434,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) } else zwd->zbd = ZBUFF_createDCtx(); - { size_t const errorCode = ZBUFF_decompressInit(zwd->zbd); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + errorCode = ZBUFF_decompressInit(zwd->zbd); + if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; srcSize = ZWRAP_HEADERSIZE; dstCapacity = 0; diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index e438789ac..d14c3a92a 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -49,8 +49,8 @@ extern "C" { #endif void useZSTD(int turn_on); -int isUsingZSTD(); -const char * zstdVersion(); +int isUsingZSTD(void); +const char * zstdVersion(void); #if defined (__cplusplus) From ff2d18928d8a491624b29a6410fd410ae2eae0b0 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 22:15:09 +0200 Subject: [PATCH 111/208] zlibWrapper: added gcc flag -Wdeclaration-after-statement --- programs/zbufftest.c | 2 +- zlibWrapper/Makefile | 2 +- zlibWrapper/zstd_zlibwrapper.c | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 355b64313..387119815 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -131,8 +131,8 @@ static unsigned FUZ_highbit32(U32 v32) static void* ZBUFF_allocFunction(void* opaque, size_t size) { - (void)opaque; void* address = malloc(size); + (void)opaque; /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ return address; } diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 962f92941..edfcdf78f 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -19,7 +19,7 @@ ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples CC = gcc CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wstrict-prototypes -Wundef +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 4740e0dbd..a4e22a118 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -667,9 +667,9 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, if (!g_useZSTD) return compress(dest, destLen, source, sourceLen); - size_t dstCapacity = *destLen; - LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); - { size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1); + { size_t dstCapacity = *destLen; + size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1); + LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; } @@ -684,8 +684,8 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, if (!g_useZSTD) return compress2(dest, destLen, source, sourceLen, level); - size_t dstCapacity = *destLen; - { size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); + { size_t dstCapacity = *destLen; + size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; } @@ -709,8 +709,8 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, // if (!g_useZSTD) return uncompress(dest, destLen, source, sourceLen); - size_t dstCapacity = *destLen; - { size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen); + { size_t dstCapacity = *destLen; + size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; } From aa224c0205eed233bd11cfeeb0b1e2f6d42eb26c Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 22:24:29 +0200 Subject: [PATCH 112/208] VS projects: remove duplicated xxhash.c and xxhash.h --- projects/VS2008/fullbench/fullbench.vcproj | 10 +--------- projects/VS2010/fullbench/fullbench.vcxproj | 1 - 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/projects/VS2008/fullbench/fullbench.vcproj index bbb7da7fa..50cbcc2cf 100644 --- a/projects/VS2008/fullbench/fullbench.vcproj +++ b/projects/VS2008/fullbench/fullbench.vcproj @@ -360,10 +360,6 @@ RelativePath="..\..\..\lib\decompress\huf_decompress.c" > - - @@ -422,12 +418,8 @@ RelativePath="..\..\..\lib\common\xxhash.h" > - - - From 764deea5f87ac8b47bb3fe5d440a52e1d33628be Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 10:11:26 +0200 Subject: [PATCH 113/208] cmake project: fixed "make install" --- projects/cmake/lib/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index 716ccbf3f..3b7bc802c 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -162,7 +162,8 @@ IF (UNIX) SET(INSTALL_INCLUDE_DIR ${PREFIX}/include) # install target - INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/zstd_buffered.h ${LIBRARY_DIR}/dictBuilder.h DESTINATION ${INSTALL_INCLUDE_DIR}) + INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zstd_static.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/common/zbuff_static.h + ${LIBRARY_DIR}/common/mem.h ${LIBRARY_DIR}/common/error_public.h ${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/zdict_static.h DESTINATION ${INSTALL_INCLUDE_DIR}) INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}) From d36ee9b845800dfbc6aa503aa6844cd98f6f80ed Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 10:24:00 +0200 Subject: [PATCH 114/208] cmake project: added "make clean-all" --- projects/cmake/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt index ae8f7be6e..9650eb4f2 100644 --- a/projects/cmake/CMakeLists.txt +++ b/projects/cmake/CMakeLists.txt @@ -52,3 +52,8 @@ ADD_SUBDIRECTORY(programs) #----------------------------------------------------------------------------- INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake) ADD_EXTRA_COMPILATION_FLAGS() + +ADD_CUSTOM_TARGET(clean-all + COMMAND ${CMAKE_BUILD_TOOL} clean + COMMAND rm -rf ${CMAKE_BINARY_DIR}/ +) From c7f008b069d25facb7b228c6a36c6dde497dd33d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 11:40:15 +0200 Subject: [PATCH 115/208] Travis cmaketest will also run "make install" and "make uninstall" --- Makefile | 2 +- projects/cmake/lib/CMakeLists.txt | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c8b5831ba..755b0c1a8 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ cmaketest: cmake --version rm -rf projects/cmake/build mkdir projects/cmake/build - cd projects/cmake/build ; cmake $(CMAKE_PARAMS) .. ; $(MAKE) + cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall c90test: clean CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index 3b7bc802c..84568b207 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -157,13 +157,15 @@ SET_TARGET_PROPERTIES( OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME}) IF (UNIX) - SET(PREFIX /usr/local) + IF ("${PREFIX}" STREQUAL "") + SET(PREFIX /usr/local) + ENDIF() + MESSAGE("the variable PREFIX=${PREFIX}") SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib) SET(INSTALL_INCLUDE_DIR ${PREFIX}/include) # install target - INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zstd_static.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/common/zbuff_static.h - ${LIBRARY_DIR}/common/mem.h ${LIBRARY_DIR}/common/error_public.h ${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/zdict_static.h DESTINATION ${INSTALL_INCLUDE_DIR}) + INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR}) INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}) From db2f54041461e09dc0cb833fa999def85fa2bf73 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 12:56:56 +0200 Subject: [PATCH 116/208] added defaultCustomNULL --- lib/common/zstd_internal.h | 3 ++- lib/compress/zbuff_compress.c | 3 +-- lib/compress/zstd_compress.c | 3 +-- lib/decompress/zbuff_decompress.c | 3 +-- lib/decompress/zstd_decompress.c | 3 +-- programs/zbufftest.c | 3 +-- zlibWrapper/Makefile | 4 ++-- 7 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 230ce79fc..b56030ba5 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -259,5 +259,6 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; - +static ZSTD_customMem const defaultCustomNULL = { NULL, NULL, NULL }; + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 6e40d3c0a..0ec27ad70 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -100,8 +100,7 @@ struct ZBUFF_CCtx_s { ZBUFF_CCtx* ZBUFF_createCCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZBUFF_createCCtx_advanced(customMem); + return ZBUFF_createCCtx_advanced(defaultCustomNULL); } ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9de86a6be..93caee0d7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -122,8 +122,7 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZSTD_createCCtx_advanced(customMem); + return ZSTD_createCCtx_advanced(defaultCustomNULL); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index c4321498b..4dfb2adae 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -88,8 +88,7 @@ struct ZBUFF_DCtx_s { ZBUFF_DCtx* ZBUFF_createDCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZBUFF_createDCtx_advanced(customMem); + return ZBUFF_createDCtx_advanced(defaultCustomNULL); } ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 461741d9a..d4426bcca 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -172,8 +172,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) ZSTD_DCtx* ZSTD_createDCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZSTD_createDCtx_advanced(customMem); + return ZSTD_createDCtx_advanced(defaultCustomNULL); } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 387119815..37a941791 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -521,7 +521,6 @@ int main(int argc, const char** argv) U32 mainPause = 0; const char* programName = argv[0]; ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL }; - ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Fri, 3 Jun 2016 13:23:04 +0200 Subject: [PATCH 117/208] removed calloc calls from lib/ --- lib/compress/huf_compress.c | 1 - lib/compress/zbuff_compress.c | 8 +------- lib/compress/zstd_compress.c | 8 +------- lib/decompress/huf_decompress.c | 1 - lib/decompress/zbuff_decompress.c | 9 +-------- lib/decompress/zstd_decompress.c | 10 ++-------- programs/zbufftest.c | 3 ++- 7 files changed, 7 insertions(+), 33 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index a1004fba3..05cf69657 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -59,7 +59,6 @@ /* ************************************************************** * Includes ****************************************************************/ -#include /* malloc, free, qsort */ #include /* memcpy, memset */ #include /* printf (debug) */ #include "huf_static.h" diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 0ec27ad70..1681bc84a 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -108,13 +108,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) ZBUFF_CCtx* zbc; if (!customMem.customAlloc && !customMem.customFree) - { - zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx)); - if (zbc==NULL) return NULL; - memcpy(&zbc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - zbc->zc = ZSTD_createCCtx(); - return zbc; - } + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 93caee0d7..3196916f9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,7 +51,6 @@ /*-************************************* * Dependencies ***************************************/ -#include /* malloc */ #include /* memset */ #include "mem.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ @@ -130,12 +129,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ctx; if (!customMem.customAlloc && !customMem.customFree) - { - ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); - if (!ctx) return NULL; - memcpy(&ctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - return ctx; - } + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 01e9c07b2..14afbb927 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -59,7 +59,6 @@ /* ************************************************************** * Includes ****************************************************************/ -#include /* malloc, free, qsort */ #include /* memcpy, memset */ #include /* printf (debug) */ #include "huf_static.h" diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 4dfb2adae..01fcc217f 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -96,14 +96,7 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) ZBUFF_DCtx* zbd; if (!customMem.customAlloc && !customMem.customFree) - { - zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx)); - if (zbd==NULL) return NULL; - memcpy(&zbd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - zbd->zd = ZSTD_createDCtx(); - zbd->stage = ZBUFFds_init; - return zbd; - } + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d4426bcca..495e8d296 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -53,7 +53,6 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include /* calloc */ #include /* memcpy, memmove */ #include /* debug only : printf */ #include "mem.h" /* low level memory routines */ @@ -152,13 +151,8 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; - if (!customMem.customAlloc && !customMem.customFree) { - dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); - if (!dctx) return NULL; - memcpy(&dctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - ZSTD_decompressBegin(dctx); - return dctx; - } + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 37a941791..387119815 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -521,6 +521,7 @@ int main(int argc, const char** argv) U32 mainPause = 0; const char* programName = argv[0]; ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL }; + ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Fri, 3 Jun 2016 13:28:20 +0200 Subject: [PATCH 118/208] defaultCustomNULL replaced with defaultCustomMem --- lib/common/zstd_internal.h | 3 +-- lib/compress/zbuff_compress.c | 2 +- lib/compress/zstd_compress.c | 2 +- lib/decompress/zbuff_decompress.c | 2 +- lib/decompress/zstd_decompress.c | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index b56030ba5..8432c1dd1 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -259,6 +259,5 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; -static ZSTD_customMem const defaultCustomNULL = { NULL, NULL, NULL }; - + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 1681bc84a..0340eeb35 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -100,7 +100,7 @@ struct ZBUFF_CCtx_s { ZBUFF_CCtx* ZBUFF_createCCtx(void) { - return ZBUFF_createCCtx_advanced(defaultCustomNULL); + return ZBUFF_createCCtx_advanced(defaultCustomMem); } ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3196916f9..68a6072e8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -121,7 +121,7 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - return ZSTD_createCCtx_advanced(defaultCustomNULL); + return ZSTD_createCCtx_advanced(defaultCustomMem); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 01fcc217f..e9e84ef7c 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -88,7 +88,7 @@ struct ZBUFF_DCtx_s { ZBUFF_DCtx* ZBUFF_createDCtx(void) { - return ZBUFF_createDCtx_advanced(defaultCustomNULL); + return ZBUFF_createDCtx_advanced(defaultCustomMem); } ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 495e8d296..128d941eb 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -166,7 +166,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) ZSTD_DCtx* ZSTD_createDCtx(void) { - return ZSTD_createDCtx_advanced(defaultCustomNULL); + return ZSTD_createDCtx_advanced(defaultCustomMem); } From 2a74609b90ceae29f5bfa8e8c651ee97c4ce3607 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 14:53:51 +0200 Subject: [PATCH 119/208] zlibWrapper: ZWRAP_createCCtx and ZWRAP_freeCCtx use custom memory allocation functions --- lib/common/zstd_internal.h | 2 +- zlibWrapper/zstd_zlibwrapper.c | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8432c1dd1..230ce79fc 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -259,5 +259,5 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; - + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index a4e22a118..937895cd2 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -38,6 +38,7 @@ #include "zstd_static.h" /* ZSTD_MAGICNUMBER */ #include "zbuff.h" #include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ +#include "zstd_internal.h" /* defaultCustomMem */ #define Z_INFLATE_SYNC 8 @@ -47,8 +48,6 @@ #define LOG_WRAPPER(...) // printf(__VA_ARGS__) -#define MIN(a,b) ((a)<(b)?(a):(b)) - #define FINISH_WITH_ERR(msg) { \ fprintf(stderr, "ERROR: %s\n", msg); \ return Z_MEM_ERROR; \ @@ -99,22 +98,31 @@ typedef struct { ZBUFF_CCtx* zbc; size_t bytesLeft; int compressionLevel; + ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_CCtx; ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) { - ZWRAP_CCtx* zwc = (ZWRAP_CCtx*)malloc(sizeof(ZWRAP_CCtx)); - if (zwc==NULL) return NULL; - memset(zwc, 0, sizeof(*zwc)); + ZWRAP_CCtx* zwc; + if (strm->zalloc && strm->zfree) { - ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; - memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); - zwc->zbc = ZBUFF_createCCtx_advanced(ZWRAP_customMem); + zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx)); + if (zwc==NULL) return NULL; + memset(zwc, 0, sizeof(ZWRAP_CCtx)); + memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; + memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem)); + } + } else { + zwc = (ZWRAP_CCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_CCtx)); + if (zwc==NULL) return NULL; + memset(zwc, 0, sizeof(ZWRAP_CCtx)); + memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - else - zwc->zbc = ZBUFF_createCCtx(); + + zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem); return zwc; } @@ -123,7 +131,7 @@ size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) { if (zwc==NULL) return 0; /* support free on NULL */ ZBUFF_freeCCtx(zwc->zbc); - free(zwc); + zwc->customMem.customFree(zwc->customMem.opaque, zwc); return 0; } From 1dd796155714eeddc635b5acd382dcf52cd40ba3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Jun 2016 15:14:09 +0200 Subject: [PATCH 120/208] minor code refactoring --- programs/zstdcli.c | 59 ++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index df7a33208..c0f3d4ba0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -176,6 +176,18 @@ static void waitEnter(void) (void)unused; } +/*! readU32FromChar() : + @return : unsigned integer value reach from input in `char` format + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : this function can overflow if result > MAX_UNIT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + return result; +} + #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -264,12 +276,7 @@ int main(int argCount, const char** argv) #ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { - cLevel = 0; - while ((*argument >= '0') && (*argument <= '9')) { - cLevel *= 10; - cLevel += *argument - '0'; - argument++; - } + cLevel = readU32FromChar(&argument); dictCLevel = cLevel; if (dictCLevel > ZSTD_maxCLevel()) CLEAN_RETURN(badusage(programName)); @@ -325,19 +332,13 @@ int main(int argCount, const char** argv) case 'e': /* compression Level */ argument++; - if ((*argument>='0') && (*argument<='9')) { - cLevelLast = 0; - while ((*argument >= '0') && (*argument <= '9')) - cLevelLast *= 10, cLevelLast += *argument++ - '0'; - } + cLevelLast = readU32FromChar(&argument); break; /* Modify Nb Iterations (benchmark only) */ case 'i': - { U32 iters= 0; - argument++; - while ((*argument >='0') && (*argument <='9')) - iters *= 10, iters += *argument++ - '0'; + argument++; + { U32 const iters = readU32FromChar(&argument); BMK_setNotificationLevel(displayLevel); BMK_SetNbIterations(iters); } @@ -345,10 +346,8 @@ int main(int argCount, const char** argv) /* cut input into blocks (benchmark only) */ case 'B': - { size_t bSize = 0; - argument++; - while ((*argument >='0') && (*argument <='9')) - bSize *= 10, bSize += *argument++ - '0'; + argument++; + { size_t bSize = readU32FromChar(&argument); if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ if (toupper(*argument)=='M') bSize<<=20, argument++; if (toupper(*argument)=='B') argument++; @@ -358,21 +357,17 @@ int main(int argCount, const char** argv) break; #endif /* ZSTD_NOBENCH */ - /* Selection level */ - case 's': argument++; - dictSelect = 0; - while ((*argument >= '0') && (*argument <= '9')) - dictSelect *= 10, dictSelect += *argument++ - '0'; + /* Dictionary Selection level */ + case 's': + argument++; + dictSelect = readU32FromChar(&argument); break; /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ case 'p': argument++; #ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { - int additionalParam = 0; - while ((*argument >= '0') && (*argument <= '9')) - additionalParam *= 10, additionalParam += *argument++ - '0'; - BMK_setAdditionalParam(additionalParam); + BMK_setAdditionalParam(readU32FromChar(&argument)); } else #endif main_pause=1; @@ -399,18 +394,14 @@ int main(int argCount, const char** argv) if (nextArgumentIsMaxDict) { nextArgumentIsMaxDict = 0; - maxDictSize = 0; - while ((*argument>='0') && (*argument<='9')) - maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++; + maxDictSize = readU32FromChar(&argument); if (toupper(*argument)=='K') maxDictSize <<= 10; continue; } if (nextArgumentIsDictID) { nextArgumentIsDictID = 0; - dictID = 0; - while ((*argument>='0') && (*argument<='9')) - dictID = dictID * 10 + (*argument - '0'), argument++; + dictID = readU32FromChar(&argument); continue; } From 441f5d61fd6599b11a0c5e384fc7dbbb911e0db4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Jun 2016 15:15:47 +0200 Subject: [PATCH 121/208] dictionary size objective can be described in Megabytes --- programs/zstdcli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c0f3d4ba0..657300998 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -396,6 +396,7 @@ int main(int argCount, const char** argv) nextArgumentIsMaxDict = 0; maxDictSize = readU32FromChar(&argument); if (toupper(*argument)=='K') maxDictSize <<= 10; + if (toupper(*argument)=='M') maxDictSize <<= 20; continue; } From fe48775868cfdababaa64fea642a93823e65f727 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Jun 2016 15:41:51 +0200 Subject: [PATCH 122/208] minor decoder code refactoring --- lib/decompress/zstd_decompress.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6d79e678a..b3cdf9883 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -652,9 +652,10 @@ typedef struct { -static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) +static seq_t ZSTD_decodeSequence(seqState_t* seqState) { - /* Literal length */ + seq_t seq; + U32 const llCode = FSE_peekSymbol(&(seqState->stateLL)); U32 const mlCode = FSE_peekSymbol(&(seqState->stateML)); U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ @@ -710,13 +711,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; } - seq->offset = offset; + seq.offset = offset; } - seq->matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ + seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); - seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ if (MEM_32bits() || (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); @@ -725,6 +726,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ + + return seq; } @@ -830,11 +833,7 @@ static size_t ZSTD_decompressSequences( /* Regen sequences */ if (nbSeq) { - seq_t sequence; seqState_t seqState; - - memset(&sequence, 0, sizeof(sequence)); - sequence.offset = REPCODE_STARTVALUE; { U32 i; for (i=0; i Date: Fri, 3 Jun 2016 16:31:57 +0200 Subject: [PATCH 123/208] zlibWrapper: ZWRAP_createDCtx and ZWRAP_freeDCtx use custom memory allocation functions --- zlibWrapper/zstd_zlibwrapper.c | 61 +++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 937895cd2..f3f6ea394 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -30,7 +30,6 @@ */ #include /* fprintf */ -#include /* malloc */ #include /* va_list */ #include #include "zstd_zlibwrapper.h" @@ -103,6 +102,15 @@ typedef struct { } ZWRAP_CCtx; +size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) +{ + if (zwc==NULL) return 0; /* support free on NULL */ + ZBUFF_freeCCtx(zwc->zbc); + zwc->customMem.customFree(zwc->customMem.opaque, zwc); + return 0; +} + + ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) { ZWRAP_CCtx* zwc; @@ -123,19 +131,11 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) } zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem); + if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; } return zwc; } -size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) -{ - if (zwc==NULL) return 0; /* support free on NULL */ - ZBUFF_freeCCtx(zwc->zbc); - zwc->customMem.customFree(zwc->customMem.opaque, zwc); - return 0; -} - - ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)) { @@ -297,16 +297,30 @@ typedef struct { int stream_size; char *version; int windowBits; + ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_DCtx; ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { - ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)malloc(sizeof(ZWRAP_DCtx)); - if (zwd==NULL) return NULL; - memset(zwd, 0, sizeof(*zwd)); - memcpy(&zwd->allocFunc, strm, sizeof(z_stream)); + ZWRAP_DCtx* zwd; + + if (strm->zalloc && strm->zfree) { + zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx)); + if (zwd==NULL) return NULL; + memset(zwd, 0, sizeof(ZWRAP_DCtx)); + memcpy(&zwd->allocFunc, strm, sizeof(z_stream)); + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; + memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem)); + } + } else { + zwd = (ZWRAP_DCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_DCtx)); + if (zwd==NULL) return NULL; + memset(zwd, 0, sizeof(ZWRAP_DCtx)); + memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); + } + return zwd; } @@ -314,9 +328,9 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) { if (zwd==NULL) return 0; /* support free on null */ - if (zwd->version) free(zwd->version); - if (zwd->zbd) ZBUFF_freeDCtx(zwd->zbd); - free(zwd); + ZBUFF_freeDCtx(zwd->zbd); + if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version); + zwd->customMem.customFree(zwd->customMem.opaque, zwd); return 0; } @@ -328,9 +342,11 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, LOG_WRAPPER("- inflateInit\n"); if (zwd == NULL) return Z_MEM_ERROR; - zwd->stream_size = stream_size; - zwd->version = strdup(version); + zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1); + if (zwd->version == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; } + strcpy(zwd->version, version); + zwd->stream_size = stream_size; strm->state = (struct internal_state*) zwd; /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; @@ -436,11 +452,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); } - if (zwd->allocFunc.zalloc && zwd->allocFunc.zfree) { - ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; - zwd->zbd = ZBUFF_createDCtx_advanced(ZWRAP_customMem); - } else - zwd->zbd = ZBUFF_createDCtx(); + zwd->zbd = ZBUFF_createDCtx_advanced(zwd->customMem); + if (zwd->zbd == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; } errorCode = ZBUFF_decompressInit(zwd->zbd); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; From 3640396b1a4fac14d62658037165d8aacf494c56 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 16:36:50 +0200 Subject: [PATCH 124/208] fixed: deallocation of structures in case of error in ZBUFF_createCCtx and ZBUFF_createDCtx --- lib/compress/zbuff_compress.c | 5 +++-- lib/compress/zstd_compress.c | 3 ++- lib/decompress/zbuff_decompress.c | 5 +++-- lib/decompress/zstd_decompress.c | 1 + zlibWrapper/Makefile | 8 ++------ 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 0340eeb35..fe8c3801e 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -118,6 +118,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) memset(zbc, 0, sizeof(ZBUFF_CCtx)); memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem)); zbc->zc = ZSTD_createCCtx_advanced(customMem); + if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; } return zbc; } @@ -125,8 +126,8 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) { if (zbc==NULL) return 0; /* support free on NULL */ ZSTD_freeCCtx(zbc->zc); - zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); - zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); + if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); + if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); zbc->customMem.customFree(zbc->customMem.opaque, zbc); return 0; } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 68a6072e8..dbc9dd206 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -143,7 +143,8 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace); + if (cctx==NULL) return 0; /* support free on NULL */ + if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace); cctx->customMem.customFree(cctx->customMem.opaque, cctx); return 0; /* reserved as a potential error code in the future */ } diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index e9e84ef7c..01937bb06 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -106,6 +106,7 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) memset(zbd, 0, sizeof(ZBUFF_DCtx)); memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem)); zbd->zd = ZSTD_createDCtx_advanced(customMem); + if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; } zbd->stage = ZBUFFds_init; return zbd; } @@ -114,8 +115,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) { if (zbd==NULL) return 0; /* support free on null */ ZSTD_freeDCtx(zbd->zd); - zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->customMem.customFree(zbd->customMem.opaque, zbd); return 0; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 128d941eb..cb2faab79 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -172,6 +172,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void) size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { + if (dctx==NULL) return 0; /* support free on NULL */ dctx->customMem.customFree(dctx->customMem.opaque, dctx); return 0; /* reserved as a potential error code in the future */ } diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 245c57d68..21d56c5e0 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -57,9 +57,5 @@ $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwra $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c clean: - -$(RM) $(ZLIBWRAPPER_PATH)/*.o - -$(RM) $(EXAMPLE_PATH)/*.o - -$(RM) *.o - -$(RM) *.exe - -$(RM) foo.gz - + -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd + @echo Cleaning completed From a5ad5279a5fb7219c9d38d47e037054271a92d37 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Jun 2016 15:41:51 +0200 Subject: [PATCH 125/208] minor decoder code refactoring --- lib/decompress/zstd_decompress.c | 29 ++++++++++++++--------------- versionsTest/test-zstd-versions.py | 1 + 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6d79e678a..8a51f821c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -652,9 +652,10 @@ typedef struct { -static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) +static seq_t ZSTD_decodeSequence(seqState_t* seqState) { - /* Literal length */ + seq_t seq; + U32 const llCode = FSE_peekSymbol(&(seqState->stateLL)); U32 const mlCode = FSE_peekSymbol(&(seqState->stateML)); U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ @@ -710,13 +711,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; } - seq->offset = offset; + seq.offset = offset; } - seq->matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ + seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); - seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ if (MEM_32bits() || (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); @@ -725,6 +726,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ + + return seq; } @@ -830,11 +833,7 @@ static size_t ZSTD_decompressSequences( /* Regen sequences */ if (nbSeq) { - seq_t sequence; seqState_t seqState; - - memset(&sequence, 0, sizeof(sequence)); - sequence.offset = REPCODE_STARTVALUE; { U32 i; for (i=0; i Date: Fri, 3 Jun 2016 17:39:31 +0200 Subject: [PATCH 126/208] zlibWrapper: improved error handling --- zlibWrapper/zstd_zlibwrapper.c | 84 ++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index f3f6ea394..42ac5c04e 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -29,7 +29,6 @@ - zstd source repository : https://github.com/Cyan4973/zstd */ -#include /* fprintf */ #include /* va_list */ #include #include "zstd_zlibwrapper.h" @@ -47,13 +46,18 @@ #define LOG_WRAPPER(...) // printf(__VA_ARGS__) -#define FINISH_WITH_ERR(msg) { \ - fprintf(stderr, "ERROR: %s\n", msg); \ +#define FINISH_WITH_GZ_ERR(msg) { \ + (void)msg; \ + return Z_MEM_ERROR; \ +} + +#define FINISH_WITH_ERR(strm, message) { \ + strm->msg = message; \ return Z_MEM_ERROR; \ } #define FINISH_WITH_NULL_ERR(msg) { \ - fprintf(stderr, "ERROR: %s\n", msg); \ + (void)msg; \ return NULL; \ } @@ -220,7 +224,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) strm->avail_in -= srcSize; } - if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR("Z_FULL_FLUSH is not supported!"); + if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!"); if (flush == Z_FINISH || flush == Z_FULL_FLUSH) { size_t bytesLeft; @@ -519,7 +523,7 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, { if (!g_useZSTD) return deflateCopy(dest, source); - FINISH_WITH_ERR("deflateCopy is not supported!"); + FINISH_WITH_ERR(source, "deflateCopy is not supported!"); } @@ -527,7 +531,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) { if (!g_useZSTD) return deflateReset(strm); - FINISH_WITH_ERR("deflateReset is not supported!"); + FINISH_WITH_ERR(strm, "deflateReset is not supported!"); } @@ -539,7 +543,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, { if (!g_useZSTD) return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); - FINISH_WITH_ERR("deflateTune is not supported!"); + FINISH_WITH_ERR(strm, "deflateTune is not supported!"); } @@ -550,7 +554,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, { if (!g_useZSTD) return deflatePending(strm, pending, bits); - FINISH_WITH_ERR("deflatePending is not supported!"); + FINISH_WITH_ERR(strm, "deflatePending is not supported!"); } #endif @@ -561,7 +565,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, { if (!g_useZSTD) return deflatePrime(strm, bits, value); - FINISH_WITH_ERR("deflatePrime is not supported!"); + FINISH_WITH_ERR(strm, "deflatePrime is not supported!"); } @@ -570,7 +574,7 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, { if (!g_useZSTD) return deflateSetHeader(strm, head); - FINISH_WITH_ERR("deflateSetHeader is not supported!"); + FINISH_WITH_ERR(strm, "deflateSetHeader is not supported!"); } @@ -584,7 +588,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, { if (!strm->reserved) return inflateGetDictionary(strm, dictionary, dictLength); - FINISH_WITH_ERR("inflateGetDictionary is not supported!"); + FINISH_WITH_ERR(strm, "inflateGetDictionary is not supported!"); } #endif @@ -594,7 +598,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, { if (!g_useZSTD) return inflateCopy(dest, source); - FINISH_WITH_ERR("inflateCopy is not supported!"); + FINISH_WITH_ERR(source, "inflateCopy is not supported!"); } @@ -602,7 +606,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) { if (!strm->reserved) return inflateReset(strm); - FINISH_WITH_ERR("inflateReset is not supported!"); + FINISH_WITH_ERR(strm, "inflateReset is not supported!"); } @@ -612,7 +616,7 @@ ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, { if (!strm->reserved) return inflateReset2(strm, windowBits); - FINISH_WITH_ERR("inflateReset2 is not supported!"); + FINISH_WITH_ERR(strm, "inflateReset2 is not supported!"); } #endif @@ -622,7 +626,7 @@ ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) { if (!strm->reserved) return inflateMark(strm); - FINISH_WITH_ERR("inflateMark is not supported!"); + FINISH_WITH_ERR(strm, "inflateMark is not supported!"); } #endif @@ -633,7 +637,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, { if (!strm->reserved) return inflatePrime(strm, bits, value); - FINISH_WITH_ERR("inflatePrime is not supported!"); + FINISH_WITH_ERR(strm, "inflatePrime is not supported!"); } @@ -642,7 +646,7 @@ ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, { if (!strm->reserved) return inflateGetHeader(strm, head); - FINISH_WITH_ERR("inflateGetHeader is not supported!"); + FINISH_WITH_ERR(strm, "inflateGetHeader is not supported!"); } @@ -653,7 +657,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits, { if (!strm->reserved) return inflateBackInit_(strm, windowBits, window, version, stream_size); - FINISH_WITH_ERR("inflateBackInit is not supported!"); + FINISH_WITH_ERR(strm, "inflateBackInit is not supported!"); } @@ -663,7 +667,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm, { if (!strm->reserved) return inflateBack(strm, in, in_desc, out, out_desc); - FINISH_WITH_ERR("inflateBack is not supported!"); + FINISH_WITH_ERR(strm, "inflateBack is not supported!"); } @@ -671,7 +675,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) { if (!strm->reserved) return inflateBackEnd(strm); - FINISH_WITH_ERR("inflateBackEnd is not supported!"); + FINISH_WITH_ERR(strm, "inflateBackEnd is not supported!"); } @@ -762,7 +766,7 @@ ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) { if (!g_useZSTD) return gzbuffer(file, size); - FINISH_WITH_ERR("gzbuffer is not supported!"); + FINISH_WITH_GZ_ERR("gzbuffer is not supported!"); } @@ -770,7 +774,7 @@ ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) { if (!g_useZSTD) return gzoffset(file); - FINISH_WITH_ERR("gzoffset is not supported!"); + FINISH_WITH_GZ_ERR("gzoffset is not supported!"); } @@ -778,7 +782,7 @@ ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) { if (!g_useZSTD) return gzclose_r(file); - FINISH_WITH_ERR("gzclose_r is not supported!"); + FINISH_WITH_GZ_ERR("gzclose_r is not supported!"); } @@ -786,7 +790,7 @@ ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) { if (!g_useZSTD) return gzclose_w(file); - FINISH_WITH_ERR("gzclose_w is not supported!"); + FINISH_WITH_GZ_ERR("gzclose_w is not supported!"); } #endif @@ -795,7 +799,7 @@ ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) { if (!g_useZSTD) return gzsetparams(file, level, strategy); - FINISH_WITH_ERR("gzsetparams is not supported!"); + FINISH_WITH_GZ_ERR("gzsetparams is not supported!"); } @@ -803,7 +807,7 @@ ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len)) { if (!g_useZSTD) return gzread(file, buf, len); - FINISH_WITH_ERR("gzread is not supported!"); + FINISH_WITH_GZ_ERR("gzread is not supported!"); } @@ -812,7 +816,7 @@ ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file, { if (!g_useZSTD) return gzwrite(file, buf, len); - FINISH_WITH_ERR("gzwrite is not supported!"); + FINISH_WITH_GZ_ERR("gzwrite is not supported!"); } @@ -834,7 +838,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) // printf("gzprintf ret=%d\n", ret); return ret; } - FINISH_WITH_ERR("gzprintf is not supported!"); + FINISH_WITH_GZ_ERR("gzprintf is not supported!"); } @@ -842,7 +846,7 @@ ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s)) { if (!g_useZSTD) return gzputs(file, s); - FINISH_WITH_ERR("gzputs is not supported!"); + FINISH_WITH_GZ_ERR("gzputs is not supported!"); } @@ -858,7 +862,7 @@ ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c)) { if (!g_useZSTD) return gzputc(file, c); - FINISH_WITH_ERR("gzputc is not supported!"); + FINISH_WITH_GZ_ERR("gzputc is not supported!"); } @@ -870,7 +874,7 @@ ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) { if (!g_useZSTD) return gzgetc(file); - FINISH_WITH_ERR("gzgetc is not supported!"); + FINISH_WITH_GZ_ERR("gzgetc is not supported!"); } @@ -878,7 +882,7 @@ ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file)) { if (!g_useZSTD) return gzungetc(c, file); - FINISH_WITH_ERR("gzungetc is not supported!"); + FINISH_WITH_GZ_ERR("gzungetc is not supported!"); } @@ -886,7 +890,7 @@ ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush)) { if (!g_useZSTD) return gzflush(file, flush); - FINISH_WITH_ERR("gzflush is not supported!"); + FINISH_WITH_GZ_ERR("gzflush is not supported!"); } @@ -894,7 +898,7 @@ ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence)) { if (!g_useZSTD) return gzseek(file, offset, whence); - FINISH_WITH_ERR("gzseek is not supported!"); + FINISH_WITH_GZ_ERR("gzseek is not supported!"); } @@ -902,7 +906,7 @@ ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file)) { if (!g_useZSTD) return gzrewind(file); - FINISH_WITH_ERR("gzrewind is not supported!"); + FINISH_WITH_GZ_ERR("gzrewind is not supported!"); } @@ -910,7 +914,7 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) { if (!g_useZSTD) return gztell(file); - FINISH_WITH_ERR("gztell is not supported!"); + FINISH_WITH_GZ_ERR("gztell is not supported!"); } @@ -918,7 +922,7 @@ ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) { if (!g_useZSTD) return gzeof(file); - FINISH_WITH_ERR("gzeof is not supported!"); + FINISH_WITH_GZ_ERR("gzeof is not supported!"); } @@ -926,7 +930,7 @@ ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file)) { if (!g_useZSTD) return gzdirect(file); - FINISH_WITH_ERR("gzdirect is not supported!"); + FINISH_WITH_GZ_ERR("gzdirect is not supported!"); } @@ -934,7 +938,7 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) { if (!g_useZSTD) return gzclose(file); - FINISH_WITH_ERR("gzclose is not supported!"); + FINISH_WITH_GZ_ERR("gzclose is not supported!"); } From de2c92f6e09ebb4068ee74748c7ff5866ece89a6 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 19:44:03 +0200 Subject: [PATCH 127/208] zlibWrapper: compress uses ZWRAP_DEFAULT_CLEVEL --- zlibWrapper/zstd_zlibwrapper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 42ac5c04e..6bd820452 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -29,11 +29,11 @@ - zstd source repository : https://github.com/Cyan4973/zstd */ -#include /* va_list */ +#include /* va_list */ #include #include "zstd_zlibwrapper.h" #include "zstd.h" -#include "zstd_static.h" /* ZSTD_MAGICNUMBER */ +#include "zstd_static.h" /* ZSTD_MAGICNUMBER */ #include "zbuff.h" #include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ #include "zstd_internal.h" /* defaultCustomMem */ @@ -693,7 +693,7 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, return compress(dest, destLen, source, sourceLen); { size_t dstCapacity = *destLen; - size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1); + size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL); LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; From f4f5affdf7a4516d305d4f8ea0cfff93cdc68a79 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Jun 2016 23:09:28 +0200 Subject: [PATCH 128/208] restore ZBUFF full-block-size, for better performance on small input --- lib/compress/zbuff_compress.c | 2 +- lib/decompress/zstd_decompress.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index fe8c3801e..bae668154 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -147,7 +147,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } - zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); + zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize); } if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index aa0dba224..be86c42b9 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -832,9 +832,8 @@ static size_t ZSTD_decompressSequences( FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { - seq_t sequence; nbSeq--; - sequence = ZSTD_decodeSequence(&seqState); + { seq_t const sequence = ZSTD_decodeSequence(&seqState); #if 0 /* debug */ static BYTE* start = NULL; @@ -845,7 +844,7 @@ static size_t ZSTD_decompressSequences( pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset); #endif - { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } } From 0d31160b4e0bd1e396c0c3d37ea62c0fb28a203d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 00:09:02 +0200 Subject: [PATCH 129/208] better dynamic memory adaptation when using CLI --- NEWS | 2 ++ programs/fileio.c | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index beb6ae3db..e54b09967 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,12 @@ v0.7.0 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski New : Support for Sparse File-systems (do not use space for zero-filled sectors) +New : Frame checksum support New : Support pass-through mode (when using `-df`) New : API : dictionary files from custom content, by Giuseppe Ottaviano New : API support for custom malloc/free functions New : controllable Dictionary ID +New : Support for skippable frames v0.6.1 New : zlib wrapper API, thanks to Przemyslaw Skibinski diff --git a/programs/fileio.c b/programs/fileio.c index 48555424c..4211e4a0f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -317,7 +317,10 @@ static int FIO_compressFilename_internal(cRess_t ress, params.fParams.contentSizeFlag = 1; params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; - if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; + if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) { + params.cParams.windowLog = g_maxWLog; + params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize); + } { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); } } From 198d127b3503045b0ebd76f53a1b5db43333b2ab Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 18:40:55 +0200 Subject: [PATCH 130/208] minor comment change (unfinished description of new header format) --- lib/decompress/zstd_decompress.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index be86c42b9..9fe1b6506 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -205,6 +205,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) /* Frame descriptor + // old 1 byte - Alloc : bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) bit 4 : reserved for windowLog (must be zero) @@ -215,6 +216,22 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) bit 0-1 : dictID (0, 1, 2 or 4 bytes) bit 2-7 : reserved (must be zero) + + // new + 1 byte - Alloc : + bit 0-1 : dictID (0, 1, 2 or 4 bytes) + bit 2-4 : reserved (must be zero) + bit 5 : WindowLog skipped (note : if 1, then fcs > 0) + bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes + + OR : 0-3 : no windowLog, 1, 2, 4, 8 + 4-7 : windowLog, 0, 2, 4, 8 + + 1 byte - WindowLog (can be skipped if fcs>0) + bit 0-2 : octal Fractional (1/8th) + bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB) + + Optional : dictID (0, 1, 2 or 4 bytes) Automatic adaptation 0 : no dictID From e69b8ccceb059af0653f9da636c8ae6ea027a154 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 18:56:23 +0200 Subject: [PATCH 131/208] merged `zdict_static.h` into `zdict.h`. Now requires `ZDICT_STATIC_LINKING_ONLY` macro. --- lib/dictBuilder/zdict.c | 3 +- lib/dictBuilder/zdict.h | 38 ++++++++++++++++ lib/dictBuilder/zdict_static.h | 81 ---------------------------------- programs/dibio.h | 3 +- 4 files changed, 42 insertions(+), 83 deletions(-) delete mode 100644 lib/dictBuilder/zdict_static.h diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 09249e9ea..ef2b1ae5b 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -63,7 +63,8 @@ #include "zstd_internal.h" #include "xxhash.h" #include "divsufsort.h" -#include "zdict_static.h" +#define ZDICT_STATIC_LINKING_ONLY +#include "zdict.h" /*-************************************* diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 3a724d0be..534d29de8 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -75,6 +75,44 @@ unsigned ZDICT_isError(size_t errorCode); const char* ZDICT_getErrorName(size_t errorCode); +#ifdef ZDICT_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used in association with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + + +/*-************************************* +* Public type +***************************************/ +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */ + unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */ + unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ + unsigned reserved[2]; /* space for future parameters */ +} ZDICT_params_t; + + +/*-************************************* +* Public functions +***************************************/ +/*! ZDICT_trainFromBuffer_advanced() : + Same as ZDICT_trainFromBuffer() with control over more parameters. + `parameters` is optional and can be provided with values set to 0 to mean "default". + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`) + or an error code, which can be tested by ZDICT_isError(). + note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel() +*/ +size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + +#endif /* ZDICT_STATIC_LINKING_ONLY */ + #if defined (__cplusplus) } #endif diff --git a/lib/dictBuilder/zdict_static.h b/lib/dictBuilder/zdict_static.h deleted file mode 100644 index e34e6c07d..000000000 --- a/lib/dictBuilder/zdict_static.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - dictBuilder header file - for static linking only - Copyright (C) Yann Collet 2016 - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Zstd source repository : https://www.zstd.net -*/ - -/* This library is EXPERIMENTAL, below API is not yet stable */ - -#ifndef DICTBUILDER_STATIC_H_002 -#define DICTBUILDER_STATIC_H_002 - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-************************************* -* Dependencies -***************************************/ -#include "zdict.h" - - -/*-************************************* -* Public type -***************************************/ -typedef struct { - unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */ - unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */ - unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ - unsigned reserved[2]; /* space for future parameters */ -} ZDICT_params_t; - - -/*-************************************* -* Public functions -***************************************/ -/*! ZDICT_trainFromBuffer_advanced() : - Same as ZDICT_trainFromBuffer() with control over more parameters. - `parameters` is optional and can be provided with values set to 0 to mean "default". - @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`) - or an error code, which can be tested by ZDICT_isError(). - note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel() -*/ -size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, - ZDICT_params_t parameters); - - -#if defined (__cplusplus) -} -#endif - -#endif /* DICTBUILDER_STATIC_H_002 */ diff --git a/programs/dibio.h b/programs/dibio.h index 0ccec4135..8ad292027 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,8 @@ /*-************************************* * Dependencies ***************************************/ -#include "zdict_static.h" /* ZDICT_params_t */ +#define ZDICT_STATIC_LINKING_ONLY +#include "zdict.h" /* ZDICT_params_t */ /*-************************************* From 5347aee8f7f5260b500e3f096dbc06a35f3cfb10 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 19:12:48 +0200 Subject: [PATCH 132/208] merged `zbuff_static.h` into `zbuff.h` . Now requires `ZBUFF_STATIC_LINKING_ONLY` macro --- lib/common/zbuff.h | 46 ++++++++++++++-- lib/common/zbuff_static.h | 87 ------------------------------- lib/compress/zbuff_compress.c | 3 +- lib/decompress/zbuff_decompress.c | 5 +- programs/fileio.c | 3 +- programs/zbufftest.c | 6 +-- 6 files changed, 53 insertions(+), 97 deletions(-) delete mode 100644 lib/common/zbuff_static.h diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 89207bd24..d761a273f 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -28,8 +28,8 @@ You can contact the author at : - zstd homepage : http://www.zstd.net/ */ -#ifndef ZSTD_BUFFERED_H -#define ZSTD_BUFFERED_H +#ifndef ZSTD_BUFFERED_H_23987 +#define ZSTD_BUFFERED_H_23987 #if defined (__cplusplus) extern "C" { @@ -160,8 +160,48 @@ ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); +#ifdef ZBUFF_STATIC_LINKING_ONLY + +/* ************************************* +* Includes +***************************************/ +#include "zstd_static.h" /* ZSTD_parameters */ + +/* internal util function */ +#define ZBUFF_MIN(a,b) ((a)<(b) ? (a) : (b)) +MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t length = ZBUFF_MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +/*-************************************* +* Advanced functions +***************************************/ +/*! ZBUFF_createCCtx_advanced() : + * Create a ZBUFF compression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZBUFF_createDCtx_advanced() : + * Create a ZBUFF decompression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); + + +/* ************************************* +* Advanced Streaming functions +***************************************/ +ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, U64 pledgedSrcSize); + + +#endif + + #if defined (__cplusplus) } #endif -#endif /* ZSTD_BUFFERED_H */ +#endif /* ZSTD_BUFFERED_H_23987 */ diff --git a/lib/common/zbuff_static.h b/lib/common/zbuff_static.h deleted file mode 100644 index 44a0321d1..000000000 --- a/lib/common/zbuff_static.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - zstd - buffered version of compression library - experimental complementary API, for static linking only - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ -#ifndef ZSTD_BUFFERED_STATIC_H -#define ZSTD_BUFFERED_STATIC_H - -/* The objects defined into this file must be considered experimental. - * Their prototype may change in future versions. - * Never use them with a dynamic library. - */ - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include "zstd_static.h" /* ZSTD_parameters */ -#include "zbuff.h" - -#define ZBUFF_MIN(a,b) ((a)<(b) ? (a) : (b)) - - -/*-************************************* -* Advanced functions -***************************************/ -/*! ZBUFF_createCCtx_advanced() : - * Create a ZBUFF compression context using external alloc and free functions */ -ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); - -/*! ZBUFF_createDCtx_advanced() : - * Create a ZBUFF decompression context using external alloc and free functions */ -ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); - - -/* ************************************* -* Advanced Streaming functions -***************************************/ -ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, - const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize); - - -/* internal util function */ - -MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t length = ZBUFF_MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_BUFFERED_STATIC_H */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index bae668154..65390c29b 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -37,7 +37,8 @@ #include "error_private.h" #include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ #include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ -#include "zbuff_static.h" +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" /* ************************************* diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 01937bb06..e3d574355 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -37,7 +37,8 @@ #include "error_private.h" #include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ #include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ -#include "zbuff_static.h" +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" /*-*************************************************************************** @@ -183,7 +184,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, } } if (zbd->fParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) zbd->fParams.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for buffer allocation */ - + /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; diff --git a/programs/fileio.c b/programs/fileio.c index 4211e4a0f..e00a123f8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -58,7 +58,8 @@ #include "fileio.h" #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd_internal.h" /* MIN, KB, MB */ -#include "zbuff_static.h" +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_legacy.h" /* ZSTD_isLegacy */ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 387119815..a50e08541 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -40,12 +40,12 @@ #include /* timeb */ #include /* strcmp */ #include "mem.h" -#include "zbuff.h" #include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ -#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" /* ZBUFF_createCCtx_advanced */ #include "datagen.h" /* RDG_genBuffer */ #define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" /* XXH64 */ +#include "xxhash.h" /* XXH64_* */ /*-************************************ From ac110a1f2131262f041a2ee7234ffa19a45f6f94 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 19:16:49 +0200 Subject: [PATCH 133/208] Removed ZBUFF internal util function from public area --- lib/common/zbuff.h | 11 +---------- lib/compress/zbuff_compress.c | 9 +++++++++ lib/decompress/zbuff_decompress.c | 9 +++++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index d761a273f..1f8cb610f 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -163,19 +163,10 @@ ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); #ifdef ZBUFF_STATIC_LINKING_ONLY /* ************************************* -* Includes +* Dependency ***************************************/ #include "zstd_static.h" /* ZSTD_parameters */ -/* internal util function */ -#define ZBUFF_MIN(a,b) ((a)<(b) ? (a) : (b)) -MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t length = ZBUFF_MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - /*-************************************* * Advanced functions diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 65390c29b..76d3afea1 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -183,6 +183,15 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) } +/* internal util function */ +MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + /* *** Compression *** */ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index e3d574355..080e3dea3 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -138,6 +138,15 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) } +/* internal util function */ +MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + /* *** Decompression *** */ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, From d3b7f8d21f0eb8650f0cb0855e934aff260c6db7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 19:47:02 +0200 Subject: [PATCH 134/208] Merged `zstd_static.h` into `zstd.h` . Now requires `ZSTD_STATIC_LINKING_ONLY` macro --- lib/common/zbuff.h | 22 ++- lib/common/zstd.h | 264 +++++++++++++++++++++++++- lib/common/zstd_common.c | 3 +- lib/common/zstd_internal.h | 3 +- lib/common/zstd_static.h | 303 ------------------------------ lib/compress/zbuff_compress.c | 3 +- lib/compress/zstd_compress.c | 6 +- lib/decompress/zbuff_decompress.c | 3 +- programs/bench.c | 3 +- programs/fileio.c | 3 +- programs/fullbench.c | 3 +- programs/fuzzer.c | 3 +- programs/paramgrill.c | 3 +- programs/zbufftest.c | 3 +- programs/zstdcli.c | 3 +- 15 files changed, 296 insertions(+), 332 deletions(-) delete mode 100644 lib/common/zstd_static.h diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 1f8cb610f..f8e8976f4 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -162,15 +162,19 @@ ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); #ifdef ZBUFF_STATIC_LINKING_ONLY -/* ************************************* -* Dependency -***************************************/ -#include "zstd_static.h" /* ZSTD_parameters */ +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used in association with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*--- Dependency ---*/ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "zstd.h" -/*-************************************* -* Advanced functions -***************************************/ +/*--- External memory ---*/ /*! ZBUFF_createCCtx_advanced() : * Create a ZBUFF compression context using external alloc and free functions */ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); @@ -180,9 +184,7 @@ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); -/* ************************************* -* Advanced Streaming functions -***************************************/ +/*--- Advanced Streaming function ---*/ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 3ac4056d1..a84b59501 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -29,8 +29,8 @@ You can contact the author at : - zstd source repository : https://github.com/Cyan4973/zstd */ -#ifndef ZSTD_H -#define ZSTD_H +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 #if defined (__cplusplus) extern "C" { @@ -148,8 +148,266 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict,size_t dictSize); +#ifdef ZSTD_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used in association with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*--- Dependency ---*/ +#include "mem.h" + + +/*--- Constants ---*/ +#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U + +#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) +#define ZSTD_WINDOWLOG_MIN 18 +#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) +#define ZSTD_CHAINLOG_MIN 4 +#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX +#define ZSTD_HASHLOG_MIN 12 +#define ZSTD_HASHLOG3_MAX 17 +#define ZSTD_HASHLOG3_MIN 15 +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_SEARCHLENGTH_MAX 7 +#define ZSTD_SEARCHLENGTH_MIN 3 +#define ZSTD_TARGETLENGTH_MIN 4 +#define ZSTD_TARGETLENGTH_MAX 999 + + +/*--- Types ---*/ +typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ + +typedef struct { + U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */ + U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + U32 hashLog; /* dispatch table : larger == faster, more memory */ + U32 searchLog; /* nb of searches : larger == more compression, slower */ + U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */ + U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; +} ZSTD_compressionParameters; + +typedef struct { + U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ + U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +/* custom memory allocation functions */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; + + +/*-************************************* +* Advanced functions +***************************************/ +/*! ZSTD_createCCtx_advanced() : + * Create a ZSTD compression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); + +ZSTDLIB_API unsigned ZSTD_maxCLevel (void); + +/*! ZSTD_getCParams() : +* @return ZSTD_compressionParameters structure for a selected compression level and srcSize. +* `srcSize` value is optional, select 0 if not known */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize); + +/*! ZSTD_checkParams() : +* Ensure param values remain within authorized range */ +ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustParams() : +* optimize params for a given `srcSize` and `dictSize`. +* both values are optional, select `0` if unknown. */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize); + +/*! ZSTD_compress_advanced() : +* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ +ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingPreparedDCtx() : +* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). +* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ +ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( + ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*- Advanced Decompression functions -*/ + +/*! ZSTD_decompress_usingPreparedDCtx() : +* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ +ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( + ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/* ************************************** +* Streaming functions (direct mode) +****************************************/ +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); + +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity); + +/* + Streaming compression, synchronous mode (bufferless) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, + or ZSTD_compressBegin_advanced(), for finer parameter control. + It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + + Then, consume your input using ZSTD_compressContinue(). + The interface is synchronous, so all input will be consumed and produce a compressed output. + You must ensure there is enough space in destination buffer to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + + Finish a frame with ZSTD_compressEnd(), which will write the epilogue. + Without the epilogue, frames will be considered incomplete by decoder. + + You can then reuse ZSTD_CCtx to compress some new frame. +*/ + +typedef struct { + U64 frameContentSize; + U32 windowLog; + U32 dictID; + U32 checksumFlag; +} ZSTD_frameParams; + +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_min = 6; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ + +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); + +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* + Streaming decompression, direct mode (bufferless) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input. + It can provide the minimum size of rolling buffer required to properly decompress data, + and optionally the final size of uncompressed content. + (Note : content size is an optional info that may not be present. 0 means : content size unknown) + Frame parameters are extracted from the beginning of compressed frame. + The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work) + If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result. + Result : 0 when successful, it means the ZSTD_frameParams structure has been filled. + >0 : means there is not enough data into `src`. Provides the expected size to successfully decode header. + errorCode, which can be tested using ZSTD_isError() + + Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail. + ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). + They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. + + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity) + It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Skippable frames allow the integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following: + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_decompressContinue() always returns 0. + For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. + It also returns Frame Size as fparamsPtr->frameContentSize. +*/ + + +/* ************************************** +* Block functions +****************************************/ +/*! Block functions produce and decode raw zstd blocks, without frame metadata. + User will have to take in charge required information to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) + - Compressing or decompressing requires a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : ZSTD_compressBegin() + + decompression : ZSTD_decompressBegin() + + variants _usingDict() are also allowed + + copyCCtx() and copyDCtx() work too + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. + In which case, nothing is produced into `dst`. + + User must test for such outcome and deal directly with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! +*/ + +#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ +ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/*-************************************* +* Error management +***************************************/ +#include "error_public.h" +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); + + +#endif /* ZSTD_STATIC_LINKING_ONLY */ + #if defined (__cplusplus) } #endif -#endif /* ZSTD_H */ +#endif /* ZSTD_H_235446 */ diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 5d3db0ea0..0d9c1154e 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -35,7 +35,8 @@ ***************************************/ #include /* malloc */ #include "error_private.h" -#include "zstd_static.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode */ +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 230ce79fc..712016369 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -37,7 +37,8 @@ ***************************************/ #include "mem.h" #include "error_private.h" -#include "zstd_static.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" /*-************************************* diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h deleted file mode 100644 index 6f28bea6a..000000000 --- a/lib/common/zstd_static.h +++ /dev/null @@ -1,303 +0,0 @@ -/* - zstd - standard compression library - Header File for static linking only - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ -#ifndef ZSTD_STATIC_H -#define ZSTD_STATIC_H - -/* The prototypes defined within this file are considered experimental. - * They should not be used in the context DLL as they may change in the future. - * Prefer static linking if you need them, to control breaking version changes issues. - */ - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-************************************* -* Dependencies -***************************************/ -#include "zstd.h" -#include "mem.h" - - -/*-************************************* -* Constants -***************************************/ -#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U - - -/*-************************************* -* Types -***************************************/ -#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) -#define ZSTD_WINDOWLOG_MIN 18 -#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -#define ZSTD_CHAINLOG_MIN 4 -#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX -#define ZSTD_HASHLOG_MIN 12 -#define ZSTD_HASHLOG3_MAX 17 -#define ZSTD_HASHLOG3_MIN 15 -#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_SEARCHLENGTH_MAX 7 -#define ZSTD_SEARCHLENGTH_MIN 3 -#define ZSTD_TARGETLENGTH_MIN 4 -#define ZSTD_TARGETLENGTH_MAX 999 - -/* from faster to stronger */ -typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; - -typedef struct { - U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */ - U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /* dispatch table : larger == faster, more memory */ - U32 searchLog; /* nb of searches : larger == more compression, slower */ - U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */ - U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */ - ZSTD_strategy strategy; -} ZSTD_compressionParameters; - -typedef struct { - U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ - U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ - U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ -} ZSTD_frameParameters; - -typedef struct { - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; -} ZSTD_parameters; - -/* custom memory allocation functions */ -typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); -typedef void (*ZSTD_freeFunction) (void* opaque, void* address); -typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; - - -/*-************************************* -* Advanced functions -***************************************/ -/*! ZSTD_createCCtx_advanced() : - * Create a ZSTD compression context using external alloc and free functions */ -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); - -/*! ZSTD_createDCtx_advanced() : - * Create a ZSTD decompression context using external alloc and free functions */ -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); - -ZSTDLIB_API unsigned ZSTD_maxCLevel (void); - -/*! ZSTD_getCParams() : -* @return ZSTD_compressionParameters structure for a selected compression level and srcSize. -* `srcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize); - -/*! ZSTD_checkParams() : -* Ensure param values remain within authorized range */ -ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); - -/*! ZSTD_adjustParams() : -* optimize params for a given `srcSize` and `dictSize`. -* both values are optional, select `0` if unknown. */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize); - -/*! ZSTD_compress_advanced() : -* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ -ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); - -/*! ZSTD_compress_usingPreparedDCtx() : -* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). -* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ -ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( - ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*- Advanced Decompression functions -*/ - -/*! ZSTD_decompress_usingPreparedDCtx() : -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( - ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - - -/* ************************************** -* Streaming functions (direct mode) -****************************************/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); - -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity); - -/* - Streaming compression, synchronous mode (bufferless) - - A ZSTD_CCtx object is required to track streaming operations. - Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it. - ZSTD_CCtx object can be re-used multiple times within successive compression operations. - - Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() - - Then, consume your input using ZSTD_compressContinue(). - The interface is synchronous, so all input will be consumed and produce a compressed output. - You must ensure there is enough space in destination buffer to store compressed data under worst case scenario. - Worst case evaluation is provided by ZSTD_compressBound(). - - Finish a frame with ZSTD_compressEnd(), which will write the epilogue. - Without the epilogue, frames will be considered incomplete by decoder. - - You can then reuse ZSTD_CCtx to compress some new frame. -*/ - -typedef struct { - U64 frameContentSize; - U32 windowLog; - U32 dictID; - U32 checksumFlag; -} ZSTD_frameParams; - -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ -static const size_t ZSTD_frameHeaderSize_min = 6; -static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; -static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ -ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ - -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); - -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -/* - Streaming decompression, direct mode (bufferless) - - A ZSTD_DCtx object is required to track streaming operations. - Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. - - First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input. - It can provide the minimum size of rolling buffer required to properly decompress data, - and optionally the final size of uncompressed content. - (Note : content size is an optional info that may not be present. 0 means : content size unknown) - Frame parameters are extracted from the beginning of compressed frame. - The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work) - If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result. - Result : 0 when successful, it means the ZSTD_frameParams structure has been filled. - >0 : means there is not enough data into `src`. Provides the expected size to successfully decode header. - errorCode, which can be tested using ZSTD_isError() - - Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). - Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). - - Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail. - ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). - They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. - - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity) - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. - - A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. - Context can then be reset to start a new decompression. - - Skippable frames allow the integration of user-defined data into a flow of concatenated frames. - Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following: - a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F - b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - c) Frame Content - any content (User Data) of length equal to Frame Size - For skippable frames ZSTD_decompressContinue() always returns 0. - For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. - It also returns Frame Size as fparamsPtr->frameContentSize. -*/ - - -/* ************************************** -* Block functions -****************************************/ -/*! Block functions produce and decode raw zstd blocks, without frame metadata. - User will have to take in charge required information to regenerate data, such as compressed and content sizes. - - A few rules to respect : - - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) - - Compressing or decompressing requires a context structure - + Use ZSTD_createCCtx() and ZSTD_createDCtx() - - It is necessary to init context before starting - + compression : ZSTD_compressBegin() - + decompression : ZSTD_decompressBegin() - + variants _usingDict() are also allowed - + copyCCtx() and copyDCtx() work too - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. - In which case, nothing is produced into `dst`. - + User must test for such outcome and deal directly with uncompressed data - + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! -*/ - -#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-************************************* -* Error management -***************************************/ -#include "error_public.h" -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); -ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_STATIC_H */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 76d3afea1..367320e2b 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -35,8 +35,7 @@ ***************************************/ #include #include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ -#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */ #define ZBUFF_STATIC_LINKING_ONLY #include "zbuff.h" diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index dbc9dd206..05b35de11 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,13 +51,13 @@ /*-************************************* * Dependencies ***************************************/ -#include /* memset */ +#include /* memset */ #include "mem.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH_reset, update, digest */ +#include "xxhash.h" /* XXH_reset, update, digest */ #include "fse_static.h" #include "huf_static.h" -#include "zstd_internal.h" +#include "zstd_internal.h" /* includes zstd.h */ /*-************************************* diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 080e3dea3..a6b99c976 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -35,8 +35,7 @@ ***************************************/ #include #include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ -#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */ #define ZBUFF_STATIC_LINKING_ONLY #include "zbuff.h" diff --git a/programs/bench.c b/programs/bench.c index 97f71162b..08a972c75 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -32,7 +32,8 @@ #include /* fprintf, fopen, ftello64 */ #include "mem.h" -#include "zstd_static.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" diff --git a/programs/fileio.c b/programs/fileio.c index e00a123f8..4d7da9130 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -56,7 +56,8 @@ #include "mem.h" #include "fileio.h" -#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ +#include "zstd.h" #include "zstd_internal.h" /* MIN, KB, MB */ #define ZBUFF_STATIC_LINKING_ONLY #include "zbuff.h" diff --git a/programs/fullbench.c b/programs/fullbench.c index 06796c25a..9fe4a1219 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -32,7 +32,8 @@ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include "mem.h" -#include "zstd_static.h" /* ZSTD_VERSION_STRING */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue */ +#include "zstd.h" /* ZSTD_VERSION_STRING */ #include "fse_static.h" #include "zbuff.h" #include "datagen.h" diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 0c3389282..480fd3072 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -40,7 +40,8 @@ #include /* timeb */ #include /* strcmp */ #include /* clock_t */ -#include "zstd_static.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue */ +#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" diff --git a/programs/paramgrill.c b/programs/paramgrill.c index cb9a2ea49..6cf4ccd89 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -48,7 +48,8 @@ #endif #include "mem.h" -#include "zstd_static.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "zstd.h" #include "datagen.h" #include "xxhash.h" diff --git a/programs/zbufftest.c b/programs/zbufftest.c index a50e08541..78122e710 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -40,7 +40,8 @@ #include /* timeb */ #include /* strcmp */ #include "mem.h" -#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ +#include "zstd.h" /* ZSTD_compressBound */ #define ZBUFF_STATIC_LINKING_ONLY #include "zbuff.h" /* ZBUFF_createCCtx_advanced */ #include "datagen.h" /* RDG_genBuffer */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 657300998..7c235c79e 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -38,10 +38,11 @@ #ifndef ZSTD_NOBENCH # include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ #endif -#include "zstd_static.h" /* ZSTD_maxCLevel, ZSTD version numbers */ #ifndef ZSTD_NODICT # include "dibio.h" #endif +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ +#include "zstd.h" /* ZSTD_VERSION_STRING */ From 16f729957f466988b11bab8ee18dee453e8802db Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 19:52:06 +0200 Subject: [PATCH 135/208] fixed zlib wrapper for new `.h` strategy --- zlibWrapper/.gitignore | 3 +++ zlibWrapper/zstd_zlibwrapper.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore index 56473513b..bf3f3d87d 100644 --- a/zlibWrapper/.gitignore +++ b/zlibWrapper/.gitignore @@ -19,6 +19,9 @@ # Default result files _* +example +example_zstd +foo.gz # Misc files *.bat diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 6bd820452..c2efe380e 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -29,13 +29,13 @@ - zstd source repository : https://github.com/Cyan4973/zstd */ -#include /* va_list */ +#include /* va_list, for z_gzprintf */ #include #include "zstd_zlibwrapper.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */ #include "zstd.h" -#include "zstd_static.h" /* ZSTD_MAGICNUMBER */ +#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ #include "zbuff.h" -#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ #include "zstd_internal.h" /* defaultCustomMem */ From 49bb0041afe1f696e08690884450a8c2fbc10807 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 20:17:38 +0200 Subject: [PATCH 136/208] removed `ZSTD_highbit()` from `zstd_internal.h`, as it is only used by `zstd_compress.c` --- lib/common/zstd.h | 4 +-- lib/common/zstd_internal.h | 26 ++-------------- lib/compress/zstd_compress.c | 57 ++++++++++++++++++++++++------------ lib/compress/zstd_opt.h | 32 ++++++++++---------- lib/dictBuilder/zdict.c | 2 +- 5 files changed, 59 insertions(+), 62 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index a84b59501..2f7ab1e87 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -162,7 +162,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, /*--- Constants ---*/ -#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) @@ -350,7 +350,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity) + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 712016369..2e238faad 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -85,7 +85,7 @@ static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; -#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */ +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; @@ -162,28 +162,6 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length) while (op < oend); } -MEM_STATIC unsigned ZSTD_highbit(U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - _BitScanReverse(&r, val); - return (unsigned)r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); -# else /* Software version */ - static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - int r; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; - return r; -# endif -} - /*-******************************************* * Private interfaces @@ -211,7 +189,7 @@ typedef struct { MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif // #if ZSTD_OPT_DEBUG == 3 +#endif /* #if ZSTD_OPT_DEBUG == 3 */ typedef struct { void* buffer; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 05b35de11..49c3d2663 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -71,6 +71,27 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat ***************************************/ size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } +static U32 ZSTD_highbit32(U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse(&r, val); + return (unsigned)r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} /*-************************************* * Sequence storage @@ -176,8 +197,6 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) } -static unsigned ZSTD_highbit(U32 val); - /** ZSTD_checkCParams_advanced() : temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize) @@ -205,7 +224,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1< srcLog) cPar.windowLog = srcLog; } } if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; @@ -649,7 +668,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) size_t u; for (u=0; u63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll]; + llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit32(ll) + LL_deltaCode : LL_Code[ll]; } if (seqStorePtr->longLengthID==1) llCodeTable[seqStorePtr->longLengthPos] = MaxLL; @@ -659,7 +678,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) { const U32* const offsetTable = seqStorePtr->offsetStart; BYTE* const ofCodeTable = seqStorePtr->offCodeStart; size_t u; - for (u=0; u127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml]; + mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit32(ml) + ML_deltaCode : ML_Code[ml]; } if (seqStorePtr->longLengthID==2) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; @@ -1425,7 +1444,7 @@ static size_t ZSTD_insertBtAndFindBestMatch ( if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ break; /* drop, to guarantee consistency (miss a little bit of compression) */ @@ -1711,14 +1730,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32; int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) matchLength = mlRep, offset = 0, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; /* search a better one */ @@ -1730,14 +1749,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32; int const gain2 = (int)(ml2 * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) matchLength = ml2, offset = 0, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; @@ -1883,7 +1902,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) matchLength = repLength, offset = 0, start = ip; } } @@ -1891,8 +1910,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* search match, depth 1 */ { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; /* search a better one */ @@ -1913,7 +1932,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; int gain2 = (int)(repLength * 4); - int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); + int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) matchLength = repLength, offset = 0, start = ip; } } @@ -1921,8 +1940,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* search match, depth 2 */ { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 86fa13067..46b2cb14a 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -41,10 +41,10 @@ ***************************************/ FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) { - ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1); - ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1); - ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1); - ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1); + ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1); + ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1); + ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1); + ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1); ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); } @@ -106,7 +106,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY U32 price, u; if (litLength == 0) - return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1); + return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1); /* literals */ if (ssPtr->cachedLiterals == literals) { @@ -114,13 +114,13 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; for (u=0; u < additional; u++) - price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1); + price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1); ssPtr->cachedPrice = price; ssPtr->cachedLitLength = litLength; } else { price = litLength * ssPtr->log2litSum; for (u=0; u < litLength; u++) - price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); + price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1); if (litLength >= 12) { ssPtr->cachedLiterals = literals; @@ -139,8 +139,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; - price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1); + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1); } return price; @@ -150,8 +150,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { /* offset */ - BYTE offCode = (BYTE)ZSTD_highbit(offset+1); - U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); + BYTE offCode = (BYTE)ZSTD_highbit32(offset+1); + U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); /* match Length */ { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, @@ -163,8 +163,8 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; - price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1); + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1); } return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; @@ -190,13 +190,13 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; seqStorePtr->litLengthFreq[llCode]++; seqStorePtr->litLengthSum++; } /* match offset */ - { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + { BYTE offCode = (BYTE)ZSTD_highbit32(offset+1); seqStorePtr->offCodeSum++; seqStorePtr->offCodeFreq[offCode]++; } @@ -211,7 +211,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; seqStorePtr->matchLengthFreq[mlCode]++; seqStorePtr->matchLengthSum++; } diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index ef2b1ae5b..ab79f7492 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -60,7 +60,7 @@ #include "error_private.h" #include "fse.h" #include "huf_static.h" -#include "zstd_internal.h" +#include "zstd_internal.h" /* includes zstd.h */ #include "xxhash.h" #include "divsufsort.h" #define ZDICT_STATIC_LINKING_ONLY From f938ad5406a69aba55cf55532c2c1946831d0324 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Jun 2016 21:59:56 +0200 Subject: [PATCH 137/208] fixed cmake project --- projects/cmake/lib/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index 84568b207..c9ec05fe9 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -81,12 +81,9 @@ SET(Headers ${LIBRARY_DIR}/common/huf_static.h ${LIBRARY_DIR}/common/mem.h ${LIBRARY_DIR}/common/zbuff.h - ${LIBRARY_DIR}/common/zbuff_static.h ${LIBRARY_DIR}/common/zstd_internal.h - ${LIBRARY_DIR}/common/zstd_static.h ${LIBRARY_DIR}/common/zstd.h - ${LIBRARY_DIR}/dictBuilder/zdict.h - ${LIBRARY_DIR}/dictBuilder/zdict_static.h) + ${LIBRARY_DIR}/dictBuilder/zdict.h) IF (ZSTD_LEGACY_SUPPORT) SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy) From 068a8c1b10d515651411c90fb2d154b6d575b5c8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 00:23:20 +0200 Subject: [PATCH 138/208] commented out versionTests, which can not work while waiting for legacy decoder v0.6 --- .travis.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 41f67c77a..879aff8a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,9 @@ language: c compiler: gcc matrix: fast_finish: true - include: + include: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - os: linux + - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install - os: linux @@ -47,9 +47,9 @@ matrix: - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest" +# - os: linux +# sudo: required +# env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32 @@ -66,7 +66,7 @@ matrix: sudo: required env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper # OS X Mavericks - - os: osx + - os: osx env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install - os: osx env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test @@ -86,6 +86,5 @@ before_install: fi fi -script: +script: - make $MAKE_PARAM - From dc048d18d3662795a60e34fa4d0025f3d56896b5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 00:32:23 +0200 Subject: [PATCH 139/208] minor comment (detailing an `#include` motivation) --- lib/compress/zbuff_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 367320e2b..9c9c8059f 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -35,7 +35,7 @@ ***************************************/ #include #include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */ +#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */ #define ZBUFF_STATIC_LINKING_ONLY #include "zbuff.h" From 130fe11394b34c36dd2733cd26db6fda72b5e49f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 00:42:28 +0200 Subject: [PATCH 140/208] merged `huf_static.h` into `huf.h` . Requires `HUF_STATIC_LINKING_ONLY` macro. --- lib/common/huf.h | 188 +++++++++++++++++++++++++ lib/common/huf_static.h | 232 ------------------------------- lib/compress/huf_compress.c | 3 +- lib/compress/zstd_compress.c | 3 +- lib/decompress/huf_decompress.c | 3 +- lib/decompress/zstd_decompress.c | 5 +- lib/dictBuilder/zdict.c | 3 +- 7 files changed, 199 insertions(+), 238 deletions(-) delete mode 100644 lib/common/huf_static.h diff --git a/lib/common/huf.h b/lib/common/huf.h index a06fd3e18..cee956a0f 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -91,6 +91,194 @@ const char* HUF_getErrorName(size_t code); /**< provides error code string (us size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + +#ifdef HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "fse.h" +#include "bitstream.h" /* includes mem.h */ + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + U32 name##hb[maxSymbolValue+1]; \ + void* name##hv = &(name##hb); \ + HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + +/* static allocation of HUF's DTable */ +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<= 64 */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/*! +HUF_compress() does the following: +1. count symbol occurrence from source[] into table count[] using FSE_count() +2. (optional) refine tableLog using HUF_optimalTableLog() +3. build Huffman table from count using HUF_buildCTable() +4. save Huffman table to memory buffer using HUF_writeCTable() +5. encode the data stream using HUF_compress4X_usingCTable() + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ +/* FSE_count() : find it within "fse.h" */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + + +/*! +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ +size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); +size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ + +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); +size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); + + +/* Loading a CTable saved with HUF_writeCTable() */ + +size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); + + +/* ************************************************************** +* Constants +****************************************************************/ +#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */ +#define HUF_SYMBOLVALUE_MAX 255 +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + + +/* ************************************************************** +* Needed by zstd in both compression and decompression +****************************************************************/ +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` +*/ +MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + size_t oSize; + + //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + if (iSize >= (242)) { /* RLE */ + static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; + oSize = l[iSize-242]; + memset(huffWeight, 1, hwSize); + iSize = 0; + } + else { /* Incompressible */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + + +#endif /* HUF_STATIC_LINKING_ONLY */ + + #if defined (__cplusplus) } #endif diff --git a/lib/common/huf_static.h b/lib/common/huf_static.h deleted file mode 100644 index ea3eb62b8..000000000 --- a/lib/common/huf_static.h +++ /dev/null @@ -1,232 +0,0 @@ -/* ****************************************************************** - Huffman codec, part of New Generation Entropy library - header file, for static linking only - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef HUF_STATIC_H -#define HUF_STATIC_H - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* Dependency -******************************************/ -#include "huf.h" -#include "fse.h" -#include "bitstream.h" - - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define HUF_CTABLEBOUND 129 -#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ -#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* static allocation of HUF's Compression Table */ -#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[maxSymbolValue+1]; \ - void* name##hv = &(name##hb); \ - HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ - -/* static allocation of HUF's DTable */ -#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<= 64 */ - - -/* **************************************** -* HUF detailed API -******************************************/ -/*! -HUF_compress() does the following: -1. count symbol occurrence from source[] into table count[] using FSE_count() -2. (optional) refine tableLog using HUF_optimalTableLog() -3. build Huffman table from count using HUF_buildCTable() -4. save Huffman table to memory buffer using HUF_writeCTable() -5. encode the data stream using HUF_compress4X_usingCTable() - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and regenerate 'CTable' using external methods. -*/ -/* FSE_count() : find it within "fse.h" */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - - -/*! -HUF_decompress() does the following: -1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics -2. build Huffman table from save, using HUF_readDTableXn() -3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -*/ -size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize); - -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - -/* single stream variants */ - -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ - -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - -/* Loading a CTable saved with HUF_writeCTable() */ - -size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); - - -/* ************************************************************** -* Constants -****************************************************************/ -#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */ -#define HUF_SYMBOLVALUE_MAX 255 -#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) -# error "HUF_TABLELOG_MAX is too large !" -#endif - - - -/* ************************************************************** -* Needed by zstd in both compression and decompression -****************************************************************/ -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` -*/ -MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - size_t oSize; - - //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - if (iSize >= (242)) { /* RLE */ - static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; - oSize = l[iSize-242]; - memset(huffWeight, 1, hwSize); - iSize = 0; - } - else { /* Incompressible */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } } - else { /* header compressed with FSE (normal case) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSE_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } } - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - return iSize+1; -} - - - -#if defined (__cplusplus) -} -#endif - -#endif /* HUF_STATIC_H */ diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 05cf69657..c1cfe0075 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -61,9 +61,10 @@ ****************************************************************/ #include /* memcpy, memset */ #include /* printf (debug) */ -#include "huf_static.h" #include "bitstream.h" #include "fse_static.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" /* ************************************************************** diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 49c3d2663..7898cf3fc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -56,7 +56,8 @@ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH_reset, update, digest */ #include "fse_static.h" -#include "huf_static.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" #include "zstd_internal.h" /* includes zstd.h */ diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 14afbb927..d1096e457 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -61,9 +61,10 @@ ****************************************************************/ #include /* memcpy, memset */ #include /* printf (debug) */ -#include "huf_static.h" #include "bitstream.h" #include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 9fe1b6506..e3d4b3d6c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -58,9 +58,10 @@ #include "mem.h" /* low level memory routines */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH64_* */ -#include "zstd_internal.h" #include "fse_static.h" -#include "huf_static.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_legacy.h" diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index ab79f7492..2e15cbbf8 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -59,7 +59,8 @@ #include "mem.h" /* read */ #include "error_private.h" #include "fse.h" -#include "huf_static.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" #include "zstd_internal.h" /* includes zstd.h */ #include "xxhash.h" #include "divsufsort.h" From d0e2cd15cbe0db1ce01835fffedd71e8c3b5f7b0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 00:58:01 +0200 Subject: [PATCH 141/208] Merged `fse_static` into `fse.h` . Now requires `FSE_STATIC_LINKING_ONLY` macro. --- lib/common/entropy_common.c | 3 +- lib/common/fse.h | 349 +++++++++++++++++++++++++++ lib/common/fse_decompress.c | 3 +- lib/common/fse_static.h | 392 ------------------------------- lib/compress/fse_compress.c | 3 +- lib/compress/huf_compress.c | 3 +- lib/compress/zstd_compress.c | 3 +- lib/decompress/zstd_decompress.c | 3 +- programs/fullbench.c | 7 +- 9 files changed, 364 insertions(+), 402 deletions(-) delete mode 100644 lib/common/fse_static.h diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index b4c366e61..8dd5b00ea 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -37,7 +37,8 @@ ***************************************/ #include #include "mem.h" -#include "fse_static.h" /* FSE_MIN_TABLELOG */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" #include "error_private.h" #include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ #include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ diff --git a/lib/common/fse.h b/lib/common/fse.h index 6dce68300..6be3e5aa0 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -272,6 +272,355 @@ If there is an error, the function will return an error code, which can be teste */ +#ifdef FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) +{ + const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + +/*<===== Decompression =====>*/ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#define FSE_MAX_MEMORY_USAGE 14 +#define FSE_DEFAULT_MEMORY_USAGE 13 + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#define FSE_MAX_SYMBOL_VALUE 255 + + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + #if defined (__cplusplus) } #endif diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index 601edd741..918de64c5 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -58,7 +58,8 @@ #include /* memcpy, memset */ #include /* printf (debug) */ #include "bitstream.h" -#include "fse_static.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" /* ************************************************************** diff --git a/lib/common/fse_static.h b/lib/common/fse_static.h deleted file mode 100644 index 797258f75..000000000 --- a/lib/common/fse_static.h +++ /dev/null @@ -1,392 +0,0 @@ -/* ****************************************************************** - FSE : Finite State Entropy coder - header file for static linking (only) - Copyright (C) 2013-2015, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ -#ifndef FSE_STATIC_H -#define FSE_STATIC_H - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* ***************************************** -* Dependencies -*******************************************/ -#include "fse.h" -#include "bitstream.h" - - -/* ***************************************** -* Static allocation -*******************************************/ -/* FSE buffer bounds */ -#define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size>>7)) -#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ -#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) -#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= BIT_DStream_completed - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - BIT_endOfDStream(&DStream); -Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. - FSE_endOfDState(&DState); -*/ - - -/* ***************************************** -* FSE unsafe API -*******************************************/ -static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); -/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ - - -/* ***************************************** -* Implementation of inlined functions -*******************************************/ -typedef struct { - int deltaFindState; - U32 deltaNbBits; -} FSE_symbolCompressionTransform; /* total 8 bytes */ - -MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) -{ - const void* ptr = ct; - const U16* u16ptr = (const U16*) ptr; - const U32 tableLog = MEM_read16(ptr); - statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; - statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); - statePtr->stateLog = tableLog; -} - -MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) -{ - FSE_initCState(statePtr, ct); - { - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; - const U16* stateTable = (const U16*)(statePtr->stateTable); - U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); - statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; - statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; - - } -} - -MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) -{ - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; - const U16* const stateTable = (const U16*)(statePtr->stateTable); - U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); - statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; -} - -MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) -{ - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); - BIT_flushBits(bitC); -} - -/*<===== Decompression =====>*/ - -typedef struct { - U16 tableLog; - U16 fastMode; -} FSE_DTableHeader; /* sizeof U32 */ - -typedef struct -{ - unsigned short newState; - unsigned char symbol; - unsigned char nbBits; -} FSE_decode_t; /* size == U32 */ - -MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - return DInfo.symbol; -} - -MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.newState + lowBits; -} - -MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBits(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -/*! FSE_decodeSymbolFast() : - unsafe, only works if no symbol has a probability > 50% */ -MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBitsFast(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) -{ - return DStatePtr->state == 0; -} - - - -#ifndef FSE_COMMONDEFS_ONLY - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSE_MAX_MEMORY_USAGE 14 -#define FSE_DEFAULT_MEMORY_USAGE 13 - -/*!FSE_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#define FSE_MAX_SYMBOL_VALUE 255 - - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSE_FUNCTION_TYPE BYTE -#define FSE_FUNCTION_EXTENSION -#define FSE_DECODE_TYPE FSE_decode_t - - -#endif /* !FSE_COMMONDEFS_ONLY */ - - -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) -#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX -#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) - - -#if defined (__cplusplus) -} -#endif - -#endif /* FSE_STATIC_H */ diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index f1068433a..5c804dcaf 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -57,7 +57,8 @@ #include /* memcpy, memset */ #include /* printf (debug) */ #include "bitstream.h" -#include "fse_static.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" /* ************************************************************** diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index c1cfe0075..3533bb613 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -62,7 +62,8 @@ #include /* memcpy, memset */ #include /* printf (debug) */ #include "bitstream.h" -#include "fse_static.h" /* header compression */ +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "fse.h" /* header compression */ #define HUF_STATIC_LINKING_ONLY #include "huf.h" diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7898cf3fc..1ae202772 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -55,7 +55,8 @@ #include "mem.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH_reset, update, digest */ -#include "fse_static.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_internal.h" /* includes zstd.h */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index e3d4b3d6c..63b674dfd 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -58,7 +58,8 @@ #include "mem.h" /* low level memory routines */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH64_* */ -#include "fse_static.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_internal.h" diff --git a/programs/fullbench.c b/programs/fullbench.c index 9fe4a1219..01e8f59ee 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -28,13 +28,13 @@ #include "util.h" /* Compiler options, UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ -#include /* strcmp */ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */ #include "zstd.h" /* ZSTD_VERSION_STRING */ -#include "fse_static.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_DTABLE_SIZE_U32 */ +#include "fse.h" #include "zbuff.h" #include "datagen.h" @@ -545,4 +545,3 @@ int main(int argc, const char** argv) return result; } - From f3a1a84c9aebbcb13cdb592400fd06d7688381d3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 01:05:01 +0200 Subject: [PATCH 142/208] fixed cmake project --- programs/zstdcli.c | 3 ++- projects/cmake/lib/CMakeLists.txt | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7c235c79e..75ce129fe 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -225,7 +225,8 @@ int main(int argCount, const char** argv) #endif /* init */ - (void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ + (void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */ (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } filenameTable[0] = stdinmark; diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index c9ec05fe9..3b988fd16 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -76,9 +76,7 @@ SET(Headers ${LIBRARY_DIR}/common/error_private.h ${LIBRARY_DIR}/common/error_public.h ${LIBRARY_DIR}/common/fse.h - ${LIBRARY_DIR}/common/fse_static.h ${LIBRARY_DIR}/common/huf.h - ${LIBRARY_DIR}/common/huf_static.h ${LIBRARY_DIR}/common/mem.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/common/zstd_internal.h From a91ca620cfea5b0732c5e1268211cecbe55eefe0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 01:33:55 +0200 Subject: [PATCH 143/208] removed `HUF_readStats()` from public space --- lib/common/entropy_common.c | 82 ++++++++++++++++++++++++++-- lib/common/huf.h | 105 +++++++----------------------------- programs/.gitignore | 2 + 3 files changed, 97 insertions(+), 92 deletions(-) diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index 8dd5b00ea..b42acb4a3 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -35,13 +35,12 @@ /* ************************************* * Dependencies ***************************************/ -#include #include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ -#include "fse.h" -#include "error_private.h" -#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ -#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ +#include "fse.h" /* FSE_isError, FSE_getErrorName */ +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" /* HUF_isError, HUF_getErrorName */ @@ -157,3 +156,76 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); return ip-istart; } + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableXn() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + size_t oSize; + + //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + if (iSize >= (242)) { /* RLE */ + static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; + oSize = l[iSize-242]; + memset(huffWeight, 1, hwSize); + iSize = 0; + } + else { /* Incompressible */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} diff --git a/lib/common/huf.h b/lib/common/huf.h index cee956a0f..d34f8685f 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -99,6 +99,16 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize #include "bitstream.h" /* includes mem.h */ +/* *** Constants *** */ +#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */ +#define HUF_SYMBOLVALUE_MAX 255 +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + /* **************************************** * Static allocation ******************************************/ @@ -183,97 +193,18 @@ size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* c size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -/* Loading a CTable saved with HUF_writeCTable() */ - -size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); - - -/* ************************************************************** -* Constants -****************************************************************/ -#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */ -#define HUF_SYMBOLVALUE_MAX 255 -#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) -# error "HUF_TABLELOG_MAX is too large !" -#endif - - - -/* ************************************************************** -* Needed by zstd in both compression and decompression -****************************************************************/ /*! HUF_readStats() : Read compact Huffman tree, saved by HUF_writeCTable(). `huffWeight` is destination buffer. - @return : size read from `src` -*/ -MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - size_t oSize; + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableXn() . +*/size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); - //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - if (iSize >= (242)) { /* RLE */ - static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; - oSize = l[iSize-242]; - memset(huffWeight, 1, hwSize); - iSize = 0; - } - else { /* Incompressible */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } } - else { /* header compressed with FSE (normal case) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSE_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } } - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - return iSize+1; -} +/** HUF_readCTable() : +* Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); #endif /* HUF_STATIC_LINKING_ONLY */ diff --git a/programs/.gitignore b/programs/.gitignore index 0f391d2da..5f50de0dd 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -1,6 +1,8 @@ # local binary (Makefile) zstd zstd32 +zstd-compress +zstd-decompress fullbench fullbench32 fuzzer From 51778b7cca9b2128a6e98d910653526106f27096 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 01:38:10 +0200 Subject: [PATCH 144/208] updated README following merging of `*_static.h` --- lib/README.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/README.md b/lib/README.md index 9a728113e..45e8e6fdc 100644 --- a/lib/README.md +++ b/lib/README.md @@ -12,27 +12,22 @@ To build the zstd library the following files are required: - [common/error_public.h](common/error_public.h) - common/fse.h - common/fse_decompress.c -- common/fse_static.h - common/huf.h -- common/huf_static.h - [common/mem.h](common/mem.h) - [common/zstd.h] - common/zstd_internal.h -- common/zstd_static.h - compress/fse_compress.c - compress/huf_compress.c - compress/zstd_compress.c -- compress/zstd_opt.h +- compress/zstd_opt.h - decompress/huf_decompress.c - decompress/zstd_decompress.c Stable API is exposed in [common/zstd.h]. -Advanced and experimental API is exposed in [common/zstd_static.h]. -API elements of [common/zstd_static.h] should be used with static linking only, -as their definition may change in future version of the library. +Advanced and experimental API can be enabled by defining `ZSTD_STATIC_LINKING_ONLY`. +Never use them with a dynamic library, as their definition may change in future versions. [common/zstd.h]: common/zstd.h -[common/zstd_static.h]: common/zstd_static.h #### Separate compressor and decompressor @@ -47,7 +42,6 @@ This complementary API makes streaming integration easier. It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) : - common/zbuff.h -- common/zbuff_static.h - compress/zbuff_compress.c - decompress/zbuff_decompress.c @@ -59,7 +53,6 @@ To create dictionaries from training sets : - dictBuilder/divsufsort.h - dictBuilder/zdict.c - dictBuilder/zdict.h -- dictBuilder/zdict_static.h #### Miscellaneous @@ -68,4 +61,3 @@ The other files are not source code. There are : - LICENSE : contains the BSD license text - Makefile : script to compile or install zstd library (static or dynamic) - libzstd.pc.in : for pkg-config (make install) - From 89703d20fbf0db85a8bbb3605bb27ac725881f4a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Jun 2016 01:50:33 +0200 Subject: [PATCH 145/208] reduced dependencies --- lib/common/huf.h | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/lib/common/huf.h b/lib/common/huf.h index d34f8685f..d54e6616f 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -39,13 +39,11 @@ extern "C" { #endif -/* **************************************** -* Dependency -******************************************/ +/* *** Dependencies *** */ #include /* size_t */ -/* **************************************** +/*-**************************************** * HUF simple functions ******************************************/ size_t HUF_compress(void* dst, size_t dstCapacity, @@ -54,17 +52,17 @@ size_t HUF_decompress(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* HUF_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster if dstCapacity >= HUF_compressBound(srcSize). - Note : srcSize must be <= 128 KB - @return : size of compressed data (<= dstCapacity) + Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + Note : `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB + @return : size of compressed data (<= `dstCapacity`) Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! if return == 1, srcData is a single repeated byte symbol (RLE compression). if HUF_isError(return), compression failed (more details using HUF_getErrorName()) HUF_decompress() : Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstSize'. + into already allocated buffer 'dst', of minimum size 'dstSize'. `dstSize` : must be the **exact** size of original (uncompressed) data. Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, @@ -78,25 +76,24 @@ HUF_decompress() : * Tool functions ******************************************/ #define HUF_BLOCKSIZE_MAX (128 * 1024) -size_t HUF_compressBound(size_t size); /**< maximum compressed size */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ /* Error Management */ unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ -/* **************************************** -* Advanced functions -******************************************/ -size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +/* *** Advanced function *** */ +/** HUF_compress2() : +* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */ +size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); #ifdef HUF_STATIC_LINKING_ONLY /* *** Dependencies *** */ -#include "fse.h" -#include "bitstream.h" /* includes mem.h */ +#include "mem.h" /* U32 */ /* *** Constants *** */ @@ -197,8 +194,8 @@ size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* c Read compact Huffman tree, saved by HUF_writeCTable(). `huffWeight` is destination buffer. @return : size read from `src` , or an error Code . - Note : Needed by HUF_readCTable() and HUF_readDTableXn() . -*/size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize); From 673f0d7cdcd5d39d2c06d5c24ebaddaf05d57e22 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 00:26:38 +0200 Subject: [PATCH 146/208] new frame format, allowing custom window size --- lib/common/zbuff.h | 3 +- lib/common/zstd.h | 11 ++-- lib/common/zstd_internal.h | 4 +- lib/compress/zstd_compress.c | 85 ++++++++++++++++-------- lib/decompress/zbuff_decompress.c | 6 +- lib/decompress/zstd_decompress.c | 103 +++++++++++++++++------------- programs/fileio.c | 2 +- programs/playTests.sh | 8 +-- programs/zbufftest.c | 23 ++++--- 9 files changed, 146 insertions(+), 99 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index f8e8976f4..e449f6d3b 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -189,8 +189,7 @@ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); - -#endif +#endif /* ZBUFF_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 2f7ab1e87..b899656d1 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -180,6 +180,11 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, #define ZSTD_TARGETLENGTH_MIN 4 #define ZSTD_TARGETLENGTH_MAX 999 +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_min = 5; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ + /*--- Types ---*/ typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ @@ -305,15 +310,11 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci typedef struct { U64 frameContentSize; - U32 windowLog; + U32 windowSize; U32 dictID; U32 checksumFlag; } ZSTD_frameParams; -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ -static const size_t ZSTD_frameHeaderSize_min = 6; -static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; -static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 2e238faad..ebf3e880c 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -81,8 +81,8 @@ #define BIT1 2 #define BIT0 1 -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12 -static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1ae202772..39f073962 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -128,6 +128,7 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; + U64 frameContentSize; XXH64_state_t xxhState; ZSTD_customMem customMem; @@ -257,7 +258,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface /*! ZSTD_resetCCtx_advanced() : note : 'params' is expected to be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, - ZSTD_parameters params, U32 reset) + ZSTD_parameters params, U64 frameContentSize, U32 reset) { /* note : params considered validated here */ const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); const U32 divider = (params.cParams.searchLength==3) ? 3 : 4; @@ -265,7 +266,10 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); const size_t hSize = ((size_t)1) << params.cParams.hashLog; - const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; + const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 : + ( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX : + ((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) ); + const size_t h3Size = 1 << hashLog3; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); /* Check if workSpace is large enough, alloc a new one if needed */ @@ -282,6 +286,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ XXH64_reset(&zc->xxhState, 0); + zc->hashLog3 = hashLog3; zc->hashTable3 = (U32*)(zc->workSpace); zc->hashTable = zc->hashTable3 + h3Size; zc->chainTable = zc->hashTable + hSize; @@ -298,6 +303,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->lowLimit = 0; zc->params = params; zc->blockSize = blockSize; + zc->frameContentSize = frameContentSize; if (params.cParams.strategy == ZSTD_btopt) { zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer); @@ -333,9 +339,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { if (srcCCtx->stage!=1) return ERROR(stage_wrong); - dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ /* copy tables */ @@ -403,8 +408,9 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) /* Frame format description Frame Header - [ Block Header - Block ] - Frame End 1) Frame Header - - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) - - 1 byte - Frame Descriptor + - 4 bytes : Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 1 byte : Frame Header Descriptor + - 1-13 bytes : Optional fields 2) Block Header - 3 bytes, starting with a 2-bits descriptor Uncompressed, Compressed, Frame End, unused @@ -417,13 +423,38 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) /* Frame descriptor - 1 byte, using : + // old + 1 byte - Alloc : bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) - bit 4 : minmatch 4(0) or 3(1) + bit 4 : reserved for windowLog (must be zero) bit 5 : reserved (must be zero) bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes - Optional : content size (0, 1, 2 or 8 bytes) + 1 byte - checker : + bit 0-1 : dictID (0, 1, 2 or 4 bytes) + bit 2-7 : reserved (must be zero) + + + // new + 1 byte - FrameHeaderDescription : + bit 0-1 : dictID (0, 1, 2 or 4 bytes) + bit 2-4 : reserved (must be zero) + bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present) + bit 6-7 : FrameContentFieldsize (0, 2, 4, or 8) + if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1; + + Optional : WindowLog (0 or 1 byte) + bit 0-2 : octal Fractional (1/8th) + bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB) + + Optional : dictID (0, 1, 2 or 4 bytes) + Automatic adaptation + 0 : no dictID + 1 : 1 - 255 + 2 : 256 - 65535 + 4 : all other values + + Optional : content size (0, 1, 2, 4 or 8 bytes) 0 : unknown 1 : 0-255 bytes 2 : 256 - 65535+256 @@ -2116,21 +2147,22 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; - U32 const fcsId = params.fParams.contentSizeFlag ? - (pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */ - 0; - BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */ - | (fcsId << 6) ); U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4)); + U32 const checksumFlag = params.fParams.checksumFlag; + U32 const windowSize = 1U << params.cParams.windowLog; + U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1)); + BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params.fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ + 0; + BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (directModeFlag<<5) + (fcsCode<<6) ); size_t pos; if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); MEM_writeLE32(dst, ZSTD_MAGICNUMBER); - op[4] = fAllocByte; - op[5] = fCheckByte; - pos = 6; + op[4] = frameHeaderDecriptionByte; pos=5; + if (!directModeFlag) op[pos++] = windowLogByte; switch(dictIDSizeCode) { default: /* impossible */ @@ -2139,12 +2171,12 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break; case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; } - switch(fcsId) + switch(fcsCode) { default: /* impossible */ - case 0 : break; - case 1 : op[pos] = (BYTE)(pledgedSrcSize); pos++; break; - case 2 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 0 : if (directModeFlag) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; } return pos; @@ -2161,7 +2193,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc, if (zc->stage==0) return ERROR(stage_wrong); if (frame && (zc->stage==1)) { /* copy saved header */ - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize, zc->dictID); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; dst = (char*)dst + fhSize; @@ -2344,11 +2376,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - { U32 const hashLog3 = (!pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); - zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; } - - { size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1); - if (ZSTD_isError(resetError)) return resetError; } + size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1); + if (ZSTD_isError(resetError)) return resetError; return ZSTD_compress_insertDictionary(zc, dict, dictSize); } diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index a6b99c976..b6e1806e3 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -191,10 +191,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (ZSTD_isError(h2Result)) return h2Result; } } - if (zbd->fParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) zbd->fParams.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for buffer allocation */ + zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); /* Frame header instruct buffer sizes */ - { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); + { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; if (zbd->inBuffSize < blockSize) { zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); @@ -202,7 +202,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } - { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; + { size_t const neededOutSize = zbd->fParams.windowSize + blockSize; if (zbd->outBuffSize < neededOutSize) { zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->outBuffSize = neededOutSize; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 63b674dfd..4630eb8a3 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -64,7 +64,7 @@ #include "huf.h" #include "zstd_internal.h" -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) # include "zstd_legacy.h" #endif @@ -218,22 +218,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) bit 0-1 : dictID (0, 1, 2 or 4 bytes) bit 2-7 : reserved (must be zero) - // new - 1 byte - Alloc : + 1 byte - FrameHeaderDescription : bit 0-1 : dictID (0, 1, 2 or 4 bytes) bit 2-4 : reserved (must be zero) - bit 5 : WindowLog skipped (note : if 1, then fcs > 0) - bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes + bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present) + bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8) + if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1; - OR : 0-3 : no windowLog, 1, 2, 4, 8 - 4-7 : windowLog, 0, 2, 4, 8 - - 1 byte - WindowLog (can be skipped if fcs>0) + Optional : WindowLog (0 or 1 byte) bit 0-2 : octal Fractional (1/8th) bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB) - Optional : dictID (0, 1, 2 or 4 bytes) Automatic adaptation 0 : no dictID @@ -241,11 +237,12 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) 2 : 256 - 65535 4 : all other values - Optional : content size (0, 1, 2 or 8 bytes) - 0 : unknown - 1 : 0-255 bytes - 2 : 256 - 65535+256 - 8 : up to 16 exa + Optional : content size (0, 1, 2, 4 or 8 bytes) + 0 : unknown (fcfs==0 and swl==0) + 1 : 0-255 bytes (fcfs==0 and swl==1) + 2 : 256 - 65535+256 (fcfs==1) + 4 : 0 - 4GB-1 (fcfs==2) + 8 : 0 - 16EB-1 (fcfs==3) */ @@ -321,15 +318,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); - { U32 const fcsId = (((const BYTE*)src)[4]) >> 6; - U32 const dictID =(((const BYTE*)src)[5]) & 3; - return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId] + ZSTD_did_fieldSize[dictID]; + { BYTE const fhd = ((const BYTE*)src)[4]; + U32 const dictID= fhd & 3; + U32 const directMode = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return ZSTD_frameHeaderSize_min + !directMode + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (directMode && !ZSTD_fcs_fieldSize[fcsId]); } } /** ZSTD_getFrameParams() : -* decode Frame Header, or provide expected `srcSize`. +* decode Frame Header, or require larger `srcSize`. * @return : 0, `fparamsPtr` is correctly filled, * >0, `srcSize` is too small, result is expected `srcSize`, * or an error code, which can be tested using ZSTD_isError() */ @@ -343,7 +343,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ memset(fparamsPtr, 0, sizeof(*fparamsPtr)); fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); - fparamsPtr->windowLog = 0; /* windowLog==0 means a frame is skippable */ + fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ return 0; } return ERROR(prefix_unknown); @@ -353,31 +353,48 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); if (srcSize < fhsize) return fhsize; } - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - { BYTE const allocByte = ip[4]; - BYTE const checkByte = ip[5]; - size_t pos = ZSTD_frameHeaderSize_min; - U32 const dictIDSizeCode = checkByte&3; - if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ - if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ - fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - fparamsPtr->checksumFlag = checkByte & 0x10; - switch(dictIDSizeCode) /* fcsId */ - { - default: /* impossible */ - case 0 : fparamsPtr->dictID = 0; break; - case 1 : fparamsPtr->dictID = ip[pos]; pos++; break; - case 2 : fparamsPtr->dictID = MEM_readLE16(ip+pos); pos+=2; break; - case 3 : fparamsPtr->dictID = MEM_readLE32(ip+pos); pos+=4; break; + { BYTE const fhdByte = ip[4]; + size_t pos = 5; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const directMode = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; + U32 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = 0; + if ((fhdByte & 0x18) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + if (!directMode) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + windowSize = (1U << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); } - switch(allocByte >> 6) /* fcsId */ + + switch(dictIDSizeCode) { default: /* impossible */ - case 0 : fparamsPtr->frameContentSize = 0; break; - case 1 : fparamsPtr->frameContentSize = ip[pos]; break; - case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+pos)+256; break; - case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+pos); break; - } } + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: /* impossible */ + case 0 : if (directMode) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (!windowSize) windowSize = (U32)frameContentSize; + if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + fparamsPtr->frameContentSize = frameContentSize; + fparamsPtr->windowSize = windowSize; + fparamsPtr->dictID = dictID; + fparamsPtr->checksumFlag = checksumFlag; + } return 0; } @@ -388,7 +405,6 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize) { size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); - if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); return result; @@ -1059,7 +1075,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c if (srcSize != dctx->expected) return ERROR(srcSize_wrong); if (dstCapacity) ZSTD_checkContinuity(dctx, dst); - /* Decompress : frame header; part 1 */ switch (dctx->stage) { case ZSTDds_getFrameHeaderSize : diff --git a/programs/fileio.c b/programs/fileio.c index 4d7da9130..37f6e7822 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -615,7 +615,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); /* Header loading (optional, saves one loop) */ - { size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */ + { size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */ size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); readSize = alreadyLoaded + loadedSize; } diff --git a/programs/playTests.sh b/programs/playTests.sh index 46e2fb6a4..f51d28ea2 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -23,7 +23,7 @@ roundTripTest() { isWindows=false ECHO="echo" case "$OS" in - Windows*) + Windows*) isWindows=true ECHO="echo -e" ;; @@ -42,8 +42,8 @@ file $ZSTD $ECHO "\n**** simple tests **** " ./datagen > tmp -$ZSTD -f tmp # trivial compression case, creates tmp.zst -$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) +$ZSTD -f tmp # trivial compression case, creates tmp.zst +$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) $ECHO "test : too large compression level (must fail)" $ZSTD -99 tmp && die "too large compression level undetected" $ECHO "test : compress to stdout" @@ -57,7 +57,7 @@ $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout $ZSTD --decompress tmpCompressed -c > tmpResult $ZSTD --decompress tmpCompressed --stdout > tmpResult if [ "$isWindows" = false ] ; then - $ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout + $ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout $ZSTD -d - < tmp.zst > /dev/null fi $ZSTD -dc < tmp.zst > /dev/null diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 78122e710..b890786cf 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -171,6 +171,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); cSize = skippableFrameSize + 8; + /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); @@ -186,14 +187,18 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo cSize += genSize; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); - /* Basic decompression test */ - DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + /* skippable frame test */ + DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSkipSize = cSize; genSize = CNBufferSize; { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); if (r != 0) goto _output_error; } if (genSize != 0) goto _output_error; /* skippable frame len is 0 */ + DISPLAYLEVEL(4, "OK \n"); + + /* Basic decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSize = cSize - readSkipSize; genSize = CNBufferSize; @@ -204,13 +209,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); { size_t i; - DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); for (i=0; i Date: Mon, 6 Jun 2016 00:46:56 +0200 Subject: [PATCH 147/208] fixed minor visual conversion warning --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 39f073962..da0409f88 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -269,7 +269,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 : ( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) ); - const size_t h3Size = 1 << hashLog3; + const size_t h3Size = ((size_t)1) << hashLog3; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); /* Check if workSpace is large enough, alloc a new one if needed */ From e3d529403d9e8ba983baa97ffeb6e8a924415ce5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 11:07:33 +0200 Subject: [PATCH 148/208] fixed initialization mismatch in `ZSTD_copyCCtx()` --- lib/compress/zstd_compress.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index da0409f88..48ca30e49 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -287,10 +287,10 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ XXH64_reset(&zc->xxhState, 0); zc->hashLog3 = hashLog3; - zc->hashTable3 = (U32*)(zc->workSpace); - zc->hashTable = zc->hashTable3 + h3Size; + zc->hashTable = (U32*)(zc->workSpace); zc->chainTable = zc->hashTable + hSize; - zc->seqStore.buffer = zc->chainTable + chainSize; + zc->hashTable3 = zc->chainTable + chainSize; + zc->seqStore.buffer = zc->hashTable3 + h3Size; zc->hufTable = (HUF_CElt*)zc->seqStore.buffer; zc->flagStaticTables = 0; zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256; @@ -346,7 +346,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) /* copy tables */ { const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; - const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0; + const size_t h3Size = (size_t)1 << srcCCtx->hashLog3; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); } @@ -2107,7 +2107,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, while (remaining) { size_t cSize; - ZSTD_statsResetFreqs(stats); + ZSTD_statsResetFreqs(stats); /* debug only */ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; From 2cc72f1fd35b9778181f6e2fe554d6ca162450d0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 17:50:07 +0200 Subject: [PATCH 149/208] fixed initialization issue in bench --- lib/compress/zstd_compress.c | 2 +- programs/bench.c | 1 + programs/fileio.c | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 48ca30e49..9c06f603c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2148,7 +2148,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - U32 const checksumFlag = params.fParams.checksumFlag; + U32 const checksumFlag = params.fParams.checksumFlag>0; U32 const windowSize = 1U << params.cParams.windowLog; U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1)); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); diff --git a/programs/bench.c b/programs/bench.c index 08a972c75..ecc609d83 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -219,6 +219,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, do { U32 blockNb; { ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); params.fParams.contentSizeFlag = 1; { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); diff --git a/programs/fileio.c b/programs/fileio.c index 37f6e7822..47302f9ca 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -622,7 +622,6 @@ unsigned long long FIO_decompressFrame(dRess_t ress, /* Main decompression Loop */ while (1) { - /* Decode */ size_t inSize=readSize, decodedSize=ress.dstBufferSize; size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); From c28f9e271358d3c258236acb2748d90006031652 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 18:00:00 +0200 Subject: [PATCH 150/208] attempt to re-enable msan tests --- .travis.yml | 3 +++ Makefile | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 879aff8a9..6a2e90309 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ matrix: - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=msan - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper diff --git a/Makefile b/Makefile index 755b0c1a8..18db04245 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ ppctest-w-install: clean ppcinstall ppctest ppc64test: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" ppc64install: clean sudo apt-get update -y -q @@ -130,7 +130,7 @@ asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" msan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c used to fail this test for no obvious reason asan32: clean $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" From f72dc6091b24141db660eeb6f56b482c9f722155 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 19:42:47 +0200 Subject: [PATCH 151/208] removed msan tests, due to issues with `datagen.c` --- .travis.yml | 3 --- programs/datagen.c | 47 +++++++++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a2e90309..879aff8a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,6 @@ matrix: - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=msan - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper diff --git a/programs/datagen.c b/programs/datagen.c index fd9b2316f..ec118f5d1 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -27,9 +27,9 @@ * Includes **************************************/ #include /* malloc */ -#include /* FILE, fwrite */ +#include /* FILE, fwrite, fprintf */ #include /* memcpy */ -#include "mem.h" +#include "mem.h" /* U32 */ /*-************************************ @@ -92,7 +92,7 @@ static void RDG_fillLiteralDistrib(BYTE* ldt, double ld) for (u=0; u lastChar) character = firstChar; } @@ -108,43 +108,52 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt) } -#define RDG_RAND15BITS ( RDG_rand(seed) & 0x7FFF ) -#define RDG_RANDLENGTH ( (RDG_rand(seed) & 7) ? (RDG_rand(seed) & 0xF) : (RDG_rand(seed) & 0x1FF) + 0xF) +static U32 RDG_rand15Bits (unsigned* seedPtr) +{ + return RDG_rand(seedPtr) & 0x7FFF; +} + +static U32 RDG_randLength(unsigned* seedPtr) +{ + if (RDG_rand(seedPtr) & 7) + return (RDG_rand(seedPtr) & 0xF); + return (RDG_rand(seedPtr) & 0x1FF) + 0xF; +} + void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr) { - BYTE* buffPtr = (BYTE*)buffer; - const U32 matchProba32 = (U32)(32768 * matchProba); + BYTE* const buffPtr = (BYTE*)buffer; + U32 const matchProba32 = (U32)(32768 * matchProba); size_t pos = prefixSize; - U32* seed = seedPtr; U32 prevOffset = 1; /* special case : sparse content */ while (matchProba >= 1.0) { - size_t size0 = RDG_rand(seed) & 3; + size_t size0 = RDG_rand(seedPtr) & 3; size0 = (size_t)1 << (16 + size0 * 2); - size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ + size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/ if (buffSize < pos + size0) { memset(buffPtr+pos, 0, buffSize-pos); return; } memset(buffPtr+pos, 0, size0); pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, ldt); + buffPtr[pos-1] = RDG_genChar(seedPtr, ldt); continue; } /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, ldt), pos=1; + if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1; /* Generate compressible data */ while (pos < buffSize) { /* Select : Literal (char) or Match (within 32K) */ - if (RDG_RAND15BITS < matchProba32) { + if (RDG_rand15Bits(seedPtr) < matchProba32) { /* Copy (within 32K) */ - U32 const length = RDG_RANDLENGTH + 4; + U32 const length = RDG_randLength(seedPtr) + 4; U32 const d = (U32) MIN(pos + length , buffSize); - U32 const repeatOffset = (RDG_rand(seed) & 15) == 2; - U32 const randOffset = RDG_RAND15BITS + 1; + U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2; + U32 const randOffset = RDG_rand15Bits(seedPtr) + 1; U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos); size_t match = pos - offset; //TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length); @@ -152,9 +161,9 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match prevOffset = offset; } else { /* Literal (noise) */ - U32 const length = RDG_RANDLENGTH; + U32 const length = RDG_randLength(seedPtr); U32 const d = (U32) MIN(pos + length, buffSize); - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt); + while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt); } } } @@ -174,7 +183,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba, { size_t const stdBlockSize = 128 KB; size_t const stdDictSize = 32 KB; - BYTE* buff = (BYTE*)malloc(stdDictSize + stdBlockSize); + BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize); U64 total = 0; BYTE ldt[LTSIZE]; From 9d504ae85b6615dbfca90db8f33978a8df44aa33 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 19:52:35 +0200 Subject: [PATCH 152/208] Added decoding of RLE blocks --- lib/decompress/zstd_decompress.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 4630eb8a3..9b7b1886b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -53,7 +53,7 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include /* memcpy, memmove */ +#include /* memcpy, memmove, memset */ #include /* debug only : printf */ #include "mem.h" /* low level memory routines */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ @@ -938,6 +938,14 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } +size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length) +{ + if (length > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, byte, length); + return length; +} + + /*! ZSTD_decompressFrame() : * `dctx` must be properly initialized */ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, @@ -982,7 +990,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); break; case bt_rle : - return ERROR(GENERIC); /* not yet supported */ + decodedSize = ZSTD_generateNxByte(op, oend-op, *ip, blockProperties.origSize); break; case bt_end : /* end of frame */ From 81e13ef7cfceb5b79e11eb4c3042b5c4cbadd9c4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jun 2016 00:51:51 +0200 Subject: [PATCH 153/208] first implementation of the new dictionary API (untested) --- lib/common/zstd.h | 125 ++++++++++++++++++++++--------- lib/compress/zstd_compress.c | 85 +++++++++++++++++++-- lib/decompress/zstd_decompress.c | 74 ++++++++++++++++++ 3 files changed, 242 insertions(+), 42 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index b899656d1..ea0840b69 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -129,9 +129,47 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa /*-*********************** * Dictionary API *************************/ +/*! ZSTD_createCDict() : +* Create a digested dictionary, ready to start compression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_CDict_s ZSTD_CDict; +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : +* Compression using a pre-digested Dictionary +* In contrast with older ZSTD_compress_usingDict(), use dictionary without significant overhead. +* Note that compression level is decided during dictionary creation */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_DDict_s ZSTD_DDict; +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* In contrast with older ZSTD_decompress_usingDict(), use dictionary without significant overhead. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/*-*********************** +* Deprecated API +*************************/ /*! ZSTD_compress_usingDict() : +* *** Warning : this function will soon be declared deprecated *** * Compression using a pre-defined Dictionary content (see dictBuilder). -* Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */ +* Note 1 : This function load the dictionary, resulting in a significant startup time. +* Note 2 : `dict` must remain valid and unmodified during compression operation. +* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -139,9 +177,12 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, int compressionLevel); /*! ZSTD_decompress_usingDict() : +* *** Warning : this function will soon be declared deprecated *** * Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */ +* Dictionary must be identical to the one used during compression. +* Note 1 : This function load the dictionary, resulting in a significant startup time +* Note 2 : `dict` must remain valid and unmodified during compression operation. +* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -158,7 +199,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, * ==================================================================================== */ /*--- Dependency ---*/ -#include "mem.h" +#include "mem.h" /* U32 */ /*--- Constants ---*/ @@ -187,22 +228,22 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f /*--- Types ---*/ -typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ +typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ typedef struct { - U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */ - U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /* dispatch table : larger == faster, more memory */ - U32 searchLog; /* nb of searches : larger == more compression, slower */ - U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */ - U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */ + U32 windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ + U32 chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + U32 hashLog; /*< dispatch table : larger == faster, more memory */ + U32 searchLog; /*< nb of searches : larger == more compression, slower */ + U32 searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ + U32 targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ ZSTD_strategy strategy; } ZSTD_compressionParameters; typedef struct { - U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ - U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ - U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ + U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */ + U32 checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + U32 noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -217,15 +258,16 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v /*-************************************* -* Advanced functions +* Advanced compression functions ***************************************/ /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -/*! ZSTD_createDCtx_advanced() : - * Create a ZSTD decompression context using external alloc and free functions */ -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_createCDict_advanced() : + * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_parameters params, ZSTD_customMem customMem); ZSTDLIB_API unsigned ZSTD_maxCLevel (void); @@ -251,27 +293,11 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params); -/*! ZSTD_compress_usingPreparedDCtx() : -* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). -* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ -ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( - ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - /*- Advanced Decompression functions -*/ -/*! ZSTD_decompress_usingPreparedDCtx() : -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( - ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); /* ************************************** @@ -405,6 +431,31 @@ ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); +/*-*********************** +* Deprecated API +*************************/ +/*! ZSTD_compress_usingPreparedCCtx() : +* *** Warning : this function will soon be deprecated ! *** +* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). +* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ +ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( + ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! ZSTD_decompress_usingPreparedDCtx() : +* *** Warning : this function will soon be deprecated ! *** +* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ +ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( + ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9c06f603c..e57f38d90 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2385,7 +2385,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, /*! ZSTD_compressBegin_advanced() : * @return : 0, or an error code */ -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { @@ -2393,17 +2393,17 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); if (ZSTD_isError(errorCode)) return errorCode; } - return ZSTD_compressBegin_internal(zc, dict, dictSize, params, pledgedSrcSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); } -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); - return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); + ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", cctx->base, compressionLevel); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); } @@ -2532,6 +2532,81 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS } +/* ===== Dictionary API ===== */ + +struct ZSTD_CDict_s { + void* dictContent; + size_t dictContentSize; + ZSTD_CCtx* refContext; +}; /* typedef'd tp ZSTD_CDict within zstd.h */ + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + { ZSTD_CDict* const cdict = (ZSTD_CDict*) customMem.customAlloc(customMem.opaque, sizeof(*cdict)); + void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); + + if (!dictContent || !cdict || !cctx) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, cdict); + customMem.customFree(customMem.opaque, cctx); + return NULL; + } + + memcpy(dictContent, dict, dictSize); + { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); + if (ZSTD_isError(errorCode)) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, cdict); + customMem.customFree(customMem.opaque, cctx); + return NULL; + } } + + cdict->dictContent = dictContent; + cdict->dictContentSize = dictSize; + cdict->refContext = cctx; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); + params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + params.fParams.contentSizeFlag = 1; + return ZSTD_createCDict_advanced(dict, dictSize, params, allocator); +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + ZSTD_freeFunction const cFree = cdict->refContext->customMem.customFree; + void* const opaque = cdict->refContext->customMem.opaque; + ZSTD_freeCCtx(cdict->refContext); + cFree(opaque, cdict->dictContent); + cFree(opaque, cdict); + return 0; +} + +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + return ZSTD_compress_usingPreparedCCtx(cctx, cdict->refContext, + dst, dstCapacity, + src, srcSize); +} + + + /*-===== Pre-defined compression levels =====-*/ #define ZSTD_DEFAULT_CLEVEL 1 diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 9b7b1886b..344bd0201 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1267,3 +1267,77 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t return 0; } + + +struct ZSTD_DDict_s { + void* dictContent; + size_t dictContentSize; + ZSTD_DCtx* refContext; +}; /* typedef'd tp ZSTD_CDict within zstd.h */ + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict)); + void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); + + if (!dictContent || !ddict || !dctx) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } + + memcpy(dictContent, dict, dictSize); + { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); + if (ZSTD_isError(errorCode)) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } } + + ddict->dictContent = dictContent; + ddict->dictContentSize = dictSize; + ddict->refContext = dctx; + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, allocator); +} + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree; + void* const opaque = ddict->refContext->customMem.opaque; + ZSTD_freeDCtx(ddict->refContext); + cFree(opaque, ddict->dictContent); + cFree(opaque, ddict); + return 0; +} + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* In contrast with older ZSTD_decompress_usingDict(), use dictionary without significant overhead. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, + dst, dstCapacity, + src, srcSize); +} From ee1a084852aba12b370a4214b72ccf0ca459e45f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jun 2016 01:40:49 +0200 Subject: [PATCH 154/208] Integrated new dictionary API into bench module --- programs/bench.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index ecc609d83..51d47bde1 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -216,24 +216,21 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_getTime(&clockStart); { U32 nbLoops = 0; + ZSTD_CDict* cdict = ZSTD_createCDict(dictBuffer, dictBufferSize, cLevel); + if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict() allocation failure"); do { U32 blockNb; - { ZSTD_parameters params; - memset(¶ms, 0, sizeof(params)); - params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - params.fParams.contentSizeFlag = 1; - { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); - if (ZSTD_isError(initResult)) break; - } } for (blockNb=0; blockNb Date: Tue, 7 Jun 2016 11:36:13 +0200 Subject: [PATCH 155/208] Added build commands for various versions of Visual Studio - For people who want to build the library without opening it in VS --- .gitignore | 1 + build/build.VS2010.cmd | 7 +++++++ build/build.VS2012.cmd | 6 ++++++ build/build.VS2013.cmd | 7 +++++++ build/build.VS2015.cmd | 7 +++++++ build/build.generic.cmd | 38 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+) create mode 100644 build/build.VS2010.cmd create mode 100644 build/build.VS2012.cmd create mode 100644 build/build.VS2013.cmd create mode 100644 build/build.VS2015.cmd create mode 100644 build/build.generic.cmd diff --git a/.gitignore b/.gitignore index 9a71d4bcd..a06c2afa2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ projects/VS2010 projects/VS2012 projects/VS2013 projects/VS2015 +build/bin # IDEA solution files *.idea diff --git a/build/build.VS2010.cmd b/build/build.VS2010.cmd new file mode 100644 index 000000000..ddaeb4eee --- /dev/null +++ b/build/build.VS2010.cmd @@ -0,0 +1,7 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" Win32 v100 + +rem build 64-bit +call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" x64 v100 \ No newline at end of file diff --git a/build/build.VS2012.cmd b/build/build.VS2012.cmd new file mode 100644 index 000000000..96431e113 --- /dev/null +++ b/build/build.VS2012.cmd @@ -0,0 +1,6 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" Win32 v110 +rem build 64-bit +call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" x64 v110 \ No newline at end of file diff --git a/build/build.VS2013.cmd b/build/build.VS2013.cmd new file mode 100644 index 000000000..06aed3e4f --- /dev/null +++ b/build/build.VS2013.cmd @@ -0,0 +1,7 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" Win32 v120 + +rem build 64-bit +call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" x64 v120 \ No newline at end of file diff --git a/build/build.VS2015.cmd b/build/build.VS2015.cmd new file mode 100644 index 000000000..0e2484949 --- /dev/null +++ b/build/build.VS2015.cmd @@ -0,0 +1,7 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" Win32 v140 + +rem build 64-bit +call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" x64 v140 \ No newline at end of file diff --git a/build/build.generic.cmd b/build/build.generic.cmd new file mode 100644 index 000000000..289776bf9 --- /dev/null +++ b/build/build.generic.cmd @@ -0,0 +1,38 @@ +@echo off + +rem +rem Parameters +rem %1: vs_solution_version: VS solution version: VS2012, VS2013, VS2015 +rem %2: vs_configuration: VS configuration: Debug, Release +rem %3: vs_target: target: Build or Rebuild +rem %4: vs_platform: platform: x64 or Win32 +rem %5: vs_toolset: toolset: v100, v110, v120, v140 +rem + +set vs_solution_version=%1 +set vs_configuration=%2 +set vs_target=%3 +set vs_platform=%4 +set vs_toolset=%5 + +set msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe" +IF %vs_solution_version% == VS2013 SET msbuild="C:\Program Files (x86)\MSBuild\12.0\Bin\MSBuild.exe" +IF %vs_solution_version% == VS2015 SET msbuild="C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" +rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ? + +set project="%~p0\..\projects\VS2010\zstd.sln" + +set output=%~p0%bin +set output="%output%/%vs_solution_version%/%vs_platform%/" + +set msbuildparams=/verbosity:minimal /p:Platform="%vs_platform%" /p:Configuration="%vs_configuration%" /p:PlatformToolset="%vs_toolset%" /t:Clean,Build /nologo /p:OutDir=%output% + +echo ### Building %vs_solution_version% project for %vs_configuration% %vs_platform% (%vs_toolset%)... +echo ### Build Params: %msbuildparams% + +%msbuild% %project% %msbuildparams% +IF ERRORLEVEL 1 EXIT /B 1 +echo # Success +echo # OutDir: %output% +echo # + From 302fb53a762a90fb8d1fa97e066bb30c78739c09 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jun 2016 12:16:49 +0200 Subject: [PATCH 156/208] Removed `ZSTD_*_usingPrepared?Ctx()` declaration from public space --- lib/common/zstd.h | 104 ++++++++++++------------------- lib/compress/zstd_compress.c | 7 ++- lib/decompress/zstd_decompress.c | 5 ++ 3 files changed, 50 insertions(+), 66 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index ea0840b69..3c1e16a7b 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -126,46 +126,10 @@ ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /*!< @return : error ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*-*********************** -* Dictionary API -*************************/ -/*! ZSTD_createCDict() : -* Create a digested dictionary, ready to start compression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTD_CDict_s ZSTD_CDict; -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); - -/*! ZSTD_compress_usingCDict() : -* Compression using a pre-digested Dictionary -* In contrast with older ZSTD_compress_usingDict(), use dictionary without significant overhead. -* Note that compression level is decided during dictionary creation */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); - -/*! ZSTD_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTD_DDict_s ZSTD_DDict; -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); - -/*! ZSTD_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* In contrast with older ZSTD_decompress_usingDict(), use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); - - -/*-*********************** -* Deprecated API -*************************/ +/*-************************ +* Simple dictionary API +***************************/ /*! ZSTD_compress_usingDict() : -* *** Warning : this function will soon be declared deprecated *** * Compression using a pre-defined Dictionary content (see dictBuilder). * Note 1 : This function load the dictionary, resulting in a significant startup time. * Note 2 : `dict` must remain valid and unmodified during compression operation. @@ -177,7 +141,6 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, int compressionLevel); /*! ZSTD_decompress_usingDict() : -* *** Warning : this function will soon be declared deprecated *** * Decompression using a pre-defined Dictionary content (see dictBuilder). * Dictionary must be identical to the one used during compression. * Note 1 : This function load the dictionary, resulting in a significant startup time @@ -189,6 +152,42 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict,size_t dictSize); +/*-************************** +* Advanced Dictionary API +****************************/ +/*! ZSTD_createCDict() : +* Create a digested dictionary, ready to start compression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_CDict_s ZSTD_CDict; +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : +* Compression using a pre-digested Dictionary. +* Much faster than ZSTD_compress_usingDict() when same dictionary is used multiple times. +* Note that compression level is decided during dictionary creation */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_DDict_s ZSTD_DDict; +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Much faster than ZSTD_decompress_usingDict() when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + + #ifdef ZSTD_STATIC_LINKING_ONLY /* ==================================================================================== @@ -431,31 +430,6 @@ ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); -/*-*********************** -* Deprecated API -*************************/ -/*! ZSTD_compress_usingPreparedCCtx() : -* *** Warning : this function will soon be deprecated ! *** -* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). -* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ -ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( - ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! ZSTD_decompress_usingPreparedDCtx() : -* *** Warning : this function will soon be deprecated ! *** -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( - ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - #endif /* ZSTD_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e57f38d90..9a94cd97c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2449,7 +2449,12 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } -size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, +/*! ZSTD_compress_usingPreparedCCtx() : +* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). +* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ +static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 344bd0201..f0666e3de 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1012,6 +1012,11 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } +/*! ZSTD_decompress_usingPreparedDCtx() : +* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) From 0faabe43f8d43387a9ab6d005243d8d0f0ea86c8 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Tue, 7 Jun 2016 14:00:02 +0200 Subject: [PATCH 157/208] Use Rebuild target instead of Clean+Build --- build/build.VS2010.cmd | 4 ++-- build/build.VS2012.cmd | 4 ++-- build/build.VS2013.cmd | 4 ++-- build/build.VS2015.cmd | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/build.VS2010.cmd b/build/build.VS2010.cmd index ddaeb4eee..19c6a52ba 100644 --- a/build/build.VS2010.cmd +++ b/build/build.VS2010.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" Win32 v100 +call "%~p0%build.generic.cmd" VS2010 Release Rebuild Win32 v100 rem build 64-bit -call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" x64 v100 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2010 Release Rebuild x64 v100 \ No newline at end of file diff --git a/build/build.VS2012.cmd b/build/build.VS2012.cmd index 96431e113..66439c907 100644 --- a/build/build.VS2012.cmd +++ b/build/build.VS2012.cmd @@ -1,6 +1,6 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" Win32 v110 +call "%~p0%build.generic.cmd" VS2012 Release Rebuild Win32 v110 rem build 64-bit -call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" x64 v110 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2012 Release Rebuild x64 v110 \ No newline at end of file diff --git a/build/build.VS2013.cmd b/build/build.VS2013.cmd index 06aed3e4f..220b26b63 100644 --- a/build/build.VS2013.cmd +++ b/build/build.VS2013.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" Win32 v120 +call "%~p0%build.generic.cmd" VS2013 Release Rebuild Win32 v120 rem build 64-bit -call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" x64 v120 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2013 Release Rebuild x64 v120 \ No newline at end of file diff --git a/build/build.VS2015.cmd b/build/build.VS2015.cmd index 0e2484949..b5784340a 100644 --- a/build/build.VS2015.cmd +++ b/build/build.VS2015.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" Win32 v140 +call "%~p0%build.generic.cmd" VS2015 Release Rebuild Win32 v140 rem build 64-bit -call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" x64 v140 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2015 Release Rebuild x64 v140 \ No newline at end of file From f1510489bdbb7b0c2df5f94c806721c8adb4b65f Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Tue, 7 Jun 2016 14:32:05 +0200 Subject: [PATCH 158/208] Simplified and reordered the arguments to build.generic.cmd and added default values - always Clean,Rebuild - output folder is now build/bin/{CONFIGURATION}/{ARCH} --- build/build.VS2010.cmd | 4 +-- build/build.VS2012.cmd | 4 +-- build/build.VS2013.cmd | 4 +-- build/build.VS2015.cmd | 4 +-- build/build.generic.cmd | 55 +++++++++++++++++++++++++---------------- 5 files changed, 42 insertions(+), 29 deletions(-) diff --git a/build/build.VS2010.cmd b/build/build.VS2010.cmd index 19c6a52ba..c3bc1763e 100644 --- a/build/build.VS2010.cmd +++ b/build/build.VS2010.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2010 Release Rebuild Win32 v100 +call "%~p0%build.generic.cmd" VS2010 Win32 Release v100 rem build 64-bit -call "%~p0%build.generic.cmd" VS2010 Release Rebuild x64 v100 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2010 x64 Release v100 \ No newline at end of file diff --git a/build/build.VS2012.cmd b/build/build.VS2012.cmd index 66439c907..d7399a9d6 100644 --- a/build/build.VS2012.cmd +++ b/build/build.VS2012.cmd @@ -1,6 +1,6 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2012 Release Rebuild Win32 v110 +call "%~p0%build.generic.cmd" VS2012 Win32 Release v110 rem build 64-bit -call "%~p0%build.generic.cmd" VS2012 Release Rebuild x64 v110 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2012 x64 Release v110 \ No newline at end of file diff --git a/build/build.VS2013.cmd b/build/build.VS2013.cmd index 220b26b63..486ba6cf0 100644 --- a/build/build.VS2013.cmd +++ b/build/build.VS2013.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2013 Release Rebuild Win32 v120 +call "%~p0%build.generic.cmd" VS2013 Win32 Release v120 rem build 64-bit -call "%~p0%build.generic.cmd" VS2013 Release Rebuild x64 v120 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2013 x64 Release v120 \ No newline at end of file diff --git a/build/build.VS2015.cmd b/build/build.VS2015.cmd index b5784340a..abc41c9a0 100644 --- a/build/build.VS2015.cmd +++ b/build/build.VS2015.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2015 Release Rebuild Win32 v140 +call "%~p0%build.generic.cmd" VS2015 Win32 Release v140 rem build 64-bit -call "%~p0%build.generic.cmd" VS2015 Release Rebuild x64 v140 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2015 x64 Release v140 \ No newline at end of file diff --git a/build/build.generic.cmd b/build/build.generic.cmd index 289776bf9..502c8db95 100644 --- a/build/build.generic.cmd +++ b/build/build.generic.cmd @@ -1,33 +1,46 @@ @echo off -rem -rem Parameters -rem %1: vs_solution_version: VS solution version: VS2012, VS2013, VS2015 -rem %2: vs_configuration: VS configuration: Debug, Release -rem %3: vs_target: target: Build or Rebuild -rem %4: vs_platform: platform: x64 or Win32 -rem %5: vs_toolset: toolset: v100, v110, v120, v140 -rem +IF "%1%" == "" GOTO display_help -set vs_solution_version=%1 -set vs_configuration=%2 -set vs_target=%3 -set vs_platform=%4 -set vs_toolset=%5 +SET vs_version=%1 -set msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe" -IF %vs_solution_version% == VS2013 SET msbuild="C:\Program Files (x86)\MSBuild\12.0\Bin\MSBuild.exe" -IF %vs_solution_version% == VS2015 SET msbuild="C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" +SET vs_platform=%2 +IF "%vs_platform%" == "" SET vs_platform=x64 + +SET vs_configuration=%3 +IF "%vs_configuration%" == "" SET vs_configuration=Release + +SET vs_toolset=%4 + +GOTO build + +:display_help + +echo Syntax: build.generic.cmd vs_version vs_platform vs_configuration vs_toolset +echo vs_version: VS installed version (VS2012, VS2013, VS2015, ...) +echo vs_platform: Platform (x64 or Win32) +echo vs_configuration: VS configuration (Release or Debug) +echo vs_toolset: Platform Toolset (v100, v110, v120, v140) + +EXIT /B 1 + +:build + +SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe" +IF %vs_version% == VS2013 SET msbuild="C:\Program Files (x86)\MSBuild\12.0\Bin\MSBuild.exe" +IF %vs_version% == VS2015 SET msbuild="C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ? -set project="%~p0\..\projects\VS2010\zstd.sln" +SET project="%~p0\..\projects\VS2010\zstd.sln" -set output=%~p0%bin -set output="%output%/%vs_solution_version%/%vs_platform%/" +SET msbuildparams=/verbosity:minimal /nologo /t:Clean,Build /p:Platform=%vs_platform% /p:Configuration=%vs_configuration% +IF NOT "%vs_toolset%" == "" SET msbuildparams=%msbuildparams% /p:PlatformToolset=%vs_toolset% -set msbuildparams=/verbosity:minimal /p:Platform="%vs_platform%" /p:Configuration="%vs_configuration%" /p:PlatformToolset="%vs_toolset%" /t:Clean,Build /nologo /p:OutDir=%output% +SET output=%~p0%bin +SET output="%output%/%vs_configuration%/%vs_platform%/" +SET msbuildparams=%msbuildparams% /p:OutDir=%output% -echo ### Building %vs_solution_version% project for %vs_configuration% %vs_platform% (%vs_toolset%)... +echo ### Building %vs_version% project for %vs_configuration% %vs_platform% (%vs_toolset%)... echo ### Build Params: %msbuildparams% %msbuild% %project% %msbuildparams% From 662a541431c19ef4997113e3820b735c45ad529c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Jun 2016 11:11:02 +0200 Subject: [PATCH 159/208] updated huff0 - now generates a common HUF_DTable type for all decoding tables --- NEWS | 1 + lib/common/huf.h | 85 ++--- lib/decompress/huf_decompress.c | 592 ++++++------------------------- lib/decompress/zstd_decompress.c | 8 +- 4 files changed, 162 insertions(+), 524 deletions(-) diff --git a/NEWS b/NEWS index e54b09967..767fe5c01 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v0.7.0 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Visual build scripts, by Christophe Chevalier New : Support for Sparse File-systems (do not use space for zero-filled sectors) New : Frame checksum support New : Support pass-through mode (when using `-df`) diff --git a/lib/common/huf.h b/lib/common/huf.h index d54e6616f..c76694576 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -31,8 +31,8 @@ You can contact the author at : - Source repository : https://github.com/Cyan4973/FiniteStateEntropy ****************************************************************** */ -#ifndef HUF_H -#define HUF_H +#ifndef HUF_H_298734234 +#define HUF_H_298734234 #if defined (__cplusplus) extern "C" { @@ -53,8 +53,9 @@ size_t HUF_decompress(void* dst, size_t dstSize, /* HUF_compress() : Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - Note : `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB + 'dst' buffer must be already allocated. + Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB @return : size of compressed data (<= `dstCapacity`) Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! if return == 1, srcData is a single repeated byte symbol (RLE compression). @@ -63,7 +64,7 @@ HUF_compress() : HUF_decompress() : Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', into already allocated buffer 'dst', of minimum size 'dstSize'. - `dstSize` : must be the **exact** size of original (uncompressed) data. + `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate. @@ -121,13 +122,12 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ /* static allocation of HUF's DTable */ -#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<= 64 */ /* **************************************** @@ -161,35 +160,6 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, un size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); -/*! -HUF_decompress() does the following: -1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics -2. build Huffman table from save, using HUF_readDTableXn() -3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -*/ -size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize); - -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - -/* single stream variants */ - -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ - -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - /*! HUF_readStats() : Read compact Huffman tree, saved by HUF_writeCTable(). `huffWeight` is destination buffer. @@ -204,6 +174,39 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); +/* +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + #endif /* HUF_STATIC_LINKING_ONLY */ @@ -211,4 +214,4 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* sr } #endif -#endif /* HUF_H */ +#endif /* HUF_H_298734234 */ diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index d1096e457..ed235f5e7 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -60,37 +60,26 @@ * Includes ****************************************************************/ #include /* memcpy, memset */ -#include /* printf (debug) */ #include "bitstream.h" #include "fse.h" /* header compression */ #define HUF_STATIC_LINKING_ONLY #include "huf.h" - /* ************************************************************** * Error Management ****************************************************************/ #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - -/* ******************************************************* -* HUF : Huffman block decompression -*********************************************************/ -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ - -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ - -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; - - - /*-***************************/ /* single-symbol decoding */ /*-***************************/ +typedef struct { BYTE maxTableLog; BYTE currentTableLog; } DTableDesc; -size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) { BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ @@ -101,16 +90,19 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) U32 nextRankStart; void* const dtPtr = DTable + 1; HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + DTableDesc dtd; - HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ + HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ + memcpy(&dtd, DTable, sizeof(dtd)); //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ - if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */ - DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */ + if (tableLog > dtd.maxTableLog) return ERROR(tableLog_tooLarge); /* DTable is too small */ + dtd.currentTableLog = (BYTE)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */ + memcpy(DTable, &dtd, sizeof(dtd)); /* Prepare ranks */ nextRankStart = 0; @@ -181,14 +173,18 @@ static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, B size_t HUF_decompress1X2_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U16* DTable) + const HUF_DTable* DTable) { BYTE* op = (BYTE*)dst; BYTE* const oend = op + dstSize; - const U32 dtLog = DTable[0]; const void* dtPtr = DTable; const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1; BIT_DStream_t bitD; + DTableDesc dtd; + U32 dtLog; + + memcpy(&dtd, DTable, sizeof(dtd)); + dtLog = dtd.currentTableLog; { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); if (HUF_isError(errorCode)) return errorCode; } @@ -219,7 +215,7 @@ size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS size_t HUF_decompress4X2_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U16* DTable) + const HUF_DTable* DTable) { /* Check */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ @@ -229,18 +225,16 @@ size_t HUF_decompress4X2_usingDTable( BYTE* const oend = ostart + dstSize; const void* const dtPtr = DTable; const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1; - const U32 dtLog = DTable[0]; - size_t errorCode; /* Init */ BIT_DStream_t bitD1; BIT_DStream_t bitD2; BIT_DStream_t bitD3; BIT_DStream_t bitD4; - const size_t length1 = MEM_readLE16(istart); - const size_t length2 = MEM_readLE16(istart+2); - const size_t length3 = MEM_readLE16(istart+4); - size_t length4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE* const istart2 = istart1 + length1; const BYTE* const istart3 = istart2 + length2; @@ -254,17 +248,21 @@ size_t HUF_decompress4X2_usingDTable( BYTE* op3 = opStart3; BYTE* op4 = opStart4; U32 endSignal; + DTableDesc dtd; + U32 dtLog; + + memcpy(&dtd, DTable, sizeof(dtd)); + dtLog = dtd.currentTableLog; - length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } /* 16-32 symbols per loop (4-8 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); @@ -315,11 +313,11 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; - size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); - ip += errorCode; - cSrcSize -= errorCode; + size_t const hSize = HUF_readDTableX2 (DTable, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; return HUF_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable); } @@ -328,6 +326,9 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS /* *************************/ /* double-symbols decoding */ /* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, const U32* rankValOrigin, const int minWeight, @@ -413,7 +414,7 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, } } -size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) { BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; @@ -422,20 +423,23 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) U32* const rankStart = rankStart0+1; rankVal_t rankVal; U32 tableLog, maxW, sizeOfSort, nbSymbols; - const U32 memLog = DTable[0]; + DTableDesc dtd; + U32 maxTableLog; size_t iSize; - void* dtPtr = DTable; - HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ - if (memLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); + memcpy(&dtd, DTable, sizeof(dtd)); + maxTableLog = dtd.maxTableLog-1; + if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ - if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ /* find maxWeight */ for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ @@ -464,7 +468,7 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) /* Build rankVal */ { U32* const rankVal0 = rankVal[0]; - { int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ U32 nextRankVal = 0; U32 w; for (w=1; w> consumed; } } } } - HUF_fillDTableX4(dt, memLog, + HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog+1); + dtd.currentTableLog = (BYTE)maxTableLog; + memcpy(DTable, &dtd, sizeof(dtd)); return iSize; } @@ -536,7 +542,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c HUF_DECODE_SYMBOLX4_0(p, bitDPtr); } - /* closer to the end */ + /* closer to end : up to 2 symbols at a time */ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) HUF_DECODE_SYMBOLX4_0(p, bitDPtr); @@ -553,23 +559,24 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c size_t HUF_decompress1X4_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U32* DTable) + const HUF_DTable* DTable) { - const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - - const U32 dtLog = DTable[0]; - const void* const dtPtr = DTable; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; + BIT_DStream_t bitD; /* Init */ - BIT_DStream_t bitD; - { size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + } /* decode */ - HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtLog); + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + DTableDesc dtd; + memcpy(&dtd, DTable, sizeof(dtd)); + HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.currentTableLog); + } /* check */ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); @@ -595,32 +602,30 @@ size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cS size_t HUF_decompress4X4_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U32* DTable) + const HUF_DTable* DTable) { if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; - const U32 dtLog = DTable[0]; - size_t errorCode; + const void* const dtPtr = DTable+1; + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; /* Init */ BIT_DStream_t bitD1; BIT_DStream_t bitD2; BIT_DStream_t bitD3; BIT_DStream_t bitD4; - const size_t length1 = MEM_readLE16(istart); - const size_t length2 = MEM_readLE16(istart+2); - const size_t length3 = MEM_readLE16(istart+4); - size_t length4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE* const istart2 = istart1 + length1; const BYTE* const istart3 = istart2 + length2; const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; + size_t const segmentSize = (dstSize+3) / 4; BYTE* const opStart2 = ostart + segmentSize; BYTE* const opStart3 = opStart2 + segmentSize; BYTE* const opStart4 = opStart3 + segmentSize; @@ -629,17 +634,21 @@ size_t HUF_decompress4X4_usingDTable( BYTE* op3 = opStart3; BYTE* op4 = opStart4; U32 endSignal; + DTableDesc dtd; + U32 dtLog; + + memcpy(&dtd, DTable, sizeof(dtd)); + dtLog = dtd.currentTableLog; - length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } /* 16-32 symbols per loop (4-8 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); @@ -677,8 +686,8 @@ size_t HUF_decompress4X4_usingDTable( HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } /* decoded size */ return dstSize; @@ -701,386 +710,6 @@ size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cS } -/* ********************************/ -/* quad-symbol decoding */ -/* ********************************/ -typedef struct { BYTE nbBits; BYTE nbBytes; } HUF_DDescX6; -typedef union { BYTE byte[4]; U32 sequence; } HUF_DSeqX6; - -/* recursive, up to level 3; may benefit from