From 60038948e6a87171f868daef1f9483871df07b6b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Sep 2016 18:52:52 +0200 Subject: [PATCH 01/16] added -- command in help --- programs/zstd.1 | 4 ++++ programs/zstdcli.c | 1 + 2 files changed, 5 insertions(+) diff --git a/programs/zstd.1 b/programs/zstd.1 index a23616529..c262a0c6a 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -89,6 +89,10 @@ It also features a very fast decoder, with speed > 500 MB/s per core. .BR \-t ", " --test Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR. No files are created or removed. +.TP +.BR -- + All arguments after -- are treated as files + .SH DICTIONARY .PP diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 4a58b05b8..14571344f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -144,6 +144,7 @@ static int usage_advanced(const char* programName) DISPLAY( "--test : test compressed file integrity \n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #endif + DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); From 88aa179347776d0f3cd492fcc9db5d429636d228 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 18 Sep 2016 11:58:30 +0200 Subject: [PATCH 02/16] added comments on buffer sizes guarantees --- examples/streaming_compression.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index 4e87130de..bc81af1a3 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -65,9 +65,9 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve { FILE* const fin = fopen_orDie(fname, "rb"); FILE* const fout = fopen_orDie(outName, "wb"); - size_t const buffInSize = ZSTD_CStreamInSize();; + size_t const buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */ void* const buffIn = malloc_orDie(buffInSize); - size_t const buffOutSize = ZSTD_CStreamOutSize();; + size_t const buffOutSize = ZSTD_CStreamOutSize(); /* can always flush a full block */ void* const buffOut = malloc_orDie(buffOutSize); ZSTD_CStream* const cstream = ZSTD_createCStream(); @@ -80,7 +80,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_inBuffer input = { buffIn, read, 0 }; while (input.pos < input.size) { ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; - toRead = ZSTD_compressStream(cstream, &output , &input); + toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ fwrite_orDie(buffOut, output.pos, fout); } } From 4ca3d4bc252177aa99fa9fedacb33c0c4aee63fc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 18 Sep 2016 12:17:51 +0200 Subject: [PATCH 03/16] streaming compression example can handle situations where input buffer size is manually set to a small value. --- examples/streaming_compression.c | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index bc81af1a3..d663a4914 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -81,6 +81,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve while (input.pos < input.size) { ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ + if (toRead > buffInSize) toRead = buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */ fwrite_orDie(buffOut, output.pos, fout); } } From 1eb2fdc74f893d2264b944829175a3b193fb1128 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 18 Sep 2016 12:21:47 +0200 Subject: [PATCH 04/16] bumped version number --- NEWS | 4 ++-- lib/zstd.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index ace6d4826..a2d77f020 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ -v1.0.1 +v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell added : NetBSD install target (#338) -Improved : variable compression speed improvements on batches of small files. +Improved : speed improvements for batches of small files. Fixed : CLI -d output to stdout by default when input is stdin (#322) Fixed : CLI correctly detects console on Mac OS-X Fixed : CLI supports recursive mode `-r` on Mac OS-X diff --git a/lib/zstd.h b/lib/zstd.h index f79a5dcac..31171d04d 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -32,8 +32,8 @@ extern "C" { /*======= Version =======*/ #define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 0 -#define ZSTD_VERSION_RELEASE 1 +#define ZSTD_VERSION_MINOR 1 +#define ZSTD_VERSION_RELEASE 0 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str From 4c9a4c18a9572b0abf4ca6e4839f966cb5caa828 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 19 Sep 2016 14:58:14 +0200 Subject: [PATCH 05/16] changed projects to build --- Makefile | 11 +-- README.md | 28 +++++++ appveyor.yml | 74 +++++++++--------- {projects => build}/.gitignore | 0 {projects => build}/README.md | 2 +- .../VS2005/fullbench/fullbench.vcproj | 0 .../VS2005/fuzzer/fuzzer.vcproj | 0 {projects => build}/VS2005/zstd.sln | 0 {projects => build}/VS2005/zstd/zstd.vcproj | 0 .../VS2005/zstdlib/zstdlib.vcproj | 0 .../VS2008/fullbench/fullbench.vcproj | 0 .../VS2008/fuzzer/fuzzer.vcproj | 0 {projects => build}/VS2008/zstd.sln | 0 {projects => build}/VS2008/zstd/zstd.vcproj | 0 .../VS2008/zstdlib/zstdlib.vcproj | 0 {projects => build}/VS2010/CompileAsCpp.props | 0 .../VS2010/datagen/datagen.vcxproj | 0 .../VS2010/fullbench/fullbench.vcxproj | 0 .../VS2010/fuzzer/fuzzer.vcxproj | 0 {projects => build}/VS2010/zstd.sln | 0 .../VS2010/zstd/generate_res/generate_res.bat | 0 .../VS2010/zstd/generate_res/verrsrc.h | 0 .../VS2010/zstd/generate_res/zstd32.res | Bin .../VS2010/zstd/generate_res/zstd64.res | Bin {projects => build}/VS2010/zstd/zstd.rc | 0 {projects => build}/VS2010/zstd/zstd.vcxproj | 0 {projects => build}/VS2010/zstdlib/zstdlib.rc | 0 .../VS2010/zstdlib/zstdlib.vcxproj | 0 .../build => build/VS_scripts}/README.md | 0 .../VS_scripts}/build.VS2010.cmd | 0 .../VS_scripts}/build.VS2012.cmd | 0 .../VS_scripts}/build.VS2013.cmd | 0 .../VS_scripts}/build.VS2015.cmd | 0 .../VS_scripts}/build.generic.cmd | 0 {projects => build}/cmake/.gitignore | 0 {projects => build}/cmake/CMakeLists.txt | 0 .../AddExtraCompilationFlags.cmake | 0 .../cmake/cmake_uninstall.cmake.in | 0 {projects => build}/cmake/lib/CMakeLists.txt | 0 {projects => build}/cmake/programs/.gitignore | 0 .../cmake/programs/CMakeLists.txt | 0 {projects => build}/cmake/tests/.gitignore | 0 .../cmake/tests/CMakeLists.txt | 0 43 files changed, 72 insertions(+), 43 deletions(-) rename {projects => build}/.gitignore (100%) rename {projects => build}/README.md (92%) rename {projects => build}/VS2005/fullbench/fullbench.vcproj (100%) rename {projects => build}/VS2005/fuzzer/fuzzer.vcproj (100%) rename {projects => build}/VS2005/zstd.sln (100%) rename {projects => build}/VS2005/zstd/zstd.vcproj (100%) rename {projects => build}/VS2005/zstdlib/zstdlib.vcproj (100%) rename {projects => build}/VS2008/fullbench/fullbench.vcproj (100%) rename {projects => build}/VS2008/fuzzer/fuzzer.vcproj (100%) rename {projects => build}/VS2008/zstd.sln (100%) rename {projects => build}/VS2008/zstd/zstd.vcproj (100%) rename {projects => build}/VS2008/zstdlib/zstdlib.vcproj (100%) rename {projects => build}/VS2010/CompileAsCpp.props (100%) rename {projects => build}/VS2010/datagen/datagen.vcxproj (100%) rename {projects => build}/VS2010/fullbench/fullbench.vcxproj (100%) rename {projects => build}/VS2010/fuzzer/fuzzer.vcxproj (100%) rename {projects => build}/VS2010/zstd.sln (100%) rename {projects => build}/VS2010/zstd/generate_res/generate_res.bat (100%) rename {projects => build}/VS2010/zstd/generate_res/verrsrc.h (100%) rename {projects => build}/VS2010/zstd/generate_res/zstd32.res (100%) rename {projects => build}/VS2010/zstd/generate_res/zstd64.res (100%) rename {projects => build}/VS2010/zstd/zstd.rc (100%) rename {projects => build}/VS2010/zstd/zstd.vcxproj (100%) rename {projects => build}/VS2010/zstdlib/zstdlib.rc (100%) rename {projects => build}/VS2010/zstdlib/zstdlib.vcxproj (100%) rename {projects/build => build/VS_scripts}/README.md (100%) rename {projects/build => build/VS_scripts}/build.VS2010.cmd (100%) rename {projects/build => build/VS_scripts}/build.VS2012.cmd (100%) rename {projects/build => build/VS_scripts}/build.VS2013.cmd (100%) rename {projects/build => build/VS_scripts}/build.VS2015.cmd (100%) rename {projects/build => build/VS_scripts}/build.generic.cmd (100%) rename {projects => build}/cmake/.gitignore (100%) rename {projects => build}/cmake/CMakeLists.txt (100%) rename {projects => build}/cmake/CMakeModules/AddExtraCompilationFlags.cmake (100%) rename {projects => build}/cmake/cmake_uninstall.cmake.in (100%) rename {projects => build}/cmake/lib/CMakeLists.txt (100%) rename {projects => build}/cmake/programs/.gitignore (100%) rename {projects => build}/cmake/programs/CMakeLists.txt (100%) rename {projects => build}/cmake/tests/.gitignore (100%) rename {projects => build}/cmake/tests/CMakeLists.txt (100%) diff --git a/Makefile b/Makefile index 7860ce1db..b50723e85 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,9 @@ # of patent rights can be found in the PATENTS file in the same directory. # ################################################################ -PRGDIR = programs -ZSTDDIR = lib +PRGDIR = programs +ZSTDDIR = lib +BUILDIR = build ZWRAPDIR = zlibWrapper TESTDIR = tests @@ -121,9 +122,9 @@ endif ifneq (,$(filter $(HOST_OS),MSYS POSIX)) cmaketest: cmake --version - $(RM) -r projects/cmake/build - mkdir projects/cmake/build - cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall + $(RM) -r $(BUILDDIR)/cmake/build + mkdir $(BUILDDIR)/cmake/build + cd $(BUILDDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall c90test: clean CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long diff --git a/README.md b/README.md index 2c8e707e9..85d5ac324 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,34 @@ Hence, deploying one dictionary per type of data will provide the greatest benef `zstd --decompress FILE.zst -D dictionaryName` +### Build + +Once you have the repository cloned, there are multiple ways provided to build Zstandard. + +#### Makefile + +If your system is compatible with `make`, you can simply run `make` at the root directory. +It will generate `zstd` within root directory. + +Other available options include : +- `make install` : create and install zstd binary, library and man page +- `make test` : create and run `zstd` and test tools on local platform + +#### cmake + +A `cmake` project generator is provided within `build/cmake`. +It can generate Makefiles or other build scripts +to create `zstd` binary, and `libzstd` dynamic and static libraries. + +#### Visual (Windows) + +Going into `build` directory, you will find additional possibilities : +- Projects for Visual Studio 2005, 2008 and 2010 + + VS2010 project is compatible with VS2012, VS2013 and VS2015 +- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`, + which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. + + ### Status Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amounts of data in multiple formats and use cases. diff --git a/appveyor.yml b/appveyor.yml index 280cbae86..8f4e45044 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -36,8 +36,8 @@ install: build_script: - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% - - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] ( make clean && ECHO *** && @@ -76,51 +76,51 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% && ECHO *** && - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && + msbuild "build\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + COPY build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && + msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe tests\ + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && + COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe tests\ ) test_script: @@ -144,7 +144,7 @@ test_script: artifacts: - path: bin\zstd.exe - - path: bin\zstd32.exe + - path: bin\zstd32.exe deploy: - provider: GitHub @@ -160,7 +160,7 @@ deploy: - provider: GitHub auth_token: secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 - artifact: bin\zstd32.exe + artifact: bin\zstd32.exe force_update: true on: branch: autobuild diff --git a/projects/.gitignore b/build/.gitignore similarity index 100% rename from projects/.gitignore rename to build/.gitignore diff --git a/projects/README.md b/build/README.md similarity index 92% rename from projects/README.md rename to build/README.md index dd60b56e8..8dc67326b 100644 --- a/projects/README.md +++ b/build/README.md @@ -8,7 +8,7 @@ The following projects are included with the zstd distribution: - `VS2005` - Visual Studio 2005 project - `VS2008` - Visual Studio 2008 project - `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) -- `build` - command line scripts prepared for Visual Studio compilation without IDE +- `VS_scripts` - command line scripts prepared for Visual Studio compilation without IDE #### How to compile zstd with Visual Studio diff --git a/projects/VS2005/fullbench/fullbench.vcproj b/build/VS2005/fullbench/fullbench.vcproj similarity index 100% rename from projects/VS2005/fullbench/fullbench.vcproj rename to build/VS2005/fullbench/fullbench.vcproj diff --git a/projects/VS2005/fuzzer/fuzzer.vcproj b/build/VS2005/fuzzer/fuzzer.vcproj similarity index 100% rename from projects/VS2005/fuzzer/fuzzer.vcproj rename to build/VS2005/fuzzer/fuzzer.vcproj diff --git a/projects/VS2005/zstd.sln b/build/VS2005/zstd.sln similarity index 100% rename from projects/VS2005/zstd.sln rename to build/VS2005/zstd.sln diff --git a/projects/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj similarity index 100% rename from projects/VS2005/zstd/zstd.vcproj rename to build/VS2005/zstd/zstd.vcproj diff --git a/projects/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj similarity index 100% rename from projects/VS2005/zstdlib/zstdlib.vcproj rename to build/VS2005/zstdlib/zstdlib.vcproj diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj similarity index 100% rename from projects/VS2008/fullbench/fullbench.vcproj rename to build/VS2008/fullbench/fullbench.vcproj diff --git a/projects/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj similarity index 100% rename from projects/VS2008/fuzzer/fuzzer.vcproj rename to build/VS2008/fuzzer/fuzzer.vcproj diff --git a/projects/VS2008/zstd.sln b/build/VS2008/zstd.sln similarity index 100% rename from projects/VS2008/zstd.sln rename to build/VS2008/zstd.sln diff --git a/projects/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj similarity index 100% rename from projects/VS2008/zstd/zstd.vcproj rename to build/VS2008/zstd/zstd.vcproj diff --git a/projects/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj similarity index 100% rename from projects/VS2008/zstdlib/zstdlib.vcproj rename to build/VS2008/zstdlib/zstdlib.vcproj diff --git a/projects/VS2010/CompileAsCpp.props b/build/VS2010/CompileAsCpp.props similarity index 100% rename from projects/VS2010/CompileAsCpp.props rename to build/VS2010/CompileAsCpp.props diff --git a/projects/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj similarity index 100% rename from projects/VS2010/datagen/datagen.vcxproj rename to build/VS2010/datagen/datagen.vcxproj diff --git a/projects/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj similarity index 100% rename from projects/VS2010/fullbench/fullbench.vcxproj rename to build/VS2010/fullbench/fullbench.vcxproj diff --git a/projects/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj similarity index 100% rename from projects/VS2010/fuzzer/fuzzer.vcxproj rename to build/VS2010/fuzzer/fuzzer.vcxproj diff --git a/projects/VS2010/zstd.sln b/build/VS2010/zstd.sln similarity index 100% rename from projects/VS2010/zstd.sln rename to build/VS2010/zstd.sln diff --git a/projects/VS2010/zstd/generate_res/generate_res.bat b/build/VS2010/zstd/generate_res/generate_res.bat similarity index 100% rename from projects/VS2010/zstd/generate_res/generate_res.bat rename to build/VS2010/zstd/generate_res/generate_res.bat diff --git a/projects/VS2010/zstd/generate_res/verrsrc.h b/build/VS2010/zstd/generate_res/verrsrc.h similarity index 100% rename from projects/VS2010/zstd/generate_res/verrsrc.h rename to build/VS2010/zstd/generate_res/verrsrc.h diff --git a/projects/VS2010/zstd/generate_res/zstd32.res b/build/VS2010/zstd/generate_res/zstd32.res similarity index 100% rename from projects/VS2010/zstd/generate_res/zstd32.res rename to build/VS2010/zstd/generate_res/zstd32.res diff --git a/projects/VS2010/zstd/generate_res/zstd64.res b/build/VS2010/zstd/generate_res/zstd64.res similarity index 100% rename from projects/VS2010/zstd/generate_res/zstd64.res rename to build/VS2010/zstd/generate_res/zstd64.res diff --git a/projects/VS2010/zstd/zstd.rc b/build/VS2010/zstd/zstd.rc similarity index 100% rename from projects/VS2010/zstd/zstd.rc rename to build/VS2010/zstd/zstd.rc diff --git a/projects/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj similarity index 100% rename from projects/VS2010/zstd/zstd.vcxproj rename to build/VS2010/zstd/zstd.vcxproj diff --git a/projects/VS2010/zstdlib/zstdlib.rc b/build/VS2010/zstdlib/zstdlib.rc similarity index 100% rename from projects/VS2010/zstdlib/zstdlib.rc rename to build/VS2010/zstdlib/zstdlib.rc diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/build/VS2010/zstdlib/zstdlib.vcxproj similarity index 100% rename from projects/VS2010/zstdlib/zstdlib.vcxproj rename to build/VS2010/zstdlib/zstdlib.vcxproj diff --git a/projects/build/README.md b/build/VS_scripts/README.md similarity index 100% rename from projects/build/README.md rename to build/VS_scripts/README.md diff --git a/projects/build/build.VS2010.cmd b/build/VS_scripts/build.VS2010.cmd similarity index 100% rename from projects/build/build.VS2010.cmd rename to build/VS_scripts/build.VS2010.cmd diff --git a/projects/build/build.VS2012.cmd b/build/VS_scripts/build.VS2012.cmd similarity index 100% rename from projects/build/build.VS2012.cmd rename to build/VS_scripts/build.VS2012.cmd diff --git a/projects/build/build.VS2013.cmd b/build/VS_scripts/build.VS2013.cmd similarity index 100% rename from projects/build/build.VS2013.cmd rename to build/VS_scripts/build.VS2013.cmd diff --git a/projects/build/build.VS2015.cmd b/build/VS_scripts/build.VS2015.cmd similarity index 100% rename from projects/build/build.VS2015.cmd rename to build/VS_scripts/build.VS2015.cmd diff --git a/projects/build/build.generic.cmd b/build/VS_scripts/build.generic.cmd similarity index 100% rename from projects/build/build.generic.cmd rename to build/VS_scripts/build.generic.cmd diff --git a/projects/cmake/.gitignore b/build/cmake/.gitignore similarity index 100% rename from projects/cmake/.gitignore rename to build/cmake/.gitignore diff --git a/projects/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt similarity index 100% rename from projects/cmake/CMakeLists.txt rename to build/cmake/CMakeLists.txt diff --git a/projects/cmake/CMakeModules/AddExtraCompilationFlags.cmake b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake similarity index 100% rename from projects/cmake/CMakeModules/AddExtraCompilationFlags.cmake rename to build/cmake/CMakeModules/AddExtraCompilationFlags.cmake diff --git a/projects/cmake/cmake_uninstall.cmake.in b/build/cmake/cmake_uninstall.cmake.in similarity index 100% rename from projects/cmake/cmake_uninstall.cmake.in rename to build/cmake/cmake_uninstall.cmake.in diff --git a/projects/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt similarity index 100% rename from projects/cmake/lib/CMakeLists.txt rename to build/cmake/lib/CMakeLists.txt diff --git a/projects/cmake/programs/.gitignore b/build/cmake/programs/.gitignore similarity index 100% rename from projects/cmake/programs/.gitignore rename to build/cmake/programs/.gitignore diff --git a/projects/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt similarity index 100% rename from projects/cmake/programs/CMakeLists.txt rename to build/cmake/programs/CMakeLists.txt diff --git a/projects/cmake/tests/.gitignore b/build/cmake/tests/.gitignore similarity index 100% rename from projects/cmake/tests/.gitignore rename to build/cmake/tests/.gitignore diff --git a/projects/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt similarity index 100% rename from projects/cmake/tests/CMakeLists.txt rename to build/cmake/tests/CMakeLists.txt From dbe70bad483dba784a60d44177a130e11f8a75cc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 19 Sep 2016 15:08:43 +0200 Subject: [PATCH 06/16] completed change from projects to build --- build/cmake/lib/CMakeLists.txt | 2 +- programs/Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 36e8afa1d..c984145ba 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -108,7 +108,7 @@ IF (ZSTD_LEGACY_SUPPORT) ENDIF (ZSTD_LEGACY_SUPPORT) IF (MSVC) - SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib) + SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/zstdlib) SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc) ENDIF (MSVC) diff --git a/programs/Makefile b/programs/Makefile index 76130fe50..6e78d0ea9 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -57,8 +57,8 @@ endif ifneq (,$(filter Windows%,$(OS))) EXT =.exe VOID = nul -RES64_FILE = ..\projects\VS2010\zstd\generate_res\zstd64.res -RES32_FILE = ..\projects\VS2010\zstd\generate_res\zstd32.res +RES64_FILE = ..\build\VS2010\zstd\generate_res\zstd64.res +RES32_FILE = ..\build\VS2010\zstd\generate_res\zstd32.res ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine))) RES_FILE = $(RES64_FILE) else From 0704df3259e762638b610b9e0e008d9cd5898b59 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 19 Sep 2016 16:55:35 +0200 Subject: [PATCH 07/16] fixed cmake test --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b50723e85..2e5eada0f 100644 --- a/Makefile +++ b/Makefile @@ -122,9 +122,9 @@ endif ifneq (,$(filter $(HOST_OS),MSYS POSIX)) cmaketest: cmake --version - $(RM) -r $(BUILDDIR)/cmake/build - mkdir $(BUILDDIR)/cmake/build - cd $(BUILDDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall + $(RM) -r $(BUILDIR)/cmake/build + mkdir $(BUILDIR)/cmake/build + cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall c90test: clean CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long From 86bdcd83c13177cee08f730e086482294450552d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 20 Sep 2016 11:54:29 +0200 Subject: [PATCH 08/16] added error checking --- examples/streaming_compression.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index d663a4914..108a63c83 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -73,7 +73,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_CStream* const cstream = ZSTD_createCStream(); if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } size_t const initResult = ZSTD_initCStream(cstream, cLevel); - if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); } + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } size_t read, toRead = buffInSize; while( (read = fread_orDie(buffIn, toRead, fin)) ) { @@ -81,6 +81,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve while (input.pos < input.size) { ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ + if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } if (toRead > buffInSize) toRead = buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */ fwrite_orDie(buffOut, output.pos, fout); } @@ -88,7 +89,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; size_t const remainingToFlush = ZSTD_endStream(cstream, &output); /* close frame */ - if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); } + if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); } fwrite_orDie(buffOut, output.pos, fout); ZSTD_freeCStream(cstream); From 47f3697f32c9ace7f8b36c3fcc9710d4a42a0a43 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 20 Sep 2016 11:59:12 +0200 Subject: [PATCH 09/16] added error check --- examples/streaming_decompression.c | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c index 51340bae7..1ba1a3d81 100644 --- a/examples/streaming_decompression.c +++ b/examples/streaming_decompression.c @@ -80,6 +80,7 @@ static void decompressFile_orDie(const char* fname) while (input.pos < input.size) { ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; toRead = ZSTD_decompressStream(dstream, &output , &input); /* toRead : size of next compressed block */ + if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_decompressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } fwrite_orDie(buffOut, output.pos, fout); } } From 6c7e7ddee98237f0e903457204d0154903c8972d Mon Sep 17 00:00:00 2001 From: jungle-boogie Date: Tue, 20 Sep 2016 10:15:16 -0700 Subject: [PATCH 10/16] gmake necessary on *BSD systems. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 85d5ac324..27c31ff55 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Once you have the repository cloned, there are multiple ways provided to build Z #### Makefile If your system is compatible with `make`, you can simply run `make` at the root directory. -It will generate `zstd` within root directory. +It will generate `zstd` within root directory. Use `gmake` on *BSD systems. Other available options include : - `make install` : create and install zstd binary, library and man page From 84484cc656634c79962d8e0be7e64d973d0f4008 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Sep 2016 11:24:22 +0200 Subject: [PATCH 11/16] minor build comment --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 27c31ff55..53609a146 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,9 @@ Once you have the repository cloned, there are multiple ways provided to build Z #### Makefile -If your system is compatible with `make`, you can simply run `make` at the root directory. -It will generate `zstd` within root directory. Use `gmake` on *BSD systems. +If your system is compatible with a standard `make` (or `gmake`) binary generator, +you can simply run it at the root directory. +It will generate `zstd` within root directory. Other available options include : - `make install` : create and install zstd binary, library and man page From 0977f7ece696f6a91a774f6c742fce8561d0c823 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Sep 2016 12:24:43 +0200 Subject: [PATCH 12/16] minor refactor for clarity --- programs/zstdcli.c | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 14571344f..7c93fdaad 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -23,7 +23,7 @@ #endif #ifndef ZSTDCLI_CLEVEL_MAX -# define ZSTDCLI_CLEVEL_MAX 19 +# define ZSTDCLI_CLEVEL_MAX 19 /* when not using --ultra */ #endif @@ -51,13 +51,11 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ +# include /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else -# if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -# else -# define IS_CONSOLE(stdStream) 0 -# endif +# define IS_CONSOLE(stdStream) 0 #endif @@ -195,7 +193,7 @@ static unsigned readU32FromChar(const char** stringPtr) #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } -int main(int argCount, char** argv) +int main(int argCount, const char* argv[]) { int argNb, bench=0, @@ -219,13 +217,12 @@ int main(int argCount, char** argv) const char* programName = argv[0]; const char* outFileName = NULL; const char* dictFileName = NULL; - char* dynNameSpace = NULL; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictID = 0; int dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; #ifdef UTIL_HAS_CREATEFILELIST - const char** fileNamesTable = NULL; + const char** extendedFileList = NULL; char* fileNamesBuf = NULL; unsigned fileNamesNb; #endif @@ -432,13 +429,13 @@ int main(int argCount, char** argv) DISPLAYLEVEL(3, WELCOME_MESSAGE); #ifdef UTIL_HAS_CREATEFILELIST - if (recursive) { - fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); - if (fileNamesTable) { + if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + if (extendedFileList) { unsigned u; - for (u=0; u Date: Wed, 21 Sep 2016 14:20:56 +0200 Subject: [PATCH 13/16] Implemented "command must be followed by argument" protection suggested by @terrelln (#375) --- programs/zstdcli.c | 19 ++++++++++++++----- tests/playTests.sh | 22 +++++++++++++++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7c93fdaad..95267aa89 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -208,7 +208,8 @@ int main(int argCount, const char* argv[]) nextArgumentIsMaxDict=0, nextArgumentIsDictID=0, nextArgumentIsFile=0, - ultra=0; + ultra=0, + lastCommand = 0; int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; @@ -268,8 +269,8 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } if (!strcmp(argument, "--test")) { testmode=1; decode=1; continue; } if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } - if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } - if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } + if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } + if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } @@ -287,6 +288,10 @@ int main(int argCount, const char* argv[]) argument++; while (argument[0]!=0) { + if (lastCommand) { + DISPLAY("error : command must be followed by argument \n"); + return 1; + } #ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { @@ -309,7 +314,7 @@ int main(int argCount, const char* argv[]) case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break; /* Use file content as dictionary */ - case 'D': nextEntryIsDictionary = 1; argument++; break; + case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; /* Overwrite */ case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; @@ -330,7 +335,7 @@ int main(int argCount, const char* argv[]) case 't': testmode=1; decode=1; argument++; break; /* destination file name */ - case 'o': nextArgumentIsOutFileName=1; argument++; break; + case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; #ifdef UTIL_HAS_CREATEFILELIST /* recursive */ @@ -394,6 +399,7 @@ int main(int argCount, const char* argv[]) if (nextArgumentIsMaxDict) { nextArgumentIsMaxDict = 0; + lastCommand = 0; maxDictSize = readU32FromChar(&argument); if (toupper(*argument)=='K') maxDictSize <<= 10; if (toupper(*argument)=='M') maxDictSize <<= 20; @@ -402,6 +408,7 @@ int main(int argCount, const char* argv[]) if (nextArgumentIsDictID) { nextArgumentIsDictID = 0; + lastCommand = 0; dictID = readU32FromChar(&argument); continue; } @@ -410,12 +417,14 @@ int main(int argCount, const char* argv[]) if (nextEntryIsDictionary) { nextEntryIsDictionary = 0; + lastCommand = 0; dictFileName = argument; continue; } if (nextArgumentIsOutFileName) { nextArgumentIsOutFileName = 0; + lastCommand = 0; outFileName = argument; if (!strcmp(outFileName, "-")) outFileName = stdoutmark; continue; diff --git a/tests/playTests.sh b/tests/playTests.sh index 042197c2d..233f0775a 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -54,6 +54,14 @@ $ZSTD -99 -f tmp # too large compression level, automatic sized down $ECHO "test : compress to stdout" $ZSTD tmp -c > tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format +$ECHO "test : compress to named file" +rm tmpCompressed +$ZSTD tmp -o tmpCompressed +ls tmpCompressed # must work +$ECHO "test : -o must be followed by filename (must fail)" +$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename" +$ECHO "test : force write, correct order" +$ZSTD tmp -fo tmpCompressed $ECHO "test : implied stdout when input is stdin" $ECHO bob | $ZSTD | $ZSTD -d $ECHO "test : null-length file roundtrip" @@ -183,18 +191,26 @@ $ECHO "- Create first dictionary" $ZSTD --train *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp $ZSTD -f tmp -D tmpDict -$ZSTD -d tmp.zst -D tmpDict -of result +$ZSTD -d tmp.zst -D tmpDict -fo result diff $TESTFILE result $ECHO "- Create second (different) dictionary" $ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC -$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!" +$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" $ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +$ECHO "- Create dictionary with wrong dictID parameter order (must fail)" +$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " +$ECHO "- Create dictionary with size limit" +$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict 4K -v +$ECHO "- Create dictionary with wrong parameter order (must fail)" +$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " $ECHO "- Compress without dictID" $ZSTD -f tmp -D tmpDict1 --no-dictID -$ZSTD -d tmp.zst -D tmpDict -of result +$ZSTD -d tmp.zst -D tmpDict -fo result diff $TESTFILE result +$ECHO "- Compress with wrong argument order (must fail)" +$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name " $ECHO "- Compress multiple files with dictionary" rm -rf dirTestDict mkdir dirTestDict From 714464f05d8715bca185bfb027d9318210beeedb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Sep 2016 16:05:03 +0200 Subject: [PATCH 14/16] fixed : cli : forgotten mandatory argument --- programs/zstdcli.c | 2 ++ tests/playTests.sh | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 95267aa89..412870e27 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -434,6 +434,8 @@ int main(int argCount, const char* argv[]) filenameTable[filenameIdx++] = argument; } + if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; } /* forgotten argument */ + /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); diff --git a/tests/playTests.sh b/tests/playTests.sh index 233f0775a..d94d8fab9 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -59,9 +59,12 @@ rm tmpCompressed $ZSTD tmp -o tmpCompressed ls tmpCompressed # must work $ECHO "test : -o must be followed by filename (must fail)" -$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename" +$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename " $ECHO "test : force write, correct order" $ZSTD tmp -fo tmpCompressed +$ECHO "test : forgotten argument" +cp tmp tmp2 +$ZSTD tmp2 -fo && die "-o must be followed by filename " $ECHO "test : implied stdout when input is stdin" $ECHO bob | $ZSTD | $ZSTD -d $ECHO "test : null-length file roundtrip" From 993060e0f23dc946b3852a40f3aa68aa6a5e468a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Sep 2016 16:46:08 +0200 Subject: [PATCH 15/16] cli : better adaptation to small files --- lib/compress/zstd_compress.c | 2 +- programs/fileio.c | 12 +++++++----- programs/zstdcli.c | 5 ++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f5b712fc3..856335e52 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2730,7 +2730,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ - { ZSTD_customMem cMem = cdict->refContext->customMem; + { ZSTD_customMem const cMem = cdict->refContext->customMem; ZSTD_freeCCtx(cdict->refContext); ZSTD_free(cdict->dictContent, cMem); ZSTD_free(cdict, cMem); diff --git a/programs/fileio.c b/programs/fileio.c index 7dee7c11b..56f22fe75 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -255,7 +255,7 @@ typedef struct { FILE* srcFile; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName, int cLevel) +static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -272,11 +272,11 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel) { void* dictBuffer; size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName); if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer"); - { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize); + { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); params.fParams.contentSizeFlag = 1; params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; - { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0); + { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); } } free(dictBuffer); @@ -408,8 +408,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel) { clock_t const start = clock(); + U64 const srcSize = UTIL_getFileSize(srcFileName); - cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; @@ -428,7 +429,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile size_t dfnSize = FNSPACE; char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; - cRess_t ress = FIO_createCResources(dictFileName, compressionLevel); + U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 412870e27..6d1d9f649 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -505,15 +505,14 @@ int main(int argCount, const char* argv[]) FIO_setNotificationLevel(displayLevel); if (!decode) { #ifndef ZSTD_NOCOMPRESS - if (filenameIdx==1 && outFileName) + if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); else operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel); #else DISPLAY("Compression not supported\n"); #endif - } else - { /* decompression */ + } else { /* decompression */ #ifndef ZSTD_NODECOMPRESS if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ if (filenameIdx==1 && outFileName) From 97b378a6f8a0114294d3f6a108a3f7c5552923ae Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Sep 2016 17:20:19 +0200 Subject: [PATCH 16/16] Streaming : dictionary compression on multiple files / segments can correctly provide srcSize into header (when provided) using pledgedSrcSize. --- lib/compress/zstd_compress.c | 7 +++---- lib/dictBuilder/zdict.c | 2 +- lib/zstd.h | 2 +- tests/fuzzer.c | 16 ++++++++-------- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 856335e52..298278c99 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -322,13 +322,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). * @return : 0, or an error code */ -size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) { if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, ZSTDcrp_noMemset); - dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset); /* copy tables */ { size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); @@ -2740,7 +2739,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) { - if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext)) + if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); return 0; } diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index cfabb20ba..8a38aadeb 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -563,7 +563,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); + { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } } cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize); diff --git a/lib/zstd.h b/lib/zstd.h index 31171d04d..d7eb9c01f 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -447,7 +447,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b8f102a9c..ae8450e40 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -173,13 +173,13 @@ static int basicUnitTests(U32 seed, double compressibility) static const size_t dictSize = 551; DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++); - { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig); + { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */ DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++); CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); - CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) ); + CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, CNBuffSize - dictSize) ); DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++); @@ -221,10 +221,10 @@ static int basicUnitTests(U32 seed, double compressibility) p.fParams.contentSizeFlag = 1; CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); } - CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) ); + CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); - CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), - (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), + CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), + (const char*)CNBuffer + dictSize, testSize), cSize = r); { ZSTD_frameParams fp; if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error; @@ -674,9 +674,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode)); } - { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx); - CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); } - } + { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0); + CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); + } } XXH64_reset(&xxhState, 0); { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; U32 n;