From f33021552b5036db1c143928f701d2b71355a4ec Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 16:38:36 +0200 Subject: [PATCH 01/12] added playTestsMSYS.sh --- appveyor.yml | 42 ++++---- programs/Makefile | 52 ++++++---- programs/{ => tests}/playTests.sh | 0 programs/tests/playTestsMSYS.sh | 160 ++++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 41 deletions(-) rename programs/{ => tests}/playTests.sh (100%) mode change 100755 => 100644 create mode 100644 programs/tests/playTestsMSYS.sh diff --git a/appveyor.yml b/appveyor.yml index b5ffb4a25..9a493a616 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,11 @@ version: 1.0.{build} environment: matrix: - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw32" + - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw64" - ZSTD_APPVEYOR: "visual" CONFIGURATION: "Debug" PLATFORM: "Win32" @@ -18,7 +23,6 @@ environment: install: - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%ZSTD_APPVEYOR%]==[gcc] ( - SET "MINGW_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen" && SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && @@ -29,24 +33,9 @@ install: build_script: - ECHO Building %ZSTD_APPVEYOR% - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw32 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw32.exe && - make clean - ) - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw64 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw64.exe && + - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] ( make clean && ECHO *** && ECHO *** Building clang && @@ -56,6 +45,15 @@ build_script: COPY programs\fuzzer.exe projects\fuzzer_clang.exe && make clean ) + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building %PLATFORM% && + ECHO *** && + ECHO make %MAKE_PARAMS% && + make %MAKE_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_%PLATFORM%.exe && + make clean + ) - if [%ZSTD_APPVEYOR%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && @@ -82,14 +80,14 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: - SET FUZZERTEST=-T1mn - if [%ZSTD_APPVEYOR%]==[gcc] ( - projects\fuzzer_mingw32.exe %FUZZERTEST% && - projects\fuzzer_mingw64.exe %FUZZERTEST% && - projects\fuzzer_clang.exe %FUZZERTEST% + projects\fuzzer_%PLATFORM%.exe %FUZZERTEST% && + if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && diff --git a/programs/Makefile b/programs/Makefile index 70434cedc..339427e2d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -163,7 +163,11 @@ clean: #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) +HOST_OS = POSIX +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) install: zstd @echo Installing binaries @@ -186,15 +190,41 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled +valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 +valgrindTest: zstd datagen fuzzer fullbench zbufftest + @echo "\n ---- valgrind tests : memory analyzer ----" + $(VALGRIND) ./datagen -g50M > $(VOID) + $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi + ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) + ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) + ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp + $(VALGRIND) ./zstd -vdf tmp -o $(VOID) + ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) + @rm tmp + $(VALGRIND) ./fuzzer -T1mn -t1 + $(VALGRIND) ./fullbench -i1 + $(VALGRIND) ./zbufftest -T1mn + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTestsMSYS.sh $(ZSTDRTTEST) +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) test: test-zstd test-fullbench test-fuzzer test-zbuff test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest -zstd-playTests: datagen - ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) - test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests @@ -223,20 +253,4 @@ test-zbuff: zbufftest test-zbuff32: zbufftest32 ./zbufftest32 $(ZBUFFTEST) - -valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: zstd datagen fuzzer fullbench zbufftest - @echo "\n ---- valgrind tests : memory analyzer ----" - $(VALGRIND) ./datagen -g50M > $(VOID) - $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi - ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) - ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp - $(VALGRIND) ./zstd -vdf tmp -o $(VOID) - ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) - @rm tmp - $(VALGRIND) ./fuzzer -T1mn -t1 - $(VALGRIND) ./fullbench -i1 - $(VALGRIND) ./zbufftest -T1mn - endif diff --git a/programs/playTests.sh b/programs/tests/playTests.sh old mode 100755 new mode 100644 similarity index 100% rename from programs/playTests.sh rename to programs/tests/playTests.sh diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh new file mode 100644 index 000000000..82f2357bd --- /dev/null +++ b/programs/tests/playTestsMSYS.sh @@ -0,0 +1,160 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 + fc tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + + +echo "\n**** simple tests **** " +./datagen > tmp +$ZSTD -f tmp # trivial compression case, creates tmp.zst +$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) +echo "test : too large compression level (must fail)" +$ZSTD -99 tmp && die "too large compression level undetected" +echo "test : compress to stdout" +$ZSTD tmp -c > tmpCompressed +$ZSTD tmp --stdout > tmpCompressed # long command format +echo "test : null-length file roundtrip" +echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout +echo "test : decompress file with wrong suffix (must fail)" +$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" +$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout +$ZSTD --decompress tmpCompressed -c > tmpResult +$ZSTD --decompress tmpCompressed --stdout > tmpResult +#$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout +#$ZSTD -d - < tmp.zst > /dev/null +$ZSTD -dc < tmp.zst > /dev/null +$ZSTD -dc - < tmp.zst > /dev/null +$ZSTD -q tmp && die "overwrite check failed!" +$ZSTD -q -f tmp +$ZSTD -q --force tmp +$ZSTD -df tmp && die "should have refused : wrong extension" +cp tmp tmp2.zst +$ZSTD -df tmp2.zst && die "should have failed : wrong format" +rm tmp2.zst + +echo "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD -c hello.tmp > hello.zstd +$ZSTD -c world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -dc helloworld.zstd > result.tmp +cat result.tmp +fc helloworld.tmp result.tmp +rm ./*.tmp ./*.zstd + +echo frame concatenation test completed + + +echo "\n**** dictionary tests **** " + +./datagen > tmpDict +./datagen -g1M | md5sum > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 +fc tmp1 tmp2 +$ZSTD --train *.c *.h -o tmpDict +$ZSTD xxhash.c -D tmpDict -of tmp +$ZSTD -d tmp -D tmpDict -of result +fc xxhash.c result + + +echo "\n**** multiple files tests **** " + +./datagen -s1 > tmp1 2> /dev/null +./datagen -s2 -g100K > tmp2 2> /dev/null +./datagen -s3 -g1M > tmp3 2> /dev/null +$ZSTD -f tmp* +echo "compress tmp* : " +ls -ls tmp* +rm tmp1 tmp2 tmp3 +echo "decompress tmp* : " +$ZSTD -df *.zst +ls -ls tmp* +echo "compress tmp* into stdout > tmpall : " +$ZSTD -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* +echo "decompress tmpall* into stdout > tmpdec : " +cp tmpall tmpall2 +# $ZSTD -dc tmpall* > tmpdec +ls -ls tmp* +echo "compress multiple files including a missing one (notHere) : " +$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" + +echo "\n**** integrity tests **** " +echo "test one file (tmp1.zst) " +$ZSTD -t tmp1.zst +$ZSTD --test tmp1.zst +echo "test multiple files (*.zst) " +$ZSTD -t *.zst +echo "test good and bad files (*) " +$ZSTD -t * && die "bad files not detected !" + +echo "\n**** zstd round-trip tests **** " + +roundTripTest +roundTripTest -g15K # TableID==3 +roundTripTest -g127K # TableID==2 +roundTripTest -g255K # TableID==1 +roundTripTest -g513K # TableID==0 +roundTripTest -g512K 6 # greedy, hash chain +roundTripTest -g512K 16 # btlazy2 +roundTripTest -g512K 19 # btopt + +rm tmp* + +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 + +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 1 + +rm tmp* + From 877cc84f859cd910355682d65e502349afae5ca8 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:03:02 +0200 Subject: [PATCH 02/12] changed appveyor.yml --- .gitattributes | 3 +++ appveyor.yml | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index 3959727b8..f314af9ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -14,3 +14,6 @@ *.vcxproj* text eol=crlf *.suo binary *.rc binary + +# Windows +*.bat text eol=crlf diff --git a/appveyor.yml b/appveyor.yml index 9a493a616..b79ef2a59 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,9 @@ install: SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe ) else ( - IF [%PLATFORM%]==[x64] SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") && + COPY programs\tests\playTests.bat playTests.bat && + COPY programs\tests\roundTripTest.bat roundTripTest.bat ) build_script: @@ -51,7 +53,6 @@ build_script: ECHO *** && ECHO make %MAKE_PARAMS% && make %MAKE_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_%PLATFORM%.exe && make clean ) - if [%ZSTD_APPVEYOR%]==[visual] ( @@ -79,14 +80,13 @@ build_script: ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: - SET FUZZERTEST=-T1mn - if [%ZSTD_APPVEYOR%]==[gcc] ( - projects\fuzzer_%PLATFORM%.exe %FUZZERTEST% && if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( @@ -94,5 +94,6 @@ test_script: projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && + zstd_VS2015_%PLATFORM%_Release.exe ) From 3aa4b2b598686973525a540e00247405d37a3d92 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:04:10 +0200 Subject: [PATCH 03/12] update file permission --- programs/tests/playTests.sh | 0 programs/tests/playTestsMSYS.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 programs/tests/playTests.sh mode change 100644 => 100755 programs/tests/playTestsMSYS.sh diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh old mode 100644 new mode 100755 diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh old mode 100644 new mode 100755 From 2bd6b13e5337ed10af0292963e51b1ac38ab3258 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 18 May 2016 18:05:56 +0200 Subject: [PATCH 04/12] added playTests.bat --- programs/tests/playTests.bat | 149 +++++++++++++++++++++++++++++++ programs/tests/roundTripTest.bat | 8 ++ 2 files changed, 157 insertions(+) create mode 100644 programs/tests/playTests.bat create mode 100644 programs/tests/roundTripTest.bat diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat new file mode 100644 index 000000000..19c57d5fc --- /dev/null +++ b/programs/tests/playTests.bat @@ -0,0 +1,149 @@ +@echo off +if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 + +echo. && echo **** simple tests **** +datagen > tmp +%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst +%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) +echo test : too large compression level (must fail) +%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1) +echo test : compress to stdout +%ZSTD% tmp -c > tmpCompressed +%ZSTD% tmp --stdout > tmpCompressed && REM long command format +echo test : null-length file roundtrip +echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1) +echo test : decompress file with wrong suffix (must fail) +%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1) +%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout +%ZSTD% --decompress tmpCompressed -c > tmpResult +%ZSTD% --decompress tmpCompressed --stdout > tmpResult +REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout +REM %ZSTD% -d - < tmp.zst > NUL +%ZSTD% -dc < tmp.zst > NUL +%ZSTD% -dc - < tmp.zst > NUL +%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1) +%ZSTD% -q -f tmp +%ZSTD% -q --force tmp +%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) +cp tmp tmp2.zst +%ZSTD% -df tmp2.zst && (echo should have failed : wrong format && exit /b 1) +rm tmp2.zst + +echo. && echo **** frame concatenation **** + +echo hello > hello.tmp +echo world! > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +%ZSTD% -c hello.tmp > hello.zstd +%ZSTD% -c world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +%ZSTD% -dc helloworld.zstd > result.tmp +cat result.tmp +fc /b helloworld.tmp result.tmp +rm *.tmp *.zstd + +echo frame concatenation test completed + + +REM echo. && echo **** flush write error test **** + +REM echo echo foo ^| %ZSTD% ^> v:\full +REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1) +REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full" +REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1) + + +echo. && echo **** dictionary tests **** + +datagen > tmpDict +datagen -g1M | md5sum > tmp1 +datagen -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 +fc tmp1 tmp2 +%ZSTD% --train *.c *.h -o tmpDict +%ZSTD% xxhash.c -D tmpDict -of tmp +%ZSTD% -d tmp -D tmpDict -of result +fc xxhash.c result + + +echo. && echo **** multiple files tests **** + +datagen -s1 > tmp1 2> NUL +datagen -s2 -g100K > tmp2 2> NUL +datagen -s3 -g1M > tmp3 2> NUL +%ZSTD% -f tmp* +echo compress tmp* : +ls -ls tmp* +rm tmp1 tmp2 tmp3 +echo decompress tmp* : +%ZSTD% -df *.zst +ls -ls tmp* + +echo compress tmp* into stdout ^> tmpall : +%ZSTD% -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* +echo decompress tmpall* into stdout ^> tmpdec : +cp tmpall tmpall2 +%ZSTD% -dc tmpall* > tmpdec +ls -ls tmp* +echo compress multiple files including a missing one (notHere) : +%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1) + + +echo. && echo **** integrity tests **** +echo test one file (tmp1.zst) +%ZSTD% -t tmp1.zst +%ZSTD% --test tmp1.zst +echo test multiple files (*.zst) +%ZSTD% -t *.zst +echo test good and bad files (*) +%ZSTD% -t * && (echo bad files not detected! && exit /b 1) + + +echo. && echo **** zstd round-trip tests **** + +CALL roundTripTest.bat +CALL roundTripTest.bat -g15K && REM TableID==3 +CALL roundTripTest.bat -g127K && REM TableID==2 +CALL roundTripTest.bat -g255K && REM TableID==1 +CALL roundTripTest.bat -g513K && REM TableID==0 +CALL roundTripTest.bat -g512K 6 && REM greedy, hash chain +CALL roundTripTest.bat -g512K 16 && REM btlazy2 +CALL roundTripTest.bat -g512K 19 && REM btopt + +rm tmp* +echo Param = %1 +if NOT "%1"=="--test-large-data" ( + echo skipping large data tests + exit /b 0 +) + +CALL roundTripTest.bat -g270000000 1 +CALL roundTripTest.bat -g270000000 2 +CALL roundTripTest.bat -g270000000 3 + +CALL roundTripTest.bat -g140000000 -P60 4 +CALL roundTripTest.bat -g140000000 -P60 5 +CALL roundTripTest.bat -g140000000 -P60 6 + +CALL roundTripTest.bat -g70000000 -P70 7 +CALL roundTripTest.bat -g70000000 -P70 8 +CALL roundTripTest.bat -g70000000 -P70 9 + +CALL roundTripTest.bat -g35000000 -P75 10 +CALL roundTripTest.bat -g35000000 -P75 11 +CALL roundTripTest.bat -g35000000 -P75 12 + +CALL roundTripTest.bat -g18000000 -P80 13 +CALL roundTripTest.bat -g18000000 -P80 14 +CALL roundTripTest.bat -g18000000 -P80 15 +CALL roundTripTest.bat -g18000000 -P80 16 +CALL roundTripTest.bat -g18000000 -P80 17 + +CALL roundTripTest.bat -g50000000 -P94 18 +CALL roundTripTest.bat -g50000000 -P94 19 + +CALL roundTripTest.bat -g99000000 -P99 20 +CALL roundTripTest.bat -g6000000000 -P99 1 + +rm tmp* +exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat new file mode 100644 index 000000000..b1541bd14 --- /dev/null +++ b/programs/tests/roundTripTest.bat @@ -0,0 +1,8 @@ +@echo off +if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) +rm -f tmp1 tmp2 +echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d +datagen %1 %P% | md5sum > tmp1 +datagen %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 +fc tmp1 tmp2 +EXIT /B %ERRORLEVEL% From 60af95d9718a54c165937fb8f325e1d278b61565 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 10:29:49 +0200 Subject: [PATCH 05/12] fixed compression with -c with multiple files on Windows --- appveyor.yml | 2 +- programs/fileio.c | 3 ++- programs/tests/playTestsMSYS.sh | 14 +++++++------- programs/zstdcli.c | 5 +---- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b79ef2a59..e002d89ef 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -95,5 +95,5 @@ test_script: projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - zstd_VS2015_%PLATFORM%_Release.exe + REM zstd_VS2015_%PLATFORM%_Release.exe ) diff --git a/programs/fileio.c b/programs/fileio.c index b3f324ac6..638e63b57 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -71,7 +71,7 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ # include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); } #else # include /* isatty */ # define SET_BINARY_MODE(file) @@ -439,6 +439,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile if (!strcmp(suffix, stdoutmark)) { unsigned u; ress.dstFile = stdout; + SET_BINARY_MODE(stdout); for (u=0; u tmp $ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) @@ -51,7 +51,7 @@ cp tmp tmp2.zst $ZSTD -df tmp2.zst && die "should have failed : wrong format" rm tmp2.zst -echo "\n**** frame concatenation **** " +echo -e "\n**** frame concatenation **** " echo "hello " > hello.tmp echo "world!" > world.tmp @@ -67,7 +67,7 @@ rm ./*.tmp ./*.zstd echo frame concatenation test completed -echo "\n**** dictionary tests **** " +echo -e "\n**** dictionary tests **** " ./datagen > tmpDict ./datagen -g1M | md5sum > tmp1 @@ -79,7 +79,7 @@ $ZSTD -d tmp -D tmpDict -of result fc xxhash.c result -echo "\n**** multiple files tests **** " +echo -e "\n**** multiple files tests **** " ./datagen -s1 > tmp1 2> /dev/null ./datagen -s2 -g100K > tmp2 2> /dev/null @@ -96,12 +96,12 @@ $ZSTD -c tmp1 tmp2 tmp3 > tmpall ls -ls tmp* echo "decompress tmpall* into stdout > tmpdec : " cp tmpall tmpall2 -# $ZSTD -dc tmpall* > tmpdec +$ZSTD -dc tmpall* > tmpdec ls -ls tmp* echo "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" -echo "\n**** integrity tests **** " +echo -e "\n**** integrity tests **** " echo "test one file (tmp1.zst) " $ZSTD -t tmp1.zst $ZSTD --test tmp1.zst @@ -110,7 +110,7 @@ $ZSTD -t *.zst echo "test good and bad files (*) " $ZSTD -t * && die "bad files not detected !" -echo "\n**** zstd round-trip tests **** " +echo -e "\n**** zstd round-trip tests **** " roundTripTest roundTripTest -g15K # TableID==3 diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 310db7f88..8a8af3a2c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -49,14 +49,11 @@ * OS-specific Includes **************************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else extern int fileno(FILE *stream); /* triggers fileno() within on POSIX */ # include /* isatty */ -# define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #endif From 44bb2f6c89161915c8ff46ddc3b3492116b1c377 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 12:16:14 +0200 Subject: [PATCH 06/12] added datagen in Visual Studio project --- appveyor.yml | 12 +- programs/datagencli.c | 2 +- programs/tests/playTests.bat | 75 ++++---- programs/tests/roundTripTest.bat | 4 +- projects/VS2010/datagen/datagen.vcxproj | 168 ++++++++++++++++++ .../VS2010/datagen/datagen.vcxproj.filters | 26 +++ projects/VS2010/zstd.sln | 10 ++ 7 files changed, 253 insertions(+), 44 deletions(-) create mode 100644 projects/VS2010/datagen/datagen.vcxproj create mode 100644 projects/VS2010/datagen/datagen.vcxproj.filters diff --git a/appveyor.yml b/appveyor.yml index e002d89ef..a68e12eab 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,9 +28,7 @@ install: SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe ) else ( - IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") && - COPY programs\tests\playTests.bat playTests.bat && - COPY programs\tests\roundTripTest.bat roundTripTest.bat + IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") ) build_script: @@ -81,7 +79,8 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\datagen.exe projects\datagen_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: @@ -95,5 +94,8 @@ test_script: projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - REM zstd_VS2015_%PLATFORM%_Release.exe + CD programs && + SET ZSTD=..\projects\zstd_VS2015_%PLATFORM%_Release.exe && + SET DATAGEN=..\projects\datagen_VS2015_%PLATFORM%_Release.exe && + CALL tests\playTests.bat --test-large-data ) diff --git a/programs/datagencli.c b/programs/datagencli.c index 6122c29bd..d437d5cb3 100644 --- a/programs/datagencli.c +++ b/programs/datagencli.c @@ -27,8 +27,8 @@ /*-************************************ * Includes **************************************/ +#include "util.h" /* Compiler options */ #include /* fprintf, stderr */ -#include "mem.h" #include "datagen.h" /* RDG_generate */ diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index 19c57d5fc..6e9fd53ff 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -1,8 +1,11 @@ @echo off if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 +if [%DATAGEN%]==[] echo DATAGEN variable must be defined! && exit /b 1 + +SET ROUNDTRIPTEST=tests\roundTripTest.bat echo. && echo **** simple tests **** -datagen > tmp +%DATAGEN% > tmp %ZSTD% -f tmp && REM trivial compression case, creates tmp.zst %ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) echo test : too large compression level (must fail) @@ -55,9 +58,9 @@ REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! echo. && echo **** dictionary tests **** -datagen > tmpDict -datagen -g1M | md5sum > tmp1 -datagen -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 +%DATAGEN% > tmpDict +%DATAGEN% -g1M | md5sum > tmp1 +%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 fc tmp1 tmp2 %ZSTD% --train *.c *.h -o tmpDict %ZSTD% xxhash.c -D tmpDict -of tmp @@ -67,9 +70,9 @@ fc xxhash.c result echo. && echo **** multiple files tests **** -datagen -s1 > tmp1 2> NUL -datagen -s2 -g100K > tmp2 2> NUL -datagen -s3 -g1M > tmp3 2> NUL +%DATAGEN% -s1 > tmp1 2> NUL +%DATAGEN% -s2 -g100K > tmp2 2> NUL +%DATAGEN% -s3 -g1M > tmp3 2> NUL %ZSTD% -f tmp* echo compress tmp* : ls -ls tmp* @@ -101,14 +104,14 @@ echo test good and bad files (*) echo. && echo **** zstd round-trip tests **** -CALL roundTripTest.bat -CALL roundTripTest.bat -g15K && REM TableID==3 -CALL roundTripTest.bat -g127K && REM TableID==2 -CALL roundTripTest.bat -g255K && REM TableID==1 -CALL roundTripTest.bat -g513K && REM TableID==0 -CALL roundTripTest.bat -g512K 6 && REM greedy, hash chain -CALL roundTripTest.bat -g512K 16 && REM btlazy2 -CALL roundTripTest.bat -g512K 19 && REM btopt +CALL %ROUNDTRIPTEST% +CALL %ROUNDTRIPTEST% -g15K && REM TableID==3 +CALL %ROUNDTRIPTEST% -g127K && REM TableID==2 +CALL %ROUNDTRIPTEST% -g255K && REM TableID==1 +CALL %ROUNDTRIPTEST% -g513K && REM TableID==0 +CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain +CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2 +CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt rm tmp* echo Param = %1 @@ -117,33 +120,33 @@ if NOT "%1"=="--test-large-data" ( exit /b 0 ) -CALL roundTripTest.bat -g270000000 1 -CALL roundTripTest.bat -g270000000 2 -CALL roundTripTest.bat -g270000000 3 +CALL %ROUNDTRIPTEST% -g270000000 1 +CALL %ROUNDTRIPTEST% -g270000000 2 +CALL %ROUNDTRIPTEST% -g270000000 3 -CALL roundTripTest.bat -g140000000 -P60 4 -CALL roundTripTest.bat -g140000000 -P60 5 -CALL roundTripTest.bat -g140000000 -P60 6 +CALL %ROUNDTRIPTEST% -g140000000 -P60 4 +CALL %ROUNDTRIPTEST% -g140000000 -P60 5 +CALL %ROUNDTRIPTEST% -g140000000 -P60 6 -CALL roundTripTest.bat -g70000000 -P70 7 -CALL roundTripTest.bat -g70000000 -P70 8 -CALL roundTripTest.bat -g70000000 -P70 9 +CALL %ROUNDTRIPTEST% -g70000000 -P70 7 +CALL %ROUNDTRIPTEST% -g70000000 -P70 8 +CALL %ROUNDTRIPTEST% -g70000000 -P70 9 -CALL roundTripTest.bat -g35000000 -P75 10 -CALL roundTripTest.bat -g35000000 -P75 11 -CALL roundTripTest.bat -g35000000 -P75 12 +CALL %ROUNDTRIPTEST% -g35000000 -P75 10 +CALL %ROUNDTRIPTEST% -g35000000 -P75 11 +CALL %ROUNDTRIPTEST% -g35000000 -P75 12 -CALL roundTripTest.bat -g18000000 -P80 13 -CALL roundTripTest.bat -g18000000 -P80 14 -CALL roundTripTest.bat -g18000000 -P80 15 -CALL roundTripTest.bat -g18000000 -P80 16 -CALL roundTripTest.bat -g18000000 -P80 17 +CALL %ROUNDTRIPTEST% -g18000000 -P80 13 +CALL %ROUNDTRIPTEST% -g18000000 -P80 14 +CALL %ROUNDTRIPTEST% -g18000000 -P80 15 +CALL %ROUNDTRIPTEST% -g18000000 -P80 16 +CALL %ROUNDTRIPTEST% -g18000000 -P80 17 -CALL roundTripTest.bat -g50000000 -P94 18 -CALL roundTripTest.bat -g50000000 -P94 19 +CALL %ROUNDTRIPTEST% -g50000000 -P94 18 +CALL %ROUNDTRIPTEST% -g50000000 -P94 19 -CALL roundTripTest.bat -g99000000 -P99 20 -CALL roundTripTest.bat -g6000000000 -P99 1 +CALL %ROUNDTRIPTEST% -g99000000 -P99 20 +CALL %ROUNDTRIPTEST% -g6000000000 -P99 1 rm tmp* exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat index b1541bd14..759891f0b 100644 --- a/programs/tests/roundTripTest.bat +++ b/programs/tests/roundTripTest.bat @@ -2,7 +2,7 @@ if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) rm -f tmp1 tmp2 echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d -datagen %1 %P% | md5sum > tmp1 -datagen %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 +%DATAGEN% %1 %P% | md5sum > tmp1 +%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 fc tmp1 tmp2 EXIT /B %ERRORLEVEL% diff --git a/projects/VS2010/datagen/datagen.vcxproj b/projects/VS2010/datagen/datagen.vcxproj new file mode 100644 index 000000000..fe2b7eec6 --- /dev/null +++ b/projects/VS2010/datagen/datagen.vcxproj @@ -0,0 +1,168 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {037E781E-81A6-494B-B1B3-438AB1200523} + Win32Proj + datagen + $(SolutionDir)bin\$(Platform)\$(Configuration)\ + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + true + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ + + + true + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + + + false + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(Platform)\$(Configuration)\ + + + false + false + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + + + + + + + + + \ No newline at end of file diff --git a/projects/VS2010/datagen/datagen.vcxproj.filters b/projects/VS2010/datagen/datagen.vcxproj.filters new file mode 100644 index 000000000..1ebbd6b03 --- /dev/null +++ b/projects/VS2010/datagen/datagen.vcxproj.filters @@ -0,0 +1,26 @@ + + + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + Source Files + + + + + Header Files + + + \ No newline at end of file diff --git a/projects/VS2010/zstd.sln b/projects/VS2010/zstd.sln index 825d7a130..698b8fe50 100644 --- a/projects/VS2010/zstd.sln +++ b/projects/VS2010/zstd.sln @@ -9,6 +9,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\full EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -49,6 +51,14 @@ Global {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32 {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64 {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.ActiveCfg = Debug|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.Build.0 = Debug|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.ActiveCfg = Debug|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.Build.0 = Debug|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.ActiveCfg = Release|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.Build.0 = Release|Win32 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.ActiveCfg = Release|x64 + {037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 96abb5a806941d07ae5e6de91ad62c566c7c40c5 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 19 May 2016 15:50:58 +0200 Subject: [PATCH 07/12] AppVeyor tests use playTests.bat --- appveyor.yml | 64 +++++++++++++++++++----------------- programs/tests/playTests.bat | 7 ++-- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a68e12eab..c6f735ea3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,28 +1,29 @@ version: 1.0.{build} environment: matrix: - - ZSTD_APPVEYOR: "gcc" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" + - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw32" - - ZSTD_APPVEYOR: "gcc" + - COMPILER: "gcc" MAKE_PARAMS: "test" PLATFORM: "mingw64" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - ZSTD_APPVEYOR: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" install: - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% + - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% + - if [%COMPILER%]==[gcc] ( SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && @@ -32,7 +33,7 @@ install: ) build_script: - - ECHO Building %ZSTD_APPVEYOR% + - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] ( @@ -45,7 +46,7 @@ build_script: COPY programs\fuzzer.exe projects\fuzzer_clang.exe && make clean ) - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - if [%COMPILER%]==[gcc] ( ECHO *** && ECHO *** Building %PLATFORM% && ECHO *** && @@ -53,7 +54,7 @@ build_script: make %MAKE_PARAMS% && make clean ) - - if [%ZSTD_APPVEYOR%]==[visual] ( + - if [%COMPILER%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -62,7 +63,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && @@ -77,25 +78,28 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\datagen.exe projects\datagen_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\ ) test_script: + - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - SET FUZZERTEST=-T1mn - - if [%ZSTD_APPVEYOR%]==[gcc] ( + - if [%COMPILER%]==[gcc] ( if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( + - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( + CD programs && + SET ZSTD=..\projects\zstd.exe && + SET DATAGEN=..\projects\datagen.exe && + CALL tests\playTests.bat --test-large-data && + CD .. && + projects\fullbench.exe -i1 && + projects\fullbench.exe -i1 -P0 && projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% && - CD programs && - SET ZSTD=..\projects\zstd_VS2015_%PLATFORM%_Release.exe && - SET DATAGEN=..\projects\datagen_VS2015_%PLATFORM%_Release.exe && - CALL tests\playTests.bat --test-large-data + projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat index 6e9fd53ff..edbbcb340 100644 --- a/programs/tests/playTests.bat +++ b/programs/tests/playTests.bat @@ -1,6 +1,9 @@ @echo off -if [%ZSTD%]==[] echo ZSTD variable must be defined! && exit /b 1 -if [%DATAGEN%]==[] echo DATAGEN variable must be defined! && exit /b 1 +ECHO Start playTests.bat +ECHO ZSTD^: %ZSTD% +ECHO DATAGEN^: %DATAGEN% +if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1 +if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1 SET ROUNDTRIPTEST=tests\roundTripTest.bat From 35b891c5b02608ddb09b5765104c52bf0819745d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 20 May 2016 19:42:20 +0200 Subject: [PATCH 08/12] zstd_stats.h updated to v0.6 --- lib/common/zstd_internal.h | 14 +--- lib/compress/.debug/zstd_stats.h | 135 ++++++++++++++++--------------- lib/compress/zstd_compress.c | 11 ++- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 4c9a7615a..8d9883d0f 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -186,6 +186,8 @@ MEM_STATIC unsigned ZSTD_highbit(U32 val) /*-******************************************* * Private interfaces *********************************************/ +typedef struct ZSTD_stats_s ZSTD_stats_t; + typedef struct { U32 off; U32 len; @@ -199,16 +201,6 @@ typedef struct { U32 rep[ZSTD_REP_INIT]; } ZSTD_optimal_t; -#if ZSTD_OPT_DEBUG == 3 - #include ".debug/zstd_stats.h" -#else - typedef struct { U32 unused; } ZSTD_stats_t; - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif - typedef struct { void* buffer; U32* offsetStart; @@ -245,7 +237,7 @@ typedef struct { U32 cachedPrice; U32 cachedLitLength; const BYTE* cachedLiterals; - ZSTD_stats_t stats; + ZSTD_stats_t* stats; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index d0189f872..51406a930 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -37,31 +37,21 @@ extern "C" { #endif -/*-************************************* -* Dependencies -***************************************/ -//#include "zstd.h" -//#include "mem.h" - - -/*-************************************* -* Constants -***************************************/ -//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#if ZSTD_OPT_DEBUG == 3 /*-************************************* * Types ***************************************/ -typedef struct { - U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength; +struct ZSTD_stats_s { + U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength; U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum; U32 litSum, matchLengthSum, litLengthSum, offCodeSum; - U32 matchLengthFreq[1<totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); - printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); - printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8); + printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); + printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8); } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1; - stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0; + stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { unsigned u; - stats->litSum = (1<litLengthSum = (1<matchLengthSum = (1<offCodeSum = (1<litSum = (2<litLengthSum = MaxLL+1; + stats->matchLengthSum = MaxML+1; + stats->offCodeSum = (MaxOff+1); for (u=0; u<=MaxLit; u++) stats->litFreq[u] = 1; @@ -99,62 +91,75 @@ MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) stats->offCodeFreq[u] = 1; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { - /* offset */ - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; - stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); - stats->priceOffset += (offCode-1) + (!offCode); - - /* match Length */ - stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); - stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1); - - if (litLength) { - /* literals */ - U32 u; - stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); - for (u=0; u < litLength; u++) - stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); - - /* literal Length */ - stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1); - } else { - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); - } - - - if (offset == 0) stats->totalRepSum++; - stats->totalSeqSum++; - stats->totalMatchSum += matchLength; - stats->totalLitSum += litLength; - U32 u; /* literals */ + stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); + for (u=0; u < litLength; u++) + stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); stats->litSum += litLength; for (u=0; u < litLength; u++) stats->litFreq[literals[u]]++; /* literal Length */ - stats->litLengthSum++; - if (litLength >= MaxLL) - stats->litLengthFreq[MaxLL]++; - else - stats->litLengthFreq[litLength]++; + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + if (litLength) { + stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1); + } else { + stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); + } + stats->litLengthFreq[llCode]++; + stats->litLengthSum++; + } /* match offset */ - stats->offCodeSum++; - stats->offCodeFreq[offCode]++; + { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); + stats->priceOffset += offCode; + stats->offCodeSum++; + stats->offCodeFreq[offCode]++; + } /* match Length */ - stats->matchLengthSum++; - if (matchLength >= MaxML) - stats->matchLengthFreq[MaxML]++; - else - stats->matchLengthFreq[matchLength]++; + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1); + stats->matchLengthFreq[mlCode]++; + stats->matchLengthSum++; + } + + if (offset == 0) stats->totalRepSum++; + stats->totalSeqSum++; + stats->totalMatchSum += matchLength; + stats->totalLitSum += litLength; } +#else + struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } +#endif // #if ZSTD_OPT_DEBUG == 3 #if defined (__cplusplus) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1bb75c68c..988a74fb7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -57,6 +57,7 @@ #include "fse_static.h" #include "huf_static.h" #include "zstd_internal.h" +#include ".debug/zstd_stats.h" /*-************************************* @@ -856,7 +857,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode); + ZSTD_statsUpdatePrices(seqStorePtr->stats, litLength, literals, offsetCode, matchCode); /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); @@ -1748,7 +1749,7 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; - ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); + ZSTD_statsUpdatePrices(seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -2020,8 +2021,9 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = &zc->seqStore.stats; - + ZSTD_stats_t* stats = malloc(sizeof(ZSTD_stats_t)); + if (!stats) return ERROR(memory_allocation); + zc->seqStore.stats = stats; ZSTD_statsInit(stats); while (remaining) { @@ -2059,6 +2061,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); + free(stats); return op-ostart; } From 50e82c015d56c00879bb8d2d551d117b840a9804 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 15:49:09 +0200 Subject: [PATCH 09/12] added ZSTD_createCCtx_advanced --- lib/common/zstd_static.h | 9 ++++++++ lib/compress/zstd_compress.c | 45 +++++++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index ef42b0a97..912eb1335 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -94,10 +94,19 @@ typedef struct { ZSTD_frameParameters fParams; } ZSTD_parameters; +typedef void* (*ZSTD_allocFunction) (size_t size); +typedef void (*ZSTD_freeFunction) (void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem; + /*-************************************* * Advanced functions ***************************************/ +/*! ZSTD_createCCtx_advanced() : + * Create ZSTD context using external alloc and free functions */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); + + ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getCParams() : diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 988a74fb7..00106f5fe 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -104,6 +104,8 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -118,13 +120,32 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - return (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + ZSTD_CCtx* ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + ctx->customAlloc = malloc; + ctx->customFree = free; + return ctx; +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + if (!customMem.customAlloc || !customMem.customFree) + return ZSTD_createCCtx(); + + ZSTD_CCtx* ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + memset(ctx, 0, sizeof(ZSTD_CCtx)); + ctx->customAlloc = customMem.customAlloc; + ctx->customFree = customMem.customFree; + return ctx; } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - free(cctx->workSpace); - free(cctx); + cctx->customFree(cctx->workSpace); + cctx->customFree(cctx); return 0; /* reserved as a potential error code in the future */ } @@ -229,8 +250,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); if (zc->workSpaceSize < neededSpace) { - free(zc->workSpace); - zc->workSpace = malloc(neededSpace); + zc->customFree(zc->workSpace); + zc->workSpace = zc->customAlloc(neededSpace); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; } } @@ -2021,7 +2042,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = malloc(sizeof(ZSTD_stats_t)); + ZSTD_stats_t* stats = (ZSTD_stats_t*) zc->customAlloc(sizeof(ZSTD_stats_t)); if (!stats) return ERROR(memory_allocation); zc->seqStore.stats = stats; ZSTD_statsInit(stats); @@ -2030,7 +2051,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { zc->customFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2041,11 +2062,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; + if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; + if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2061,7 +2082,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - free(stats); + zc->customFree(stats); return op-ostart; } @@ -2427,8 +2448,10 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); + ctxBody.customAlloc = malloc; + ctxBody.customFree = free; result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); - free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ + ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } From 107e2431955a7bb0d78f31d74bddd880f2250f75 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 16:24:52 +0200 Subject: [PATCH 10/12] added ZSTD_createDCtx_advanced --- lib/common/zstd_static.h | 5 ++++- lib/compress/zstd_compress.c | 27 +++++++++++++++++---------- lib/decompress/zstd_decompress.c | 31 ++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index 912eb1335..e0f50cf40 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -103,9 +103,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } * Advanced functions ***************************************/ /*! ZSTD_createCCtx_advanced() : - * Create ZSTD context using external alloc and free functions */ + * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API unsigned ZSTD_maxCLevel (void); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 00106f5fe..966d9acbd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -120,20 +120,25 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - ZSTD_CCtx* ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); - if (!ctx) return NULL; - - ctx->customAlloc = malloc; - ctx->customFree = free; - return ctx; + ZSTD_customMem customMem = { NULL, NULL }; + return ZSTD_createCCtx_advanced(customMem); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { - if (!customMem.customAlloc || !customMem.customFree) - return ZSTD_createCCtx(); + ZSTD_CCtx* ctx; - ZSTD_CCtx* ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + if (!customMem.customAlloc || !customMem.customFree) + { + ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); + if (!ctx) return NULL; + + ctx->customAlloc = malloc; + ctx->customFree = free; + return ctx; + } + + ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); if (!ctx) return NULL; memset(ctx, 0, sizeof(ZSTD_CCtx)); @@ -307,7 +312,9 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { if (srcCCtx->stage!=1) return ERROR(stage_wrong); - dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ + dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ + dstCCtx->customAlloc = srcCCtx->customAlloc; + dstCCtx->customFree = srcCCtx->customFree; ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 177e9c8a2..a3eee709e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -116,6 +116,8 @@ struct ZSTD_DCtx_s size_t expected; size_t headerSize; ZSTD_frameParams fParams; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 flagRepeatTable; @@ -143,15 +145,38 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) ZSTD_DCtx* ZSTD_createDCtx(void) { - ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx)); - if (dctx==NULL) return NULL; + ZSTD_customMem customMem = { NULL, NULL }; + return ZSTD_createDCtx_advanced(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_DCtx* dctx; + + if (!customMem.customAlloc || !customMem.customFree) + { + dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); + if (!dctx) return NULL; + dctx->customAlloc = malloc; + dctx->customFree = free; + + ZSTD_decompressBegin(dctx); + return dctx; + } + + dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); + if (!dctx) return NULL; + dctx->customAlloc = customMem.customAlloc; + dctx->customFree = customMem.customFree; + ZSTD_decompressBegin(dctx); return dctx; } + size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { - free(dctx); + dctx->customFree(dctx); return 0; /* reserved as a potential error code in the future */ } From 13ba880b4974817cca5962c0c94acbd1cc3aec16 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 23 May 2016 17:04:23 +0200 Subject: [PATCH 11/12] added ZBUFF_createCCtx_advanced and ZBUFF_createDCtx_advanced --- lib/common/zbuff.h | 5 +++- lib/compress/zbuff_compress.c | 45 +++++++++++++++++++++++------- lib/compress/zstd_compress.c | 5 +++- lib/decompress/zbuff_decompress.c | 46 ++++++++++++++++++++++++------- lib/decompress/zstd_decompress.c | 5 +++- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 3caa39caf..4d7f12a1f 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -38,7 +38,8 @@ extern "C" { /* ************************************* * Dependencies ***************************************/ -#include /* size_t */ +#include /* size_t */ +#include "zstd_static.h" /* ZSTD_customMem */ /* *************************************************************** @@ -60,6 +61,7 @@ extern "C" { ***************************************/ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); @@ -112,6 +114,7 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 260aca087..cabf447a4 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -95,14 +95,39 @@ struct ZBUFF_CCtx_s { size_t outBuffContentSize; size_t outBuffFlushedSize; ZBUFF_cStage stage; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; }; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ ZBUFF_CCtx* ZBUFF_createCCtx(void) { - ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx)); + ZSTD_customMem customMem = { NULL, NULL }; + return ZBUFF_createCCtx_advanced(customMem); +} + +ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_CCtx* zbc; + + if (!customMem.customAlloc && !customMem.customFree) + { + zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx)); + if (zbc==NULL) return NULL; + zbc->customAlloc = malloc; + zbc->customFree = free; + zbc->zc = ZSTD_createCCtx(); + return zbc; + } + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; - memset(zbc, 0, sizeof(*zbc)); - zbc->zc = ZSTD_createCCtx(); + memset(zbc, 0, sizeof(ZBUFF_CCtx)); + zbc->customAlloc = customMem.customAlloc; + zbc->customFree = customMem.customFree; + zbc->zc = ZSTD_createCCtx_advanced(customMem); return zbc; } @@ -110,9 +135,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) { if (zbc==NULL) return 0; /* support free on NULL */ ZSTD_freeCCtx(zbc->zc); - free(zbc->inBuff); - free(zbc->outBuff); - free(zbc); + zbc->customFree(zbc->inBuff); + zbc->customFree(zbc->outBuff); + zbc->customFree(zbc); return 0; } @@ -127,16 +152,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zbc->inBuffSize < neededInBuffSize) { zbc->inBuffSize = neededInBuffSize; - free(zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)malloc(neededInBuffSize); + zbc->customFree(zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); } if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; - free(zbc->outBuff); /* should not be necessary */ - zbc->outBuff = (char*)malloc(zbc->outBuffSize); + zbc->customFree(zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize); if (zbc->outBuff == NULL) return ERROR(memory_allocation); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 966d9acbd..bca725f54 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -128,7 +128,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ZSTD_CCtx* ctx; - if (!customMem.customAlloc || !customMem.customFree) + if (!customMem.customAlloc && !customMem.customFree) { ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); if (!ctx) return NULL; @@ -138,6 +138,9 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) return ctx; } + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); if (!ctx) return NULL; diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 858b442f7..d4ca550a7 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -82,15 +82,41 @@ struct ZBUFF_DCtx_s { size_t blockSize; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ ZBUFF_DCtx* ZBUFF_createDCtx(void) { - ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); + ZSTD_customMem customMem = { NULL, NULL }; + return ZBUFF_createDCtx_advanced(customMem); +} + +ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_DCtx* zbd; + + if (!customMem.customAlloc && !customMem.customFree) + { + zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx)); + if (zbd==NULL) return NULL; + zbd->customAlloc = malloc; + zbd->customFree = free; + zbd->zd = ZSTD_createDCtx(); + zbd->stage = ZBUFFds_init; + return zbd; + } + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx)); if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(*zbd)); - zbd->zd = ZSTD_createDCtx(); + memset(zbd, 0, sizeof(ZBUFF_DCtx)); + zbd->customAlloc = customMem.customAlloc; + zbd->customFree = customMem.customFree; + zbd->zd = ZSTD_createDCtx_advanced(customMem); zbd->stage = ZBUFFds_init; return zbd; } @@ -99,9 +125,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) { if (zbd==NULL) return 0; /* support free on null */ ZSTD_freeDCtx(zbd->zd); - free(zbd->inBuff); - free(zbd->outBuff); - free(zbd); + zbd->customFree(zbd->inBuff); + zbd->customFree(zbd->outBuff); + zbd->customFree(zbd); return 0; } @@ -170,16 +196,16 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; if (zbd->inBuffSize < blockSize) { - free(zbd->inBuff); + zbd->customFree(zbd->inBuff); zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)malloc(blockSize); + zbd->inBuff = (char*)zbd->customAlloc(blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; if (zbd->outBuffSize < neededOutSize) { - free(zbd->outBuff); + zbd->customFree(zbd->outBuff); zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)malloc(neededOutSize); + zbd->outBuff = (char*)zbd->customAlloc(neededOutSize); if (zbd->outBuff == NULL) return ERROR(memory_allocation); } } } zbd->stage = ZBUFFds_read; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index a3eee709e..b670c548c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -153,7 +153,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; - if (!customMem.customAlloc || !customMem.customFree) + if (!customMem.customAlloc && !customMem.customFree) { dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; @@ -164,6 +164,9 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) return dctx; } + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; dctx->customAlloc = customMem.customAlloc; From 71dd67cf57182c31686d1262b785512d2149c8ae Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 24 May 2016 10:57:14 +0200 Subject: [PATCH 12/12] fullbench: tests for ZBUFF_createCCtx_advanced and ZBUFF_createDCtx_advanced --- lib/common/mem.h | 7 +++-- lib/compress/.debug/zstd_stats.h | 7 ++++- lib/compress/zstd_compress.c | 11 ++++--- programs/fullbench.c | 51 +++++++++++++++++++++++++++++++- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 5abd83d1a..7f5a2ef7c 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -42,8 +42,11 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ -#include /* size_t, ptrdiff_t */ -#include /* memcpy */ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +#endif /*-**************************************** diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h index 51406a930..428514bfe 100644 --- a/lib/compress/.debug/zstd_stats.h +++ b/lib/compress/.debug/zstd_stats.h @@ -55,8 +55,11 @@ struct ZSTD_stats_s { /*-************************************* -* Advanced functions +* Stats functions ***************************************/ +MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc() { return malloc(sizeof(ZSTD_stats_t)); } +MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { free(stats); } + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); @@ -155,6 +158,8 @@ MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, co #else struct ZSTD_stats_s { U32 unused; }; + MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc(void) { return NULL; } + MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { (void)stats; } MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bca725f54..93efa99c9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2052,8 +2052,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = (ZSTD_stats_t*) zc->customAlloc(sizeof(ZSTD_stats_t)); - if (!stats) return ERROR(memory_allocation); + ZSTD_stats_t* stats = ZSTD_statsAlloc(); zc->seqStore.stats = stats; ZSTD_statsInit(stats); @@ -2061,7 +2060,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, size_t cSize; ZSTD_statsResetFreqs(stats); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { zc->customFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { ZSTD_statsFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { @@ -2072,11 +2071,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } + if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); - if (ZSTD_isError(cSize)) { zc->customFree(stats); return cSize; } + if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; } } else { op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); @@ -2092,7 +2091,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } ZSTD_statsPrint(stats, zc->params.cParams.searchLength); - zc->customFree(stats); + ZSTD_statsFree(stats); return op-ostart; } diff --git a/programs/fullbench.c b/programs/fullbench.c index 6f5d6a782..fe1abf1a2 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -213,6 +213,19 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, /*_******************************************************* * Bench functions *********************************************************/ +void* BMK_allocFunction(size_t size) +{ + void* address = malloc(size); + /* printf("alloc %p, %d \n", address, (int)size); */ + return address; +} + +void BMK_freeFunction(void* address) +{ +/* printf("free %p \n", address); */ + free(address); +} + static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { BYTE* dstBuff; @@ -220,6 +233,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) BYTE* buff2; const char* benchName; size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); + ZSTD_customMem customMem = { BMK_allocFunction, BMK_freeFunction }; double bestTime = 100000000.; /* Selection */ @@ -247,8 +261,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue"; break; case 42: + benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_createCCtx_advanced/ZBUFF_compressContinue"; + break; + case 43: benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue"; break; + case 44: + benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_createDCtx_advanced/ZBUFF_decompressContinue"; + break; default : return 0; } @@ -323,9 +343,16 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx(); break; case 42 : + if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx_advanced(customMem); + break; + case 43 : if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx(); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); break; + case 44 : + if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx_advanced(customMem); + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + break; /* test functions */ /* by convention, test functions can be added > 100 */ @@ -355,7 +382,29 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds; if (averageTime < bestTime) bestTime = averageTime; DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult); - }} + } + + /* free allocated structures */ + switch(benchNb) + { + case 11 : + if (g_zcc) { ZSTD_freeCCtx(g_zcc); g_zcc=NULL; } + break; + case 12 : + case 31: + case 32: + if (g_zdc) { ZSTD_freeDCtx(g_zdc); g_zdc=NULL; } + break; + case 41 : + case 42 : + if (g_zbcc) { ZBUFF_freeCCtx(g_zbcc); g_zbcc=NULL; } + break; + case 43 : + case 44 : + if (g_zbdc) { ZBUFF_freeDCtx(g_zbdc); g_zbdc=NULL; } + break; + default : ; + } } DISPLAY("%2u\n", benchNb); _cleanOut: