ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[cmake] Add playTests.sh as a test

Browse Source
This commit is contained in:
Nick Terrell
2020-01-13 14:16:15 -08:00
committed by Nick Terrell
parent f27b4b4ec5
commit 3ed0f65158
9 changed files with 86 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1 -1
View File
@@ -84,7 +84,7 @@ matrix:
- make clean - make clean
- make -j check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - make -j check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
- name: cmake build check # ~2mn - name: cmake build and test check # ~2mn
script: script:
- make cmakebuild - make cmakebuild
Makefile
+6 -2
View File
@@ -344,7 +344,7 @@ endif
ifneq (,$(filter MSYS%,$(shell uname))) ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
endif endif
@@ -356,7 +356,11 @@ cmakebuild:
cmake --version cmake --version
$(RM) -r $(BUILDIR)/cmake/build $(RM) -r $(BUILDIR)/cmake/build
mkdir $(BUILDIR)/cmake/build mkdir $(BUILDIR)/cmake/build
cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
$(MAKE) -C $(BUILDIR)/cmake/build -j4;
$(MAKE) -C $(BUILDIR)/cmake/build install;
$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
cd $(BUILDIR)/cmake/build; ctest -V
c89build: clean c89build: clean
$(CC) -v $(CC) -v
appveyor.yml
+2 -1
View File
@@ -249,8 +249,9 @@
C:\cygwin64\bin\bash --login -c " C:\cygwin64\bin\bash --login -c "
set -e; set -e;
cd build/cmake; cd build/cmake;
CFLAGS='-Werror' cmake -G 'Unix Makefiles' .; CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON .;
make -j4; make -j4;
ctest -V;
" "
) )
- if [%HOST%]==[mingw] ( - if [%HOST%]==[mingw] (
build/cmake/CMakeLists.txt
+3
View File
@@ -23,6 +23,8 @@ else()
endif() endif()
cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION}) cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib) set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
@@ -137,6 +139,7 @@ if (ZSTD_BUILD_PROGRAMS)
endif () endif ()
if (ZSTD_BUILD_TESTS) if (ZSTD_BUILD_TESTS)
enable_testing()
if (NOT ZSTD_BUILD_STATIC) if (NOT ZSTD_BUILD_STATIC)
message(SEND_ERROR "You need to build static library to build tests") message(SEND_ERROR "You need to build static library to build tests")
endif () endif ()
build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
+8
View File
@@ -33,6 +33,10 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
EnableCompilerFlag("-Wcast-align" true true) EnableCompilerFlag("-Wcast-align" true true)
EnableCompilerFlag("-Wcast-qual" true true) EnableCompilerFlag("-Wcast-qual" true true)
EnableCompilerFlag("-Wstrict-prototypes" true false) EnableCompilerFlag("-Wstrict-prototypes" true false)
# Enable asserts in Debug mode
if (CMAKE_BUILD_TYPE MATCHES "Debug")
EnableCompilerFlag("-DDEBUGLEVEL=1" true true)
endif ()
elseif (MSVC) # Add specific compilation flags for Windows Visual elseif (MSVC) # Add specific compilation flags for Windows Visual
set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)") set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
@@ -43,6 +47,10 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
# UNICODE SUPPORT # UNICODE SUPPORT
EnableCompilerFlag("/D_UNICODE" true true) EnableCompilerFlag("/D_UNICODE" true true)
EnableCompilerFlag("/DUNICODE" true true) EnableCompilerFlag("/DUNICODE" true true)
# Enable asserts in Debug mode
if (CMAKE_BUILD_TYPE MATCHES "Debug")
EnableCompilerFlag("/DDEBUGLEVEL=1" true true)
endif ()
endif () endif ()
# Remove duplicates compilation flags # Remove duplicates compilation flags
build/cmake/tests/CMakeLists.txt
+4
View File
@@ -52,6 +52,10 @@ target_link_libraries(fuzzer libzstd_static)
add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c) add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
target_link_libraries(zstreamtest libzstd_static) target_link_libraries(zstreamtest libzstd_static)
add_test(
NAME playTests
COMMAND sh -c "ZSTD_BIN='$<TARGET_FILE:zstd>' DATAGEN_BIN='$<TARGET_FILE:datagen>' '${TESTS_DIR}/playTests.sh'")
if (UNIX) if (UNIX)
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c) add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
target_link_libraries(paramgrill libzstd_static m) #m is math library target_link_libraries(paramgrill libzstd_static m) #m is math library
tests/Makefile
+1 -1
View File
@@ -333,7 +333,7 @@ test-zstd-nolegacy: zstd-nolegacy
test-zstd test-zstd32 test-zstd-nolegacy: datagen test-zstd test-zstd32 test-zstd-nolegacy: datagen
file $(ZSTD) file $(ZSTD)
EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" ./playTests.sh $(ZSTDRTTEST) EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
test-gzstd: gzstd test-gzstd: gzstd
tests/fuzzer.c
+1
View File
@@ -25,6 +25,7 @@
#include <stdlib.h> /* free */ #include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */ #include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */ #include <string.h> /* strcmp */
#undef NDEBUG
#include <assert.h> #include <assert.h>
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "fse.h" #include "fse.h"
tests/playTests.sh
+60 -58
View File
@@ -22,9 +22,9 @@ roundTripTest() {
fi fi
rm -f tmp1 tmp2 rm -f tmp1 tmp2
println "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel" println "roundTripTest: $DATAGEN $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
./datagen $1 $proba | $MD5SUM > tmp1 $DATAGEN $1 $proba | $MD5SUM > tmp1
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2 $DATAGEN $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2 $DIFF -q tmp1 tmp2
} }
@@ -43,8 +43,8 @@ fileRoundTripTest() {
fi fi
rm -f tmp.zstd tmp.md5.1 tmp.md5.2 rm -f tmp.zstd tmp.md5.1 tmp.md5.2
println "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d" println "fileRoundTripTest: $DATAGEN $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
./datagen $1 $local_p > tmp $DATAGEN $1 $local_p > tmp
< tmp $MD5SUM > tmp.md5.1 < tmp $MD5SUM > tmp.md5.1
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2 $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
$DIFF -q tmp.md5.1 tmp.md5.2 $DIFF -q tmp.md5.1 tmp.md5.2
@@ -105,8 +105,10 @@ esac
println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN'" println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN'"
[ -n "$ZSTD_BIN" ] || die "ZSTD_BIN variable must be defined!" [ -n "$ZSTD_BIN" ] || die "ZSTD_BIN variable must be defined!"
[ -n "$DATAGEN_BIN" ] || die "DATAGEN_BIN variable must be defined!"
ZSTD="$EXE_PREFIX $ZSTD_BIN" ZSTD="$EXE_PREFIX $ZSTD_BIN"
DATAGEN="$DATAGEN_BIN"
if echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' if echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
then then
@@ -119,7 +121,7 @@ fi
println "\n===> simple tests " println "\n===> simple tests "
./datagen > tmp $DATAGEN > tmp
println "test : basic compression " println "test : basic compression "
$ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -f tmp # trivial compression case, creates tmp.zst
println "test : basic decompression" println "test : basic decompression"
@@ -172,7 +174,7 @@ fi
println "test : null-length file roundtrip" println "test : null-length file roundtrip"
println -n '' | $ZSTD - --stdout | $ZSTD -d --stdout println -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
println "test : ensure small file doesn't add 3-bytes null block" println "test : ensure small file doesn't add 3-bytes null block"
./datagen -g1 > tmp1 $DATAGEN -g1 > tmp1
$ZSTD tmp1 -c | wc -c | grep "14" $ZSTD tmp1 -c | wc -c | grep "14"
$ZSTD < tmp1 | wc -c | grep "14" $ZSTD < tmp1 | wc -c | grep "14"
println "test : decompress file with wrong suffix (must fail)" println "test : decompress file with wrong suffix (must fail)"
@@ -226,11 +228,11 @@ $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
println "\n===> --exclude-compressed flag" println "\n===> --exclude-compressed flag"
rm -rf precompressedFilterTestDir rm -rf precompressedFilterTestDir
mkdir -p precompressedFilterTestDir mkdir -p precompressedFilterTestDir
./datagen $size > precompressedFilterTestDir/input.5 $DATAGEN $size > precompressedFilterTestDir/input.5
./datagen $size > precompressedFilterTestDir/input.6 $DATAGEN $size > precompressedFilterTestDir/input.6
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
./datagen $size > precompressedFilterTestDir/input.7 $DATAGEN $size > precompressedFilterTestDir/input.7
./datagen $size > precompressedFilterTestDir/input.8 $DATAGEN $size > precompressedFilterTestDir/input.8
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
test ! -f precompressedFilterTestDir/input.5.zst.zst test ! -f precompressedFilterTestDir/input.5.zst.zst
test ! -f precompressedFilterTestDir/input.6.zst.zst test ! -f precompressedFilterTestDir/input.6.zst.zst
@@ -242,7 +244,7 @@ else
println "Test is not successful" println "Test is not successful"
fi fi
# File Extension check. # File Extension check.
./datagen $size > precompressedFilterTestDir/input.zstbar $DATAGEN $size > precompressedFilterTestDir/input.zstbar
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
# ZSTD should compress input.zstbar # ZSTD should compress input.zstbar
test -f precompressedFilterTestDir/input.zstbar.zst test -f precompressedFilterTestDir/input.zstbar.zst
@@ -318,9 +320,9 @@ if [ "$?" -eq 139 ]; then
die "should not have segfaulted" die "should not have segfaulted"
fi fi
println "\n===> multiple files and shell completion " println "\n===> multiple files and shell completion "
./datagen -s1 > tmp1 2> $INTOVOID $DATAGEN -s1 > tmp1 2> $INTOVOID
./datagen -s2 -g100K > tmp2 2> $INTOVOID $DATAGEN -s2 -g100K > tmp2 2> $INTOVOID
./datagen -s3 -g1M > tmp3 2> $INTOVOID $DATAGEN -s3 -g1M > tmp3 2> $INTOVOID
println "compress tmp* : " println "compress tmp* : "
$ZSTD -f tmp* $ZSTD -f tmp*
test -f tmp1.zst test -f tmp1.zst
@@ -361,7 +363,7 @@ if [ -n "$DEVNULLRIGHTS" ] ; then
# these tests requires sudo rights, which is uncommon. # these tests requires sudo rights, which is uncommon.
# they are only triggered if DEVNULLRIGHTS macro is defined. # they are only triggered if DEVNULLRIGHTS macro is defined.
println "\n===> checking /dev/null permissions are unaltered " println "\n===> checking /dev/null permissions are unaltered "
./datagen > tmp $DATAGEN > tmp
sudo $ZSTD tmp -o $INTOVOID # sudo rights could modify /dev/null permissions sudo $ZSTD tmp -o $INTOVOID # sudo rights could modify /dev/null permissions
sudo $ZSTD tmp -c > $INTOVOID sudo $ZSTD tmp -c > $INTOVOID
$ZSTD tmp -f -o tmp.zst $ZSTD tmp -f -o tmp.zst
@@ -442,11 +444,11 @@ test -f tmp3
test -f tmp4 test -f tmp4
println "test : survive a list of files which is text garbage (--filelist=FILE)" println "test : survive a list of files which is text garbage (--filelist=FILE)"
./datagen > tmp_badList $DATAGEN > tmp_badList
$ZSTD -f --filelist=tmp_badList && die "should have failed : list is text garbage" $ZSTD -f --filelist=tmp_badList && die "should have failed : list is text garbage"
println "test : survive a list of files which is binary garbage (--filelist=FILE)" println "test : survive a list of files which is binary garbage (--filelist=FILE)"
./datagen -P0 -g1M > tmp_badList $DATAGEN -P0 -g1M > tmp_badList
$ZSTD -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console $ZSTD -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console
println "test : try to overflow internal list of files (--filelist=FILE)" println "test : try to overflow internal list of files (--filelist=FILE)"
@@ -550,7 +552,7 @@ fi
println "\n===> test sparse file support " println "\n===> test sparse file support "
./datagen -g5M -P100 > tmpSparse $DATAGEN -g5M -P100 > tmpSparse
$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
$DIFF -s tmpSparse tmpSparseRegen $DIFF -s tmpSparse tmpSparseRegen
$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse $ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse
@@ -558,14 +560,14 @@ $DIFF -s tmpSparse tmpOutSparse
$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse $ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
$DIFF -s tmpSparse tmpOutNoSparse $DIFF -s tmpSparse tmpOutNoSparse
ls -ls tmpSparse* # look at file size and block size on disk ls -ls tmpSparse* # look at file size and block size on disk
./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) $DATAGEN -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd $DATAGEN -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
ls -ls tmpSparseOdd # look at file size and block size on disk ls -ls tmpSparseOdd # look at file size and block size on disk
println "\n Sparse Compatibility with Console :" println "\n Sparse Compatibility with Console :"
println "Hello World 1 !" | $ZSTD | $ZSTD -d -c println "Hello World 1 !" | $ZSTD | $ZSTD -d -c
println "Hello World 2 !" | $ZSTD | $ZSTD -d | cat println "Hello World 2 !" | $ZSTD | $ZSTD -d | cat
println "\n Sparse Compatibility with Append :" println "\n Sparse Compatibility with Append :"
./datagen -P100 -g1M > tmpSparse1M $DATAGEN -P100 -g1M > tmpSparse1M
cat tmpSparse1M tmpSparse1M > tmpSparse2M cat tmpSparse1M tmpSparse1M > tmpSparse2M
$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed $ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated $ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
@@ -577,7 +579,7 @@ rm tmpSparse*
println "\n===> stream-size mode" println "\n===> stream-size mode"
./datagen -g11000 > tmp $DATAGEN -g11000 > tmp
println "test : basic file compression vs sized streaming compression" println "test : basic file compression vs sized streaming compression"
file_size=$($ZSTD -14 -f tmp -o tmp.zst && wc -c < tmp.zst) file_size=$($ZSTD -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
stream_size=$(cat tmp | $ZSTD -14 --stream-size=11000 | wc -c) stream_size=$(cat tmp | $ZSTD -14 --stream-size=11000 | wc -c)
@@ -610,9 +612,9 @@ rm -rf tmp*
println "\n===> size-hint mode" println "\n===> size-hint mode"
./datagen -g11000 > tmp $DATAGEN -g11000 > tmp
./datagen -g11000 > tmp2 $DATAGEN -g11000 > tmp2
./datagen > tmpDict $DATAGEN > tmpDict
println "test : basic file compression vs hinted streaming compression" println "test : basic file compression vs hinted streaming compression"
file_size=$($ZSTD -14 -f tmp -o tmp.zst && wc -c < tmp.zst) file_size=$($ZSTD -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
stream_size=$(cat tmp | $ZSTD -14 --size-hint=11000 | wc -c) stream_size=$(cat tmp | $ZSTD -14 --size-hint=11000 | wc -c)
@@ -641,9 +643,9 @@ cat tmp | $ZSTD -14 -f --size-hint=5500 | $ZSTD -t # considerably too low
println "\n===> dictionary tests " println "\n===> dictionary tests "
println "- test with raw dict (content only) " println "- test with raw dict (content only) "
./datagen > tmpDict $DATAGEN > tmpDict
./datagen -g1M | $MD5SUM > tmp1 $DATAGEN -g1M | $MD5SUM > tmp1
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2 $DATAGEN -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2 $DIFF -q tmp1 tmp2
println "- Create first dictionary " println "- Create first dictionary "
TESTFILE="$PRGDIR"/zstdcli.c TESTFILE="$PRGDIR"/zstdcli.c
@@ -662,7 +664,7 @@ $DIFF "$TESTFILE" result
if [ -n "$hasMT" ] if [ -n "$hasMT" ]
then then
println "- Test dictionary compression with multithreading " println "- Test dictionary compression with multithreading "
./datagen -g5M | $ZSTD -T2 -D tmpDict | $ZSTD -t -D tmpDict # fails with v1.3.2 $DATAGEN -g5M | $ZSTD -T2 -D tmpDict | $ZSTD -t -D tmpDict # fails with v1.3.2
fi fi
println "- Create second (different) dictionary " println "- Create second (different) dictionary "
$ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
@@ -701,7 +703,7 @@ rm -rf dirTestDict
println "- dictionary builder on bogus input" println "- dictionary builder on bogus input"
println "Hello World" > tmp println "Hello World" > tmp
$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source"
./datagen -P0 -g10M > tmp $DATAGEN -P0 -g10M > tmp
$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise"
println "- Test -o before --train" println "- Test -o before --train"
rm -f tmpDict dictionary rm -f tmpDict dictionary
@@ -723,7 +725,7 @@ rm tmp* dictionary
println "\n===> fastCover dictionary builder : advanced options " println "\n===> fastCover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c TESTFILE="$PRGDIR"/zstdcli.c
./datagen > tmpDict $DATAGEN > tmpDict
println "- Create first dictionary" println "- Create first dictionary"
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp cp "$TESTFILE" tmp
@@ -766,7 +768,7 @@ rm tmp* dictionary
println "\n===> legacy dictionary builder " println "\n===> legacy dictionary builder "
TESTFILE="$PRGDIR"/zstdcli.c TESTFILE="$PRGDIR"/zstdcli.c
./datagen > tmpDict $DATAGEN > tmpDict
println "- Create first dictionary" println "- Create first dictionary"
$ZSTD --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict $ZSTD --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp cp "$TESTFILE" tmp
@@ -794,7 +796,7 @@ rm tmp* dictionary
println "\n===> integrity tests " println "\n===> integrity tests "
println "test one file (tmp1.zst) " println "test one file (tmp1.zst) "
./datagen > tmp1 $DATAGEN > tmp1
$ZSTD tmp1 $ZSTD tmp1
$ZSTD -t tmp1.zst $ZSTD -t tmp1.zst
$ZSTD --test tmp1.zst $ZSTD --test tmp1.zst
@@ -805,14 +807,14 @@ $ZSTD -t ./* && die "bad files not detected !"
$ZSTD -t tmp1 && die "bad file not detected !" $ZSTD -t tmp1 && die "bad file not detected !"
cp tmp1 tmp2.zst cp tmp1 tmp2.zst
$ZSTD -t tmp2.zst && die "bad file not detected !" $ZSTD -t tmp2.zst && die "bad file not detected !"
./datagen -g0 > tmp3 $DATAGEN -g0 > tmp3
$ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad $ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad
println "test --rm and --test combined " println "test --rm and --test combined "
$ZSTD -t --rm tmp1.zst $ZSTD -t --rm tmp1.zst
test -f tmp1.zst # check file is still present test -f tmp1.zst # check file is still present
split -b16384 tmp1.zst tmpSplit. split -b16384 tmp1.zst tmpSplit.
$ZSTD -t tmpSplit.* && die "bad file not detected !" $ZSTD -t tmpSplit.* && die "bad file not detected !"
./datagen | $ZSTD -c | $ZSTD -t $DATAGEN | $ZSTD -c | $ZSTD -t
println "\n===> golden files tests " println "\n===> golden files tests "
@@ -824,7 +826,7 @@ $ZSTD -c -r "$TESTDIR/golden-compression" | $ZSTD -t
println "\n===> benchmark mode tests " println "\n===> benchmark mode tests "
println "bench one file" println "bench one file"
./datagen > tmp1 $DATAGEN > tmp1
$ZSTD -bi0 tmp1 $ZSTD -bi0 tmp1
println "bench multiple levels" println "bench multiple levels"
$ZSTD -i0b0e3 tmp1 $ZSTD -i0b0e3 tmp1
@@ -839,7 +841,7 @@ $ZSTD -b -d -i0 tmp1.zst
println "\n===> zstd compatibility tests " println "\n===> zstd compatibility tests "
./datagen > tmp $DATAGEN > tmp
rm -f tmp.zst rm -f tmp.zst
$ZSTD --format=zstd -f tmp $ZSTD --format=zstd -f tmp
test -f tmp.zst test -f tmp.zst
@@ -854,7 +856,7 @@ if [ $GZIPMODE -eq 1 ]; then
GZIPEXE=1 GZIPEXE=1
gzip -V || GZIPEXE=0 gzip -V || GZIPEXE=0
if [ $GZIPEXE -eq 1 ]; then if [ $GZIPEXE -eq 1 ]; then
./datagen > tmp $DATAGEN > tmp
$ZSTD --format=gzip -f tmp $ZSTD --format=gzip -f tmp
gzip -t -v tmp.gz gzip -t -v tmp.gz
gzip -f tmp gzip -f tmp
@@ -871,7 +873,7 @@ fi
println "\n===> gzip frame tests " println "\n===> gzip frame tests "
if [ $GZIPMODE -eq 1 ]; then if [ $GZIPMODE -eq 1 ]; then
./datagen > tmp $DATAGEN > tmp
$ZSTD -f --format=gzip tmp $ZSTD -f --format=gzip tmp
$ZSTD -f tmp $ZSTD -f tmp
cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
@@ -882,7 +884,7 @@ else
fi fi
if [ $GZIPMODE -eq 1 ]; then if [ $GZIPMODE -eq 1 ]; then
./datagen > tmp $DATAGEN > tmp
rm -f tmp.zst rm -f tmp.zst
$ZSTD --format=gzip --format=zstd -f tmp $ZSTD --format=gzip --format=zstd -f tmp
test -f tmp.zst test -f tmp.zst
@@ -898,7 +900,7 @@ if [ $LZMAMODE -eq 1 ]; then
xz -Q -V && lzma -Q -V || XZEXE=0 xz -Q -V && lzma -Q -V || XZEXE=0
if [ $XZEXE -eq 1 ]; then if [ $XZEXE -eq 1 ]; then
println "Testing zstd xz and lzma support" println "Testing zstd xz and lzma support"
./datagen > tmp $DATAGEN > tmp
$ZSTD --format=lzma -f tmp $ZSTD --format=lzma -f tmp
$ZSTD --format=xz -f tmp $ZSTD --format=xz -f tmp
xz -Q -t -v tmp.xz xz -Q -t -v tmp.xz
@@ -914,7 +916,7 @@ if [ $LZMAMODE -eq 1 ]; then
ln -s $ZSTD ./lzma ln -s $ZSTD ./lzma
ln -s $ZSTD ./unlzma ln -s $ZSTD ./unlzma
println "Testing xz and lzma symlinks" println "Testing xz and lzma symlinks"
./datagen > tmp $DATAGEN > tmp
./xz tmp ./xz tmp
xz -Q -d tmp.xz xz -Q -d tmp.xz
./lzma tmp ./lzma tmp
@@ -937,7 +939,7 @@ fi
println "\n===> xz frame tests " println "\n===> xz frame tests "
if [ $LZMAMODE -eq 1 ]; then if [ $LZMAMODE -eq 1 ]; then
./datagen > tmp $DATAGEN > tmp
$ZSTD -f --format=xz tmp $ZSTD -f --format=xz tmp
$ZSTD -f --format=lzma tmp $ZSTD -f --format=lzma tmp
$ZSTD -f tmp $ZSTD -f tmp
@@ -958,7 +960,7 @@ if [ $LZ4MODE -eq 1 ]; then
LZ4EXE=1 LZ4EXE=1
lz4 -V || LZ4EXE=0 lz4 -V || LZ4EXE=0
if [ $LZ4EXE -eq 1 ]; then if [ $LZ4EXE -eq 1 ]; then
./datagen > tmp $DATAGEN > tmp
$ZSTD --format=lz4 -f tmp $ZSTD --format=lz4 -f tmp
lz4 -t -v tmp.lz4 lz4 -t -v tmp.lz4
lz4 -f tmp lz4 -f tmp
@@ -974,7 +976,7 @@ fi
if [ $LZ4MODE -eq 1 ]; then if [ $LZ4MODE -eq 1 ]; then
println "\n===> lz4 frame tests " println "\n===> lz4 frame tests "
./datagen > tmp $DATAGEN > tmp
$ZSTD -f --format=lz4 tmp $ZSTD -f --format=lz4 tmp
$ZSTD -f tmp $ZSTD -f tmp
cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp
@@ -1007,7 +1009,7 @@ println "\n===> tar extension tests "
rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4
./datagen > tmp $DATAGEN > tmp
tar cf tmp.tar tmp tar cf tmp.tar tmp
$ZSTD tmp.tar -o tmp.tzst $ZSTD tmp.tar -o tmp.tzst
rm tmp.tar rm tmp.tar
@@ -1077,7 +1079,7 @@ then
roundTripTest -g8M "3 --long=24 -T2" roundTripTest -g8M "3 --long=24 -T2"
println "\n===> ovLog tests " println "\n===> ovLog tests "
./datagen -g2MB > tmp $DATAGEN -g2MB > tmp
refSize=$($ZSTD tmp -6 -c --zstd=wlog=18 | wc -c) refSize=$($ZSTD tmp -6 -c --zstd=wlog=18 | wc -c)
ov9Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) ov9Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c)
ov1Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) ov1Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c)
@@ -1101,9 +1103,9 @@ fi
rm tmp* rm tmp*
println "\n===> zstd --list/-l single frame tests " println "\n===> zstd --list/-l single frame tests "
./datagen > tmp1 $DATAGEN > tmp1
./datagen > tmp2 $DATAGEN > tmp2
./datagen > tmp3 $DATAGEN > tmp3
$ZSTD tmp* $ZSTD tmp*
$ZSTD -l ./*.zst $ZSTD -l ./*.zst
$ZSTD -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header $ZSTD -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header
@@ -1126,7 +1128,7 @@ println "test : detect truncated compressed file "
TEST_DATA_FILE=truncatable-input.txt TEST_DATA_FILE=truncatable-input.txt
FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst
TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst
./datagen -g50000 > $TEST_DATA_FILE $DATAGEN -g50000 > $TEST_DATA_FILE
$ZSTD -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE $ZSTD -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE
dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE
$ZSTD --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" $ZSTD --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file"
@@ -1145,7 +1147,7 @@ $ZSTD -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin"
$ZSTD -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty $ZSTD -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty
println "\n===> zstd --list/-l test with null files " println "\n===> zstd --list/-l test with null files "
./datagen -g0 > tmp5 $DATAGEN -g0 > tmp5
$ZSTD tmp5 $ZSTD tmp5
$ZSTD -l tmp5.zst $ZSTD -l tmp5.zst
$ZSTD -l tmp5* && die "-l must fail on non-zstd file" $ZSTD -l tmp5* && die "-l must fail on non-zstd file"
@@ -1153,7 +1155,7 @@ $ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 siz
$ZSTD -lv tmp5* && die "-l must fail on non-zstd file" $ZSTD -lv tmp5* && die "-l must fail on non-zstd file"
println "\n===> zstd --list/-l test with no content size field " println "\n===> zstd --list/-l test with no content size field "
./datagen -g513K | $ZSTD > tmp6.zst $DATAGEN -g513K | $ZSTD > tmp6.zst
$ZSTD -l tmp6.zst $ZSTD -l tmp6.zst
$ZSTD -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" $ZSTD -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file"
@@ -1192,7 +1194,7 @@ then
roundTripTest -g270000000 " --adapt" roundTripTest -g270000000 " --adapt"
roundTripTest -g27000000 " --adapt=min=1,max=4" roundTripTest -g27000000 " --adapt=min=1,max=4"
println "===> test: --adapt must fail on incoherent bounds " println "===> test: --adapt must fail on incoherent bounds "
./datagen > tmp $DATAGEN > tmp
$ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" $ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
println "\n===> rsyncable mode " println "\n===> rsyncable mode "
@@ -1204,8 +1206,8 @@ fi
println "\n===> patch-from tests" println "\n===> patch-from tests"
./datagen -g1000 -P50 > tmp_dict $DATAGEN -g1000 -P50 > tmp_dict
./datagen -g1000 -P10 > tmp_patch $DATAGEN -g1000 -P10 > tmp_patch
$ZSTD --memory=10000 --patch-from=tmp_dict tmp_patch -o tmp_patch_diff $ZSTD --memory=10000 --patch-from=tmp_dict tmp_patch -o tmp_patch_diff
$ZSTD -d --memory=10000 --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon $ZSTD -d --memory=10000 --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch $DIFF -s tmp_patch_recon tmp_patch
@@ -1273,7 +1275,7 @@ fi
println "\n===> cover dictionary builder : advanced options " println "\n===> cover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c TESTFILE="$PRGDIR"/zstdcli.c
./datagen > tmpDict $DATAGEN > tmpDict
println "- Create first dictionary" println "- Create first dictionary"
$ZSTD --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict $ZSTD --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp cp "$TESTFILE" tmp