CI failure fixes

This commit is contained in:
Danielle Rozenblit
2022-10-11 13:12:19 -07:00
parent a70ca2bd7d
commit 8888a2ddcc
48 changed files with 944 additions and 235 deletions
+1
View File
@@ -12,6 +12,7 @@ zstreamtest
zstreamtest32
zstreamtest_asan
zstreamtest_tsan
zstreamtest_ubsan
zstreamtest-dll
datagen
paramgrill
+47 -24
View File
@@ -1,5 +1,5 @@
# ################################################################
# ################################################################
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
@@ -37,7 +37,7 @@ TESTARTEFACT := versionsTest
DEBUGFLAGS += -g -Wno-c++-compat
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
-DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
-DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES))
ZSTDCOMP_FILES := $(sort $(ZSTD_COMPRESS_FILES))
@@ -134,14 +134,17 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S
$(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@
FULLBENCHS := fullbench fullbench32
CLEAN += $(FULLBENCHS)
fullbench32: CPPFLAGS += -m32
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
fullbench fullbench32 : $(ZSTD_FILES)
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(FULLBENCHS) : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
$(FULLBENCHS) : LDFLAGS += $(MULTITHREAD_LD)
$(FULLBENCHS) : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
$(FULLBENCHS) : $(ZSTD_FILES)
$(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(LINK.c) $^ -o $@$(EXT)
CLEAN += fullbench-lib
fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c
$(LINK.c) $^ -o $@$(EXT)
@@ -151,6 +154,7 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR
# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
$(LINK.c) $^ $(LDLIBS) -o $@$(EXT)
CLEAN += fuzzer fuzzer32
fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
fuzzer : LDFLAGS += $(MULTITHREAD_LD)
fuzzer : $(ZSTDMT_OBJECTS)
@@ -164,6 +168,7 @@ fuzzer32 : $(ZSTD_FILES)
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
CLEAN += zstreamtest zstreamtest32
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
@@ -175,42 +180,58 @@ zstreamtest32 : $(ZSTREAMFILES)
zstreamtest zstreamtest32 :
$(LINK.c) $^ -o $@$(EXT)
CLEAN += zstreamtest_asan
zstreamtest_asan : CFLAGS += -fsanitize=address
zstreamtest_asan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
CLEAN += zstreamtest_tsan
zstreamtest_tsan : CFLAGS += -fsanitize=thread
zstreamtest_tsan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
CLEAN += zstreamtest_ubsan
zstreamtest_ubsan : CFLAGS += -fsanitize=undefined
zstreamtest_ubsan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
# note : broken : requires symbols unavailable from dynamic library
zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll
zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
CLEAN += paramgrill
paramgrill : DEBUGFLAGS = # turn off debug for speed measurements
paramgrill : LDLIBS += -lm
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
CLEAN += datagen
datagen : $(PRGDIR)/datagen.c datagencli.c
$(LINK.c) $^ -o $@$(EXT)
CLEAN += roundTripCrash
roundTripCrash: CFLAGS += $(MULTITHREAD)
roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c
CLEAN += longmatch
longmatch : $(ZSTD_OBJECTS) longmatch.c
CLEAN += bigdict
bigdict: CFLAGS += $(MULTITHREAD)
bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c
CLEAN += invalidDictionaries
invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
CLEAN += legacy
legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
legacy : $(ZSTD_FILES) $(sort $(wildcard $(ZSTDDIR)/legacy/*.c)) legacy.c
CLEAN += decodecorpus
decodecorpus : LDLIBS += -lm
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c
CLEAN += poolTests
poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
@@ -222,7 +243,8 @@ versionsTest: clean
automated_benchmarking: clean
$(PYTHON) automated_benchmarking.py
# make checkTag
# make checkTag : check that release tag corresponds to release version
CLEAN += checkTag
checkTag.o : $(ZSTDDIR)/zstd.h
.PHONY: clean
@@ -231,28 +253,22 @@ clean:
$(MAKE) -C $(PRGDIR) clean
$(RM) -fR $(TESTARTEFACT)
$(RM) -rf tmp* # some test directories are named tmp*
$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
$(RM) $(CLEAN) core *.o *.tmp result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) \
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \
zstreamtest$(EXT) zstreamtest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
decodecorpus$(EXT) checkTag$(EXT) bigdict$(EXT)
fullbench-dll$(EXT) fuzzer-dll$(EXT) zstreamtest-dll$(EXT)
@echo Cleaning completed
#----------------------------------------------------------------------------------
# valgrind tests are validated only for some posix platforms
# valgrind tests validated only for some posix platforms
#----------------------------------------------------------------------------------
UNAME := $(shell uname)
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX))
HOST_OS = POSIX
valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
valgrindTest: zstd datagen fuzzer fullbench
.PHONY: test-valgrind
test-valgrind: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
test-valgrind: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
$(VALGRIND) ./datagen -g50M > $(VOID)
$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
@@ -261,7 +277,7 @@ valgrindTest: zstd datagen fuzzer fullbench
./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID)
./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
@rm tmp
$(RM) tmp
$(VALGRIND) ./fuzzer -T1mn -t1
$(VALGRIND) ./fullbench -i1
@@ -306,9 +322,9 @@ endif
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
.PHONY: test-all
test-all: test test32 valgrindTest test-decodecorpus-cli
test-all: test test32 test-decodecorpus-cli
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
test-zstd: ZSTD = $(PRGDIR)/zstd
test-zstd: zstd
@@ -322,29 +338,36 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen
file $(ZSTD)
EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
.PHONY: test-cli-tests
test-cli-tests: ZSTD = $(PRGDIR)/zstd
test-cli-tests: zstd datagen
file $(ZSTD)
./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
.PHONY: test-fullbench
test-fullbench: fullbench datagen
$(QEMU_SYS) ./fullbench -i1
$(QEMU_SYS) ./fullbench -i1 -P0
.PHONY: test-fullbench32
test-fullbench32: fullbench32 datagen
$(QEMU_SYS) ./fullbench32 -i1
$(QEMU_SYS) ./fullbench32 -i1 -P0
.PHONY: test-fuzzer
test-fuzzer: fuzzer
$(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)
# Note : this test presumes `fuzzer` will be built
.PHONY: test-fuzzer-stackmode
test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0
test-fuzzer-stackmode: test-fuzzer
.PHONY: test-fuzzer32
test-fuzzer32: fuzzer32
$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
.PHONY: test-zstream
test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+2 -1
View File
@@ -7,7 +7,8 @@ Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standa
-d, --decompress decompression
-f, --force disable input and output checks. Allows overwriting existing files,
input from console, output to stdout, operating on links,
block devices, etc.
block devices, etc. During decompression and when the output
destination is stdout, pass-through unrecognized formats as-is.
--rm remove source file(s) after successful de/compression
-k, --keep preserve source file(s) (default)
-D DICT use DICT as Dictionary for compression or decompression
+40
View File
@@ -0,0 +1,40 @@
#!/bin/sh
echo "some data" > file
println "+ zstd --memory=32LB file"
zstd --memory=32LB file && die "Should not allow bogus suffix"
println "+ zstd --memory=32LiB file"
zstd --memory=32LiB file && die "Should not allow bogus suffix"
println "+ zstd --memory=32A file"
zstd --memory=32A file && die "Should not allow bogus suffix"
println "+ zstd --memory=32r82347dn83 file"
zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
println "+ zstd --memory=32asbdf file"
zstd --memory=32asbdf file && die "Should not allow bogus suffix"
println "+ zstd --memory=hello file"
zstd --memory=hello file && die "Should not allow non-numeric parameter"
println "+ zstd --memory=1 file"
zstd --memory=1 file && die "Should allow numeric parameter without suffix"
rm file.zst
println "+ zstd --memory=1K file"
zstd --memory=1K file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1KB file"
zstd --memory=1KB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1KiB file"
zstd --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1M file"
zstd --memory=1M file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1MB file"
zstd --memory=1MB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1MiB file"
zstd --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
rm file
exit 0
@@ -0,0 +1,13 @@
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
Should allow numeric parameter without suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
@@ -0,0 +1,13 @@
+ zstd --memory=32LB file
+ zstd --memory=32LiB file
+ zstd --memory=32A file
+ zstd --memory=32r82347dn83 file
+ zstd --memory=32asbdf file
+ zstd --memory=hello file
+ zstd --memory=1 file
+ zstd --memory=1K file
+ zstd --memory=1KB file
+ zstd --memory=1KiB file
+ zstd --memory=1M file
+ zstd --memory=1MB file
+ zstd --memory=1MiB file
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -e
. "$COMMON/platform.sh"
zstd < file -vv -19 -o file.19.zst
zstd -vv -l file.19.zst
zstd < file -vv -19 --long -o file.19.long.zst
zstd -vv -l file.19.long.zst
@@ -0,0 +1,5 @@
...
*wlog=23*
...
*wlog=27*
...
@@ -0,0 +1,5 @@
...
*Window Size: 8388608 B*
...
*Window Size: 134217728 B*
...
+57
View File
@@ -0,0 +1,57 @@
#!/bin/sh
set -e
. "$COMMON/platform.sh"
echo "" > 1
echo "2" > 2
echo "23" > 3
echo "234" > 4
echo "some data" > file
println "+ passthrough enabled"
zstd file
# Test short files
zstd -dc --pass-through 1 2 3 4
# Test *cat symlinks
zstdcat file
"$ZSTD_SYMLINK_DIR/zcat" file
"$ZSTD_SYMLINK_DIR/gzcat" file
# Test multiple files with mix of compressed & not
zstdcat file file.zst
zstdcat file.zst file
# Test --pass-through
zstd -dc --pass-through file
zstd -d --pass-through file -o pass-through-file
# Test legacy implicit passthrough with -fc
zstd -dcf file
zstd -dcf file file.zst
zstd -df < file
zstd -dcf < file file.zst -
zstd -dcf < file.zst file -
$DIFF file pass-through-file
println "+ passthrough disabled"
# Test *cat
zstdcat --no-pass-through file && die "should fail"
"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
# Test zstd without implicit passthrough
zstd -d file -o no-pass-through-file && die "should fail"
zstd -d < file && die "should fail"
# Test legacy implicit passthrough with -fc
zstd --no-pass-through -dcf file && die "should fail"
zstd --no-pass-through -dcf file file.zst && die "should fail"
zstd --no-pass-through -df < file && die "should fail"
zstd --no-pass-through -dcf < file file.zst - && die "should fail"
zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:
@@ -0,0 +1,10 @@
zstd: file: unsupported format
zstd: file: unsupported format
zstd: file: unsupported format
zstd: file: unsupported format
zstd: /*stdin*\: unsupported format
zstd: file: unsupported format
zstd: file: unsupported format
zstd: /*stdin*\: unsupported format
zstd: /*stdin*\: unsupported format
zstd: file: unsupported format
@@ -0,0 +1,25 @@
+ passthrough enabled
2
23
234
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
+ passthrough disabled
some data
some data
some data
+3 -1
View File
@@ -83,7 +83,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
HUF_DTable* dt = (HUF_DTable*)FUZZ_malloc(HUF_DTABLE_SIZE(tableLog) * sizeof(HUF_DTable));
dt[0] = tableLog * 0x01000001;
tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol);
HUF_depth_mode depthMode = rand() & 1 ? HUF_depth_fast : HUF_depth_optimal;
tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, depthMode);
FUZZ_ASSERT(tableLog <= 12);
tableLog = HUF_buildCTable_wksp(ct, count, maxSymbol, tableLog, wksp, wkspSize);
FUZZ_ZASSERT(tableLog);
+34 -3
View File
@@ -522,7 +522,7 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
inBuff.src = NULL;
inBuff.size = 0;
inBuff.pos = 0;
@@ -548,6 +548,36 @@ static int basicUnitTests(U32 seed, double compressibility)
{ size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ret != 0) goto _output_error;
}
DISPLAYLEVEL(3, "OK\n");
DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
{
const char* test = "aa";
inBuff.src = test;
inBuff.size = 2;
inBuff.pos = 0;
outBuff.dst = NULL;
outBuff.size = 0;
outBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
CHECK_Z( ZSTD_endStream(zc, &outBuff) );
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_endStream(zc, &outBuff);
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
}
inBuff.src = outBuff.dst;
inBuff.size = outBuff.pos;
inBuff.pos = 0;
outBuff.dst = NULL;
outBuff.size = 0;
outBuff.pos = 0;
CHECK_Z( ZSTD_initDStream(zd) );
CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
}
DISPLAYLEVEL(3, "OK\n");
/* _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
@@ -1944,8 +1974,9 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
U32 n;
for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
cSize=0;
totalTestSize=0;
while(totalTestSize < maxTestSize) {
/* compress random chunks into randomly sized dst buffers */
{ size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);