ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #861 from terrelln/zip

Browse Source 
[fuzz][CI] Set up regression tests
This commit is contained in:
Yann Collet
2017-09-25 17:11:35 -07:00
committed by GitHub
parent 586df82a78 917a213254
commit 8a4d0abc1c
7 changed files with 201 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2
View File
@@ -21,6 +21,8 @@ matrix:
- env: Cmd='make arminstall && make aarch64fuzz'
- env: Cmd='make ppcinstall && make ppcfuzz'
- env: Cmd='make ppcinstall && make ppc64fuzz'
- env: Cmd='make -j uasanregressiontest'
- env: Cmd='make -j msanregressiontest'
git:
depth: 1
Makefile
+10
View File
@@ -12,6 +12,7 @@ ZSTDDIR = lib
BUILDIR = build
ZWRAPDIR = zlibWrapper
TESTDIR = tests
FUZZDIR = $(TESTDIR)/fuzz
# Define nul output
VOID = /dev/null
@@ -215,6 +216,15 @@ arm-ppc-compilation:
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
regressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest
uasanregressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
msanregressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
# run UBsan with -fsanitize-recover=signed-integer-overflow
# due to a bug in UBsan when doing pointer subtraction
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
circle.yml
+6 -1
View File
@@ -45,7 +45,7 @@ test:
parallel: true
- ? |
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make ppc64build && make clean; fi &&
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi #could add another test here
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi
:
parallel: true
- ? |
@@ -53,6 +53,11 @@ test:
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
:
parallel: true
- ? |
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi &&
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # Could add another test here
:
parallel: true
post:
- echo Circle CI tests finished
tests/fuzz/Makefile
+30 -3
View File
@@ -14,6 +14,13 @@ CPPFLAGS ?=
LDFLAGS ?=
ARFLAGS ?=
LIB_FUZZING_ENGINE ?= libregression.a
PYTHON ?= python
ifeq ($(shell uname), Darwin)
DOWNLOAD?=curl -L -o
else
DOWNLOAD?=wget -O
endif
CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/
ZSTDDIR = ../../lib
PRGDIR = ../../programs
@@ -48,18 +55,20 @@ FUZZ_SRC := \
FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC)))
.PHONY: default all clean
.PHONY: default all clean cleanall
default: all
all: \
FUZZ_TARGETS := \
simple_round_trip \
stream_round_trip \
block_round_trip \
block_round_trip \
simple_decompress \
stream_decompress \
block_decompress
all: $(FUZZ_TARGETS)
%.o: %.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@
@@ -93,7 +102,25 @@ libFuzzer:
@git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer
@cd Fuzzer && ./build.sh
corpora/%_seed_corpus.zip:
@mkdir -p corpora
$(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip
corpora/%: corpora/%_seed_corpus.zip
unzip -q $^ -d $@
.PHONY: corpora
corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS))
regressiontest: corpora
CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all
$(PYTHON) ./fuzz.py regression all
clean:
@$(MAKE) -C $(ZSTDDIR) clean
@$(RM) -f *.a *.o
@$(RM) -f simple_round_trip stream_round_trip simple_decompress stream_decompress
cleanall:
@$(RM) -rf Fuzzer
@$(RM) -rf corpora
tests/fuzz/README.md
+8
View File
@@ -1,6 +1,14 @@
# Fuzzing
Each fuzzing target can be built with multiple engines.
Zstd provides a fuzz corpus for each target that can be downloaded with
the command:
```
make corpora
```
It will download each corpus into `./corpora/TARGET`.
## fuzz.py
tests/fuzz/fuzz.py
+143 -53
View File
@@ -82,11 +82,39 @@ def tmpdir():
shutil.rmtree(dirpath, ignore_errors=True)
def parse_targets(in_targets):
targets = set()
for target in in_targets:
if not target:
continue
if target == 'all':
targets = targets.union(TARGETS)
elif target in TARGETS:
targets.add(target)
else:
raise RuntimeError('{} is not a valid target'.format(target))
return list(targets)
def targets_parser(args, description):
parser = argparse.ArgumentParser(prog=args.pop(0), description=description)
parser.add_argument(
'TARGET',
nargs='*',
type=str,
help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)))
args, extra = parser.parse_known_args(args)
args.extra = extra
args.TARGET = parse_targets(args.TARGET)
return args
def parse_env_flags(args, flags):
"""
Look for flags set by environment variables.
"""
flags = ' '.join(flags)
san_flags = ','.join(re.findall('-fsanitize=((?:[a-z]+,?)+)', flags))
nosan_flags = ','.join(re.findall('-fno-sanitize=((?:[a-z]+,?)+)', flags))
@@ -112,6 +140,34 @@ def parse_env_flags(args, flags):
return args
def compiler_version(cc, cxx):
"""
Determines the compiler and version.
Only works for clang and gcc.
"""
cc_version_bytes = subprocess.check_output([cc, "--version"])
cxx_version_bytes = subprocess.check_output([cxx, "--version"])
if cc_version_bytes.startswith(b'clang'):
assert(cxx_version_bytes.startswith(b'clang'))
compiler = 'clang'
if cc_version_bytes.startswith(b'gcc'):
assert(cxx_version_bytes.startswith(b'g++'))
compiler = 'gcc'
version_regex = b'([0-9])+\.([0-9])+\.([0-9])+'
version_match = re.search(version_regex, cc_version_bytes)
version = tuple(int(version_match.group(i)) for i in range(1, 4))
return compiler, version
def overflow_ubsan_flags(cc, cxx):
compiler, version = compiler_version(cc, cxx)
if compiler == 'gcc':
return ['-fno-sanitize=signed-integer-overflow']
if compiler == 'clang' and version >= (5, 0, 0):
return ['-fno-sanitize=pointer-overflow']
return []
def build_parser(args):
description = """
Cleans the repository and builds a fuzz target (or all).
@@ -336,7 +392,7 @@ def build(args):
if args.ubsan:
ubsan_flags = ['-fsanitize=undefined']
if not args.ubsan_pointer_overflow:
ubsan_flags += ['-fno-sanitize=pointer-overflow']
ubsan_flags += overflow_ubsan_flags(cc, cxx)
common_flags += ubsan_flags
if args.stateful_fuzzing:
@@ -424,36 +480,42 @@ def libfuzzer_parser(args):
if args.TARGET and args.TARGET not in TARGETS:
raise RuntimeError('{} is not a valid target'.format(args.TARGET))
if not args.corpora:
args.corpora = abs_join(CORPORA_DIR, args.TARGET)
if not args.artifact:
args.artifact = abs_join(CORPORA_DIR, '{}-crash'.format(args.TARGET))
if not args.seed:
args.seed = abs_join(CORPORA_DIR, '{}-seed'.format(args.TARGET))
return args
def libfuzzer(args):
try:
args = libfuzzer_parser(args)
except Exception as e:
print(e)
return 1
target = abs_join(FUZZ_DIR, args.TARGET)
def libfuzzer(target, corpora=None, artifact=None, seed=None, extra_args=None):
if corpora is None:
corpora = abs_join(CORPORA_DIR, target)
if artifact is None:
artifact = abs_join(CORPORA_DIR, '{}-crash'.format(target))
if seed is None:
seed = abs_join(CORPORA_DIR, '{}-seed'.format(target))
if extra_args is None:
extra_args = []
corpora = [create(args.corpora)]
artifact = create(args.artifact)
seed = check(args.seed)
target = abs_join(FUZZ_DIR, target)
corpora = [create(corpora)]
artifact = create(artifact)
seed = check(seed)
corpora += [artifact]
if seed is not None:
corpora += [seed]
cmd = [target, '-artifact_prefix={}/'.format(artifact)]
cmd += corpora + args.extra
cmd += corpora + extra_args
print(' '.join(cmd))
subprocess.call(cmd)
subprocess.check_call(cmd)
def libfuzzer_cmd(args):
try:
args = libfuzzer_parser(args)
except Exception as e:
print(e)
return 1
libfuzzer(args.TARGET, args.corpora, args.artifact, args.seed, args.extra)
return 0
@@ -518,39 +580,15 @@ def afl(args):
return 0
def regression_parser(args):
description = """
Runs one or more regression tests.
The fuzzer should have been built with with
LIB_FUZZING_ENGINE='libregression.a'.
Takes input from CORPORA.
"""
parser = argparse.ArgumentParser(prog=args.pop(0), description=description)
parser.add_argument(
'TARGET',
nargs='*',
type=str,
help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)))
args = parser.parse_args(args)
targets = set()
for target in args.TARGET:
if not target:
continue
if target == 'all':
targets = targets.union(TARGETS)
elif target in TARGETS:
targets.add(target)
else:
raise RuntimeError('{} is not a valid target'.format(target))
args.TARGET = list(targets)
return args
def regression(args):
try:
args = regression_parser(args)
description = """
Runs one or more regression tests.
The fuzzer should have been built with with
LIB_FUZZING_ENGINE='libregression.a'.
Takes input from CORPORA.
"""
args = targets_parser(args, description)
except Exception as e:
print(e)
return 1
@@ -673,6 +711,52 @@ def gen(args):
return 0
def minimize(args):
try:
description = """
Runs a libfuzzer fuzzer with -merge=1 to build a minimal corpus in
TARGET_seed_corpus. All extra args are passed to libfuzzer.
"""
args = targets_parser(args, description)
except Exception as e:
print(e)
return 1
for target in args.TARGET:
# Merge the corpus + anything else into the seed_corpus
corpus = abs_join(CORPORA_DIR, target)
seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target))
extra_args = [corpus, "-merge=1"] + args.extra
libfuzzer(target, corpora=seed_corpus, extra_args=extra_args)
seeds = set(os.listdir(seed_corpus))
# Copy all crashes directly into the seed_corpus if not already present
crashes = abs_join(CORPORA_DIR, '{}-crash'.format(target))
for crash in os.listdir(crashes):
if crash not in seeds:
shutil.copy(abs_join(crashes, crash), seed_corpus)
seeds.add(crash)
def zip_cmd(args):
try:
description = """
Zips up the seed corpus.
"""
args = targets_parser(args, description)
except Exception as e:
print(e)
return 1
for target in args.TARGET:
# Zip the seed_corpus
seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target))
seeds = [abs_join(seed_corpus, f) for f in os.listdir(seed_corpus)]
zip_file = "{}.zip".format(seed_corpus)
cmd = ["zip", "-q", "-j", "-9", zip_file]
print(' '.join(cmd + [abs_join(seed_corpus, '*')]))
subprocess.check_call(cmd + seeds)
def short_help(args):
name = args[0]
print("Usage: {} [OPTIONS] COMMAND [ARGS]...\n".format(name))
@@ -690,6 +774,8 @@ def help(args):
print("\tafl\t\tRun an AFL fuzzer")
print("\tregression\tRun a regression test")
print("\tgen\t\tGenerate a seed corpus for a fuzzer")
print("\tminimize\tMinimize the test corpora")
print("\tzip\t\tZip the minimized corpora up")
def main():
@@ -705,13 +791,17 @@ def main():
if command == "build":
return build(args)
if command == "libfuzzer":
return libfuzzer(args)
return libfuzzer_cmd(args)
if command == "regression":
return regression(args)
if command == "afl":
return afl(args)
if command == "gen":
return gen(args)
if command == "minimize":
return minimize(args)
if command == "zip":
return zip_cmd(args)
short_help(args)
print("Error: No such command {} (pass -h for help)".format(command))
return 1
tests/fuzz/simple_round_trip.c
+2 -1
View File
@@ -38,10 +38,11 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
if (FUZZ_rand(&seed) & 1) {
ZSTD_inBuffer in = {src, srcSize, 0};
ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
size_t err;
ZSTD_CCtx_reset(cctx);
FUZZ_setRandomParameters(cctx, &seed);
size_t const err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
if (err != 0) {
return err;
}