diff --git a/.travis.yml b/.travis.yml index a52d57af3..67da248d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ matrix: - env: Cmd='make arminstall && make aarch64fuzz' - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' + - env: Cmd='make -j uasanregressiontest' + - env: Cmd='make -j msanregressiontest' git: depth: 1 diff --git a/Makefile b/Makefile index e8bdcea33..7baff751c 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ ZSTDDIR = lib BUILDIR = build ZWRAPDIR = zlibWrapper TESTDIR = tests +FUZZDIR = $(TESTDIR)/fuzz # Define nul output VOID = /dev/null @@ -215,6 +216,15 @@ arm-ppc-compilation: $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" +regressiontest: + $(MAKE) -C $(FUZZDIR) regressiontest + +uasanregressiontest: + $(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined" + +msanregressiontest: + $(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory" + # run UBsan with -fsanitize-recover=signed-integer-overflow # due to a bug in UBsan when doing pointer subtraction # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303 diff --git a/circle.yml b/circle.yml index e89d548ac..5bc0ce643 100644 --- a/circle.yml +++ b/circle.yml @@ -45,7 +45,7 @@ test: parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make ppc64build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi #could add another test here + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi : parallel: true - ? | @@ -53,6 +53,11 @@ test: if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi : parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # Could add another test here + : + parallel: true post: - echo Circle CI tests finished diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 60822d498..6d2a0cfa9 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -14,6 +14,13 @@ CPPFLAGS ?= LDFLAGS ?= ARFLAGS ?= LIB_FUZZING_ENGINE ?= libregression.a +PYTHON ?= python +ifeq ($(shell uname), Darwin) + DOWNLOAD?=curl -L -o +else + DOWNLOAD?=wget -O +endif +CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/ ZSTDDIR = ../../lib PRGDIR = ../../programs @@ -48,18 +55,20 @@ FUZZ_SRC := \ FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC))) -.PHONY: default all clean +.PHONY: default all clean cleanall default: all -all: \ +FUZZ_TARGETS := \ simple_round_trip \ stream_round_trip \ - block_round_trip \ + block_round_trip \ simple_decompress \ stream_decompress \ block_decompress +all: $(FUZZ_TARGETS) + %.o: %.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@ @@ -93,7 +102,25 @@ libFuzzer: @git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer @cd Fuzzer && ./build.sh +corpora/%_seed_corpus.zip: + @mkdir -p corpora + $(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip + +corpora/%: corpora/%_seed_corpus.zip + unzip -q $^ -d $@ + +.PHONY: corpora +corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS)) + +regressiontest: corpora + CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all + $(PYTHON) ./fuzz.py regression all + clean: @$(MAKE) -C $(ZSTDDIR) clean @$(RM) -f *.a *.o @$(RM) -f simple_round_trip stream_round_trip simple_decompress stream_decompress + +cleanall: + @$(RM) -rf Fuzzer + @$(RM) -rf corpora diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 6d0fab556..f184be646 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -1,6 +1,14 @@ # Fuzzing Each fuzzing target can be built with multiple engines. +Zstd provides a fuzz corpus for each target that can be downloaded with +the command: + +``` +make corpora +``` + +It will download each corpus into `./corpora/TARGET`. ## fuzz.py diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index 0ce201cdd..9864d822d 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -82,11 +82,39 @@ def tmpdir(): shutil.rmtree(dirpath, ignore_errors=True) +def parse_targets(in_targets): + targets = set() + for target in in_targets: + if not target: + continue + if target == 'all': + targets = targets.union(TARGETS) + elif target in TARGETS: + targets.add(target) + else: + raise RuntimeError('{} is not a valid target'.format(target)) + return list(targets) + + +def targets_parser(args, description): + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + 'TARGET', + nargs='*', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + args.TARGET = parse_targets(args.TARGET) + + return args + + def parse_env_flags(args, flags): """ Look for flags set by environment variables. """ - flags = ' '.join(flags) san_flags = ','.join(re.findall('-fsanitize=((?:[a-z]+,?)+)', flags)) nosan_flags = ','.join(re.findall('-fno-sanitize=((?:[a-z]+,?)+)', flags)) @@ -112,6 +140,34 @@ def parse_env_flags(args, flags): return args +def compiler_version(cc, cxx): + """ + Determines the compiler and version. + Only works for clang and gcc. + """ + cc_version_bytes = subprocess.check_output([cc, "--version"]) + cxx_version_bytes = subprocess.check_output([cxx, "--version"]) + if cc_version_bytes.startswith(b'clang'): + assert(cxx_version_bytes.startswith(b'clang')) + compiler = 'clang' + if cc_version_bytes.startswith(b'gcc'): + assert(cxx_version_bytes.startswith(b'g++')) + compiler = 'gcc' + version_regex = b'([0-9])+\.([0-9])+\.([0-9])+' + version_match = re.search(version_regex, cc_version_bytes) + version = tuple(int(version_match.group(i)) for i in range(1, 4)) + return compiler, version + + +def overflow_ubsan_flags(cc, cxx): + compiler, version = compiler_version(cc, cxx) + if compiler == 'gcc': + return ['-fno-sanitize=signed-integer-overflow'] + if compiler == 'clang' and version >= (5, 0, 0): + return ['-fno-sanitize=pointer-overflow'] + return [] + + def build_parser(args): description = """ Cleans the repository and builds a fuzz target (or all). @@ -336,7 +392,7 @@ def build(args): if args.ubsan: ubsan_flags = ['-fsanitize=undefined'] if not args.ubsan_pointer_overflow: - ubsan_flags += ['-fno-sanitize=pointer-overflow'] + ubsan_flags += overflow_ubsan_flags(cc, cxx) common_flags += ubsan_flags if args.stateful_fuzzing: @@ -424,36 +480,42 @@ def libfuzzer_parser(args): if args.TARGET and args.TARGET not in TARGETS: raise RuntimeError('{} is not a valid target'.format(args.TARGET)) - if not args.corpora: - args.corpora = abs_join(CORPORA_DIR, args.TARGET) - if not args.artifact: - args.artifact = abs_join(CORPORA_DIR, '{}-crash'.format(args.TARGET)) - if not args.seed: - args.seed = abs_join(CORPORA_DIR, '{}-seed'.format(args.TARGET)) - return args -def libfuzzer(args): - try: - args = libfuzzer_parser(args) - except Exception as e: - print(e) - return 1 - target = abs_join(FUZZ_DIR, args.TARGET) +def libfuzzer(target, corpora=None, artifact=None, seed=None, extra_args=None): + if corpora is None: + corpora = abs_join(CORPORA_DIR, target) + if artifact is None: + artifact = abs_join(CORPORA_DIR, '{}-crash'.format(target)) + if seed is None: + seed = abs_join(CORPORA_DIR, '{}-seed'.format(target)) + if extra_args is None: + extra_args = [] - corpora = [create(args.corpora)] - artifact = create(args.artifact) - seed = check(args.seed) + target = abs_join(FUZZ_DIR, target) + + corpora = [create(corpora)] + artifact = create(artifact) + seed = check(seed) corpora += [artifact] if seed is not None: corpora += [seed] cmd = [target, '-artifact_prefix={}/'.format(artifact)] - cmd += corpora + args.extra + cmd += corpora + extra_args print(' '.join(cmd)) - subprocess.call(cmd) + subprocess.check_call(cmd) + + +def libfuzzer_cmd(args): + try: + args = libfuzzer_parser(args) + except Exception as e: + print(e) + return 1 + libfuzzer(args.TARGET, args.corpora, args.artifact, args.seed, args.extra) return 0 @@ -518,39 +580,15 @@ def afl(args): return 0 -def regression_parser(args): - description = """ - Runs one or more regression tests. - The fuzzer should have been built with with - LIB_FUZZING_ENGINE='libregression.a'. - Takes input from CORPORA. - """ - parser = argparse.ArgumentParser(prog=args.pop(0), description=description) - parser.add_argument( - 'TARGET', - nargs='*', - type=str, - help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS))) - args = parser.parse_args(args) - - targets = set() - for target in args.TARGET: - if not target: - continue - if target == 'all': - targets = targets.union(TARGETS) - elif target in TARGETS: - targets.add(target) - else: - raise RuntimeError('{} is not a valid target'.format(target)) - args.TARGET = list(targets) - - return args - - def regression(args): try: - args = regression_parser(args) + description = """ + Runs one or more regression tests. + The fuzzer should have been built with with + LIB_FUZZING_ENGINE='libregression.a'. + Takes input from CORPORA. + """ + args = targets_parser(args, description) except Exception as e: print(e) return 1 @@ -673,6 +711,52 @@ def gen(args): return 0 +def minimize(args): + try: + description = """ + Runs a libfuzzer fuzzer with -merge=1 to build a minimal corpus in + TARGET_seed_corpus. All extra args are passed to libfuzzer. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + + for target in args.TARGET: + # Merge the corpus + anything else into the seed_corpus + corpus = abs_join(CORPORA_DIR, target) + seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) + extra_args = [corpus, "-merge=1"] + args.extra + libfuzzer(target, corpora=seed_corpus, extra_args=extra_args) + seeds = set(os.listdir(seed_corpus)) + # Copy all crashes directly into the seed_corpus if not already present + crashes = abs_join(CORPORA_DIR, '{}-crash'.format(target)) + for crash in os.listdir(crashes): + if crash not in seeds: + shutil.copy(abs_join(crashes, crash), seed_corpus) + seeds.add(crash) + + +def zip_cmd(args): + try: + description = """ + Zips up the seed corpus. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + + for target in args.TARGET: + # Zip the seed_corpus + seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) + seeds = [abs_join(seed_corpus, f) for f in os.listdir(seed_corpus)] + zip_file = "{}.zip".format(seed_corpus) + cmd = ["zip", "-q", "-j", "-9", zip_file] + print(' '.join(cmd + [abs_join(seed_corpus, '*')])) + subprocess.check_call(cmd + seeds) + + def short_help(args): name = args[0] print("Usage: {} [OPTIONS] COMMAND [ARGS]...\n".format(name)) @@ -690,6 +774,8 @@ def help(args): print("\tafl\t\tRun an AFL fuzzer") print("\tregression\tRun a regression test") print("\tgen\t\tGenerate a seed corpus for a fuzzer") + print("\tminimize\tMinimize the test corpora") + print("\tzip\t\tZip the minimized corpora up") def main(): @@ -705,13 +791,17 @@ def main(): if command == "build": return build(args) if command == "libfuzzer": - return libfuzzer(args) + return libfuzzer_cmd(args) if command == "regression": return regression(args) if command == "afl": return afl(args) if command == "gen": return gen(args) + if command == "minimize": + return minimize(args) + if command == "zip": + return zip_cmd(args) short_help(args) print("Error: No such command {} (pass -h for help)".format(command)) return 1 diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index f853485ad..617e45df6 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -38,10 +38,11 @@ static size_t roundTripTest(void *result, size_t resultCapacity, if (FUZZ_rand(&seed) & 1) { ZSTD_inBuffer in = {src, srcSize, 0}; ZSTD_outBuffer out = {compressed, compressedCapacity, 0}; + size_t err; ZSTD_CCtx_reset(cctx); FUZZ_setRandomParameters(cctx, &seed); - size_t const err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); if (err != 0) { return err; }