Merge pull request #861 from terrelln/zip

[fuzz][CI] Set up regression tests

.travis.yml

@@ -21,6 +21,8 @@ matrix:
     - env: Cmd='make arminstall && make aarch64fuzz'
     - env: Cmd='make ppcinstall && make ppcfuzz'
     - env: Cmd='make ppcinstall && make ppc64fuzz'
+    - env: Cmd='make -j uasanregressiontest'
+    - env: Cmd='make -j msanregressiontest'
 
 git:
   depth: 1

Makefile

@@ -12,6 +12,7 @@ ZSTDDIR  = lib
 BUILDIR  = build
 ZWRAPDIR = zlibWrapper
 TESTDIR  = tests
+FUZZDIR  = $(TESTDIR)/fuzz
 
 # Define nul output
 VOID = /dev/null
@@ -215,6 +216,15 @@ arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
+regressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest
+
+uasanregressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
+
+msanregressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
+
 # run UBsan with -fsanitize-recover=signed-integer-overflow
 # due to a bug in UBsan when doing pointer subtraction
 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303

circle.yml

@@ -45,7 +45,7 @@ test:
         parallel: true
     - ? |
         if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make ppc64build   && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build    && make clean; fi #could add another test here
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build    && make clean; fi
       :
         parallel: true
     - ? |
@@ -53,6 +53,11 @@ test:
         if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
       :
         parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make -j regressiontest && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # Could add another test here
+      :
+        parallel: true
 
   post:
     - echo Circle CI tests finished

tests/fuzz/Makefile

@@ -14,6 +14,13 @@ CPPFLAGS ?=
 LDFLAGS ?=
 ARFLAGS ?=
 LIB_FUZZING_ENGINE ?= libregression.a
+PYTHON ?= python
+ifeq ($(shell uname), Darwin)
+	DOWNLOAD?=curl -L -o
+else
+	DOWNLOAD?=wget -O
+endif
+CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/
 
 ZSTDDIR = ../../lib
 PRGDIR = ../../programs
@@ -48,18 +55,20 @@ FUZZ_SRC       := \
 FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC)))
 
 
-.PHONY: default all clean
+.PHONY: default all clean cleanall
 
 default: all
 
-all: \
+FUZZ_TARGETS :=       \
 	simple_round_trip \
 	stream_round_trip \
-	block_round_trip \
+	block_round_trip  \
 	simple_decompress \
 	stream_decompress \
 	block_decompress
 
+all: $(FUZZ_TARGETS)
+
 %.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@
 
@@ -93,7 +102,25 @@ libFuzzer:
 	@git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer
 	@cd Fuzzer && ./build.sh
 
+corpora/%_seed_corpus.zip:
+	@mkdir -p corpora
+	$(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip
+
+corpora/%: corpora/%_seed_corpus.zip
+	unzip -q $^ -d $@
+
+.PHONY: corpora
+corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS))
+
+regressiontest: corpora
+	CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all
+	$(PYTHON) ./fuzz.py regression all
+
 clean:
 	@$(MAKE) -C $(ZSTDDIR) clean
 	@$(RM) -f *.a *.o
 	@$(RM) -f simple_round_trip stream_round_trip simple_decompress stream_decompress
+
+cleanall:
+	@$(RM) -rf Fuzzer
+	@$(RM) -rf corpora

tests/fuzz/README.md

@@ -1,6 +1,14 @@
 # Fuzzing
 
 Each fuzzing target can be built with multiple engines.
+Zstd provides a fuzz corpus for each target that can be downloaded with
+the command:
+
+```
+make corpora
+```
+
+It will download each corpus into `./corpora/TARGET`.
 
 ## fuzz.py
 

tests/fuzz/fuzz.py

@@ -82,11 +82,39 @@ def tmpdir():
         shutil.rmtree(dirpath, ignore_errors=True)
 
 
+def parse_targets(in_targets):
+    targets = set()
+    for target in in_targets:
+        if not target:
+            continue
+        if target == 'all':
+            targets = targets.union(TARGETS)
+        elif target in TARGETS:
+            targets.add(target)
+        else:
+            raise RuntimeError('{} is not a valid target'.format(target))
+    return list(targets)
+
+
+def targets_parser(args, description):
+    parser = argparse.ArgumentParser(prog=args.pop(0), description=description)
+    parser.add_argument(
+        'TARGET',
+        nargs='*',
+        type=str,
+        help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)))
+    args, extra = parser.parse_known_args(args)
+    args.extra = extra
+
+    args.TARGET = parse_targets(args.TARGET)
+
+    return args
+
+
 def parse_env_flags(args, flags):
     """
     Look for flags set by environment variables.
     """
-    flags = ' '.join(flags)
     san_flags = ','.join(re.findall('-fsanitize=((?:[a-z]+,?)+)', flags))
     nosan_flags = ','.join(re.findall('-fno-sanitize=((?:[a-z]+,?)+)', flags))
 
@@ -112,6 +140,34 @@ def parse_env_flags(args, flags):
     return args
 
 
+def compiler_version(cc, cxx):
+    """
+    Determines the compiler and version.
+    Only works for clang and gcc.
+    """
+    cc_version_bytes = subprocess.check_output([cc, "--version"])
+    cxx_version_bytes = subprocess.check_output([cxx, "--version"])
+    if cc_version_bytes.startswith(b'clang'):
+        assert(cxx_version_bytes.startswith(b'clang'))
+        compiler = 'clang'
+    if cc_version_bytes.startswith(b'gcc'):
+        assert(cxx_version_bytes.startswith(b'g++'))
+        compiler = 'gcc'
+    version_regex = b'([0-9])+\.([0-9])+\.([0-9])+'
+    version_match = re.search(version_regex, cc_version_bytes)
+    version = tuple(int(version_match.group(i)) for i in range(1, 4))
+    return compiler, version
+
+
+def overflow_ubsan_flags(cc, cxx):
+    compiler, version = compiler_version(cc, cxx)
+    if compiler == 'gcc':
+        return ['-fno-sanitize=signed-integer-overflow']
+    if compiler == 'clang' and version >= (5, 0, 0):
+        return ['-fno-sanitize=pointer-overflow']
+    return []
+
+
 def build_parser(args):
     description = """
     Cleans the repository and builds a fuzz target (or all).
@@ -336,7 +392,7 @@ def build(args):
     if args.ubsan:
         ubsan_flags = ['-fsanitize=undefined']
         if not args.ubsan_pointer_overflow:
-            ubsan_flags += ['-fno-sanitize=pointer-overflow']
+            ubsan_flags += overflow_ubsan_flags(cc, cxx)
         common_flags += ubsan_flags
 
     if args.stateful_fuzzing:
@@ -424,36 +480,42 @@ def libfuzzer_parser(args):
     if args.TARGET and args.TARGET not in TARGETS:
         raise RuntimeError('{} is not a valid target'.format(args.TARGET))
 
-    if not args.corpora:
-        args.corpora = abs_join(CORPORA_DIR, args.TARGET)
-    if not args.artifact:
-        args.artifact = abs_join(CORPORA_DIR, '{}-crash'.format(args.TARGET))
-    if not args.seed:
-        args.seed = abs_join(CORPORA_DIR, '{}-seed'.format(args.TARGET))
-
     return args
 
 
-def libfuzzer(args):
+def libfuzzer(target, corpora=None, artifact=None, seed=None, extra_args=None):
-    try:
+    if corpora is None:
-        args = libfuzzer_parser(args)
+        corpora = abs_join(CORPORA_DIR, target)
-    except Exception as e:
+    if artifact is None:
-        print(e)
+        artifact = abs_join(CORPORA_DIR, '{}-crash'.format(target))
-        return 1
+    if seed is None:
-    target = abs_join(FUZZ_DIR, args.TARGET)
+        seed = abs_join(CORPORA_DIR, '{}-seed'.format(target))
+    if extra_args is None:
+        extra_args = []
 
-    corpora = [create(args.corpora)]
+    target = abs_join(FUZZ_DIR, target)
-    artifact = create(args.artifact)
+
-    seed = check(args.seed)
+    corpora = [create(corpora)]
+    artifact = create(artifact)
+    seed = check(seed)
 
     corpora += [artifact]
     if seed is not None:
         corpora += [seed]
 
     cmd = [target, '-artifact_prefix={}/'.format(artifact)]
-    cmd += corpora + args.extra
+    cmd += corpora + extra_args
     print(' '.join(cmd))
-    subprocess.call(cmd)
+    subprocess.check_call(cmd)
+
+
+def libfuzzer_cmd(args):
+    try:
+        args = libfuzzer_parser(args)
+    except Exception as e:
+        print(e)
+        return 1
+    libfuzzer(args.TARGET, args.corpora, args.artifact, args.seed, args.extra)
     return 0
 
 
@@ -518,39 +580,15 @@ def afl(args):
     return 0
 
 
-def regression_parser(args):
-    description = """
-    Runs one or more regression tests.
-    The fuzzer should have been built with with
-    LIB_FUZZING_ENGINE='libregression.a'.
-    Takes input from CORPORA.
-    """
-    parser = argparse.ArgumentParser(prog=args.pop(0), description=description)
-    parser.add_argument(
-        'TARGET',
-        nargs='*',
-        type=str,
-        help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)))
-    args = parser.parse_args(args)
-
-    targets = set()
-    for target in args.TARGET:
-        if not target:
-            continue
-        if target == 'all':
-            targets = targets.union(TARGETS)
-        elif target in TARGETS:
-            targets.add(target)
-        else:
-            raise RuntimeError('{} is not a valid target'.format(target))
-    args.TARGET = list(targets)
-
-    return args
-
-
 def regression(args):
     try:
-        args = regression_parser(args)
+        description = """
+        Runs one or more regression tests.
+        The fuzzer should have been built with with
+        LIB_FUZZING_ENGINE='libregression.a'.
+        Takes input from CORPORA.
+        """
+        args = targets_parser(args, description)
     except Exception as e:
         print(e)
         return 1
@@ -673,6 +711,52 @@ def gen(args):
     return 0
 
 
+def minimize(args):
+    try:
+        description = """
+        Runs a libfuzzer fuzzer with -merge=1 to build a minimal corpus in
+        TARGET_seed_corpus. All extra args are passed to libfuzzer.
+        """
+        args = targets_parser(args, description)
+    except Exception as e:
+        print(e)
+        return 1
+
+    for target in args.TARGET:
+        # Merge the corpus + anything else into the seed_corpus
+        corpus = abs_join(CORPORA_DIR, target)
+        seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target))
+        extra_args = [corpus, "-merge=1"] + args.extra
+        libfuzzer(target, corpora=seed_corpus, extra_args=extra_args)
+        seeds = set(os.listdir(seed_corpus))
+        # Copy all crashes directly into the seed_corpus if not already present
+        crashes = abs_join(CORPORA_DIR, '{}-crash'.format(target))
+        for crash in os.listdir(crashes):
+            if crash not in seeds:
+                shutil.copy(abs_join(crashes, crash), seed_corpus)
+                seeds.add(crash)
+
+
+def zip_cmd(args):
+    try:
+        description = """
+        Zips up the seed corpus.
+        """
+        args = targets_parser(args, description)
+    except Exception as e:
+        print(e)
+        return 1
+
+    for target in args.TARGET:
+        # Zip the seed_corpus
+        seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target))
+        seeds = [abs_join(seed_corpus, f) for f in os.listdir(seed_corpus)]
+        zip_file = "{}.zip".format(seed_corpus)
+        cmd = ["zip", "-q", "-j", "-9", zip_file]
+        print(' '.join(cmd + [abs_join(seed_corpus, '*')]))
+        subprocess.check_call(cmd + seeds)
+
+
 def short_help(args):
     name = args[0]
     print("Usage: {} [OPTIONS] COMMAND [ARGS]...\n".format(name))
@@ -690,6 +774,8 @@ def help(args):
     print("\tafl\t\tRun an AFL fuzzer")
     print("\tregression\tRun a regression test")
     print("\tgen\t\tGenerate a seed corpus for a fuzzer")
+    print("\tminimize\tMinimize the test corpora")
+    print("\tzip\t\tZip the minimized corpora up")
 
 
 def main():
@@ -705,13 +791,17 @@ def main():
     if command == "build":
         return build(args)
     if command == "libfuzzer":
-        return libfuzzer(args)
+        return libfuzzer_cmd(args)
     if command == "regression":
         return regression(args)
     if command == "afl":
         return afl(args)
     if command == "gen":
         return gen(args)
+    if command == "minimize":
+        return minimize(args)
+    if command == "zip":
+        return zip_cmd(args)
     short_help(args)
     print("Error: No such command {} (pass -h for help)".format(command))
     return 1

tests/fuzz/simple_round_trip.c

@@ -38,10 +38,11 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
     if (FUZZ_rand(&seed) & 1) {
         ZSTD_inBuffer in = {src, srcSize, 0};
         ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
+        size_t err;
 
         ZSTD_CCtx_reset(cctx);
         FUZZ_setRandomParameters(cctx, &seed);
-        size_t const err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+        err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
         if (err != 0) {
             return err;
         }