diff --git a/cmd/puffs-test-c/main.go b/cmd/puffs-test-c/main.go index a79a9d70..fbd9652a 100644 --- a/cmd/puffs-test-c/main.go +++ b/cmd/puffs-test-c/main.go @@ -1,7 +1,7 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. -// puffs-test-c runs C test programs. +// puffs-test-c runs C benchmark and test programs. package main import ( @@ -13,6 +13,10 @@ import ( "path/filepath" ) +var ( + bench = flag.Bool("b", false, "whether to run benchmarks instead of regular tests") +) + func main() { if err := main1(); err != nil { os.Stderr.WriteString(err.Error() + "\n") @@ -47,14 +51,26 @@ func do(filename string) (failed bool, err error) { in := filename + ".c" out := filepath.Join(workDir, cc+".out") - ccCmd := exec.Command(cc, "-std=c99", "-Wall", "-Werror", "-o", out, in) + ccArgs := []string(nil) + if *bench { + ccArgs = append(ccArgs, "-O3") + } else { + // TODO: set these flags even if we pass -O3. + ccArgs = append(ccArgs, "-Wall", "-Werror") + } + ccArgs = append(ccArgs, "-std=c99", "-o", out, in) + ccCmd := exec.Command(cc, ccArgs...) ccCmd.Stdout = os.Stdout ccCmd.Stderr = os.Stderr if err := ccCmd.Run(); err != nil { return false, err } - outCmd := exec.Command(out) + outArgs := []string(nil) + if *bench { + outArgs = append(outArgs, "-b") + } + outCmd := exec.Command(out, outArgs...) outCmd.Stdout = os.Stdout outCmd.Stderr = os.Stderr outCmd.Dir = filepath.Dir(filename) diff --git a/cmd/puffs/main.go b/cmd/puffs/main.go index dab0a7b7..f84f8570 100644 --- a/cmd/puffs/main.go +++ b/cmd/puffs/main.go @@ -19,6 +19,7 @@ var commands = []struct { name string do func(puffsRoot string, args []string) error }{ + {"bench", doBench}, {"gen", doGen}, {"test", doTest}, } diff --git a/cmd/puffs/test.go b/cmd/puffs/test.go index fed6f9e4..f83404bb 100644 --- a/cmd/puffs/test.go +++ b/cmd/puffs/test.go @@ -13,7 +13,10 @@ import ( "strings" ) -func doTest(puffsRoot string, args []string) error { +func doBench(puffsRoot string, args []string) error { return doBenchTest(puffsRoot, args, true) } +func doTest(puffsRoot string, args []string) error { return doBenchTest(puffsRoot, args, false) } + +func doBenchTest(puffsRoot string, args []string, bench bool) error { flags := flag.NewFlagSet("test", flag.ExitOnError) langsFlag := flags.String("langs", langsDefault, langsUsage) if err := flags.Parse(args); err != nil { @@ -40,8 +43,8 @@ func doTest(puffsRoot string, args []string) error { return err } - // Proceed with testing the generated code. - f, err := test(puffsRoot, arg, langs, recursive) + // Proceed with benching / testing the generated code. + f, err := benchTest(puffsRoot, arg, langs, bench, recursive) if err != nil { return err } @@ -53,13 +56,13 @@ func doTest(puffsRoot string, args []string) error { return nil } -func test(puffsRoot, dirname string, langs []string, recursive bool) (failed bool, err error) { +func benchTest(puffsRoot, dirname string, langs []string, bench bool, recursive bool) (failed bool, err error) { filenames, dirnames, err := listDir(puffsRoot, dirname, recursive) if err != nil { return false, err } if len(filenames) > 0 { - f, err := testDir(puffsRoot, dirname, langs) + f, err := benchTestDir(puffsRoot, dirname, langs, bench) if err != nil { return false, err } @@ -67,7 +70,7 @@ func test(puffsRoot, dirname string, langs []string, recursive bool) (failed boo } if len(dirnames) > 0 { for _, d := range dirnames { - f, err := test(puffsRoot, filepath.Join(dirname, d), langs, recursive) + f, err := benchTest(puffsRoot, filepath.Join(dirname, d), langs, bench, recursive) if err != nil { return false, err } @@ -77,14 +80,19 @@ func test(puffsRoot, dirname string, langs []string, recursive bool) (failed boo return failed, nil } -func testDir(puffsRoot string, dirname string, langs []string) (failed bool, err error) { +func benchTestDir(puffsRoot string, dirname string, langs []string, bench bool) (failed bool, err error) { if packageName := filepath.Base(dirname); !validName(packageName) { return false, fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName) } for _, lang := range langs { command := "puffs-test-" + lang - cmd := exec.Command(command, filepath.Join(puffsRoot, "test", lang, filepath.FromSlash(dirname))) + args := []string(nil) + if bench { + args = append(args, "-b") + } + args = append(args, filepath.Join(puffsRoot, "test", lang, filepath.FromSlash(dirname))) + cmd := exec.Command(command, args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err == nil { diff --git a/test/c/std/gif.c b/test/c/std/gif.c index ffd7137b..39336428 100644 --- a/test/c/std/gif.c +++ b/test/c/std/gif.c @@ -15,7 +15,7 @@ Each edition should print "PASS", amongst other information, and exit(0). #include "../../../gen/c/std/gif.c" #include "../testlib/testlib.c" -const char* test_filename = "std/gif.c"; +const char* proc_filename = "std/gif.c"; #define BUFFER_SIZE (1024 * 1024) @@ -27,7 +27,7 @@ uint8_t global_palette_buffer[3 * 256]; // ---------------- Basic Tests void test_basic_bad_argument_out_of_range() { - test_funcname = __func__; + proc_funcname = __func__; puffs_gif_lzw_decoder dec; puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0); @@ -54,7 +54,7 @@ void test_basic_bad_argument_out_of_range() { } void test_basic_bad_receiver() { - test_funcname = __func__; + proc_funcname = __func__; puffs_base_writer1 dst = {0}; puffs_base_reader1 src = {0}; puffs_gif_status status = puffs_gif_lzw_decoder_decode(NULL, dst, src); @@ -64,7 +64,7 @@ void test_basic_bad_receiver() { } void test_basic_constructor_not_called() { - test_funcname = __func__; + proc_funcname = __func__; puffs_gif_lzw_decoder dec = {{0}}; puffs_base_writer1 dst = {0}; puffs_base_reader1 src = {0}; @@ -76,7 +76,7 @@ void test_basic_constructor_not_called() { } void test_basic_puffs_version_bad() { - test_funcname = __func__; + proc_funcname = __func__; puffs_gif_lzw_decoder dec; puffs_gif_lzw_decoder_constructor(&dec, 0, 0); // 0 is not PUFFS_VERSION. if (dec.private_impl.status != puffs_gif_error_bad_version) { @@ -89,7 +89,7 @@ cleanup0: } void test_basic_puffs_version_good() { - test_funcname = __func__; + proc_funcname = __func__; puffs_gif_lzw_decoder dec; puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0); if (dec.private_impl.magic != PUFFS_MAGIC) { @@ -106,7 +106,7 @@ cleanup0: } void test_basic_status_is_error() { - test_funcname = __func__; + proc_funcname = __func__; if (puffs_gif_status_is_error(puffs_gif_status_ok)) { FAIL("is_error(ok) returned true"); return; @@ -126,7 +126,7 @@ void test_basic_status_is_error() { } void test_basic_status_strings() { - test_funcname = __func__; + proc_funcname = __func__; const char* s0 = puffs_gif_status_string(puffs_gif_status_ok); const char* t0 = "gif: ok"; if (strcmp(s0, t0)) { @@ -161,7 +161,7 @@ void test_basic_status_strings() { } void test_basic_sub_struct_constructor() { - test_funcname = __func__; + proc_funcname = __func__; puffs_gif_decoder dec; puffs_gif_decoder_constructor(&dec, PUFFS_VERSION, 0); if (dec.private_impl.magic != PUFFS_MAGIC) { @@ -272,15 +272,65 @@ cleanup0:; } void test_lzw_decode_many_small_inputs() { - test_funcname = __func__; + proc_funcname = __func__; test_lzw_decode_xxx(100); } void test_lzw_decode_one_large_input() { - test_funcname = __func__; + proc_funcname = __func__; test_lzw_decode_xxx(1 << 30); } +// ---------------- LZW Benches + +void bench_lzw_decode_xxx(const char* filename, uint64_t reps) { + puffs_base_buf1 dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE}; + puffs_base_buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE}; + puffs_base_writer1 dst_writer = {.buf = &dst}; + puffs_base_reader1 src_reader = {.buf = &src}; + + if (!read_file(&src, filename)) { + return; + } + if (src.wi <= 0) { + FAIL("src size: got %d, want > 0", (int)(src.wi)); + return; + } + uint8_t literal_width = src.ptr[0]; + if (literal_width != 0x08) { + FAIL("LZW literal width: got %d, want %d", (int)(src.ptr[0]), 0x08); + return; + } + + bench_start(); + uint64_t n_bytes = 0; + for (uint64_t i = 0; i < reps; i++) { + dst.wi = 0; + src.ri = 1; // Skip the literal width. + puffs_gif_lzw_decoder dec; + puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0); + puffs_gif_status s = + puffs_gif_lzw_decoder_decode(&dec, dst_writer, src_reader); + puffs_gif_lzw_decoder_destructor(&dec); + if (s) { + FAIL("decode: %s", puffs_gif_status_string(s)); + return; + } + n_bytes += dst.wi; + } + bench_finish(reps, n_bytes); +} + +void bench_lzw_decode_20k() { + proc_funcname = __func__; + bench_lzw_decode_xxx("../../testdata/bricks-gray.giflzw", 5000); +} + +void bench_lzw_decode_100k() { + proc_funcname = __func__; + bench_lzw_decode_xxx("../../testdata/pi.txt.giflzw", 1000); +} + // ---------------- GIF Tests void test_gif_decode_input_is_a_xxx(const char* filename, @@ -343,7 +393,7 @@ cleanup0:; } void test_gif_decode_input_is_a_gif() { - test_funcname = __func__; + proc_funcname = __func__; test_gif_decode_input_is_a_xxx("../../testdata/bricks-dither.gif", "../../testdata/bricks-dither.palette", "../../testdata/bricks-dither.indexes", @@ -351,15 +401,65 @@ void test_gif_decode_input_is_a_gif() { } void test_gif_decode_input_is_a_png() { - test_funcname = __func__; + proc_funcname = __func__; test_gif_decode_input_is_a_xxx("../../testdata/bricks-dither.png", "", "", puffs_gif_error_bad_gif_header); } +// ---------------- GIF Benches + +void bench_gif_decode_xxx(const char* filename, uint64_t reps) { + puffs_base_buf1 dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE}; + puffs_base_buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE}; + puffs_base_writer1 dst_writer = {.buf = &dst}; + puffs_base_reader1 src_reader = {.buf = &src}; + + if (!read_file(&src, filename)) { + return; + } + + bench_start(); + uint64_t n_bytes = 0; + for (uint64_t i = 0; i < reps; i++) { + dst.wi = 0; + src.ri = 0; + puffs_gif_decoder dec; + puffs_gif_decoder_constructor(&dec, PUFFS_VERSION, 0); + puffs_gif_status s = puffs_gif_decoder_decode(&dec, dst_writer, src_reader); + puffs_gif_decoder_destructor(&dec); + if (s) { + FAIL("decode: %s", puffs_gif_status_string(s)); + return; + } + n_bytes += dst.wi; + } + bench_finish(reps, n_bytes); +} + +void bench_gif_decode_1k() { + proc_funcname = __func__; + bench_gif_decode_xxx("../../testdata/pjw-thumbnail.gif", 200000); +} + +void bench_gif_decode_20k() { + proc_funcname = __func__; + bench_gif_decode_xxx("../../testdata/bricks-dither.gif", 5000); +} + +void bench_gif_decode_100k() { + proc_funcname = __func__; + bench_gif_decode_xxx("../../testdata/hibiscus.gif", 1000); +} + +void bench_gif_decode_1000k() { + proc_funcname = __func__; + bench_gif_decode_xxx("../../testdata/harvesters.gif", 100); +} + // ---------------- Manifest // The empty comments forces clang-format to place one element per line. -test tests[] = { +proc tests[] = { // Basic Tests test_basic_bad_argument_out_of_range, // test_basic_bad_receiver, // @@ -369,12 +469,31 @@ test tests[] = { test_basic_status_is_error, // test_basic_status_strings, // test_basic_sub_struct_constructor, // + // LZW Tests test_lzw_decode_many_small_inputs, // test_lzw_decode_one_large_input, // + // GIF Tests test_gif_decode_input_is_a_gif, // test_gif_decode_input_is_a_png, // + + // End + NULL, +}; + +// The empty comments forces clang-format to place one element per line. +proc benches[] = { + // LZW Benches + bench_lzw_decode_20k, // + bench_lzw_decode_100k, // + + // GIF Benches + bench_gif_decode_1k, // + bench_gif_decode_20k, // + bench_gif_decode_100k, // + bench_gif_decode_1000k, // + // End NULL, }; diff --git a/test/c/testlib/testlib.c b/test/c/testlib/testlib.c index 4fc91620..c5d7750f 100644 --- a/test/c/testlib/testlib.c +++ b/test/c/testlib/testlib.c @@ -2,8 +2,10 @@ // in the LICENSE file. #include +#include #include #include +#include char fail_msg[65536] = {0}; @@ -13,36 +15,88 @@ char fail_msg[65536] = {0}; int tests_run = 0; -typedef void (*test)(); +typedef void (*proc)(); -test tests[]; +proc benches[]; +proc tests[]; -const char* test_filename; -const char* test_funcname = ""; +const char* proc_filename; +const char* proc_funcname = ""; -int main(int argc, char** argv) { // The order matters here. Clang also defines "__GNUC__". #if defined(__clang__) - const char* cc = "clang"; +const char* cc = "clang"; #elif defined(__GNUC__) - const char* cc = "gcc"; +const char* cc = "gcc"; #elif defined(_MSC_VER) - const char* cc = "cl"; +const char* cc = "cl"; #else - const char* cc = "cc"; +const char* cc = "cc"; #endif - for (test* t = tests; *t; t++) { - test_funcname = "unknown_test_funcname"; - (*t)(); - if (fail_msg[0]) { - printf("%-16s%-8sFAIL %s: %s\n", test_filename, cc, test_funcname, - fail_msg); - return 1; +bool bench_warm_up; +struct timeval bench_start_tv; + +void bench_start() { + gettimeofday(&bench_start_tv, NULL); +} + +void bench_finish(uint64_t reps, uint64_t n_bytes) { + struct timeval bench_finish_tv; + gettimeofday(&bench_finish_tv, NULL); + int64_t micros = + (int64_t)(bench_finish_tv.tv_sec - bench_start_tv.tv_sec) * 1000000 + + (int64_t)(bench_finish_tv.tv_usec - bench_start_tv.tv_usec); + uint64_t nanos = 1; + if (micros > 0) { + nanos = (uint64_t)(micros)*1000; + } + uint64_t kb_per_s = n_bytes * 1000000 / nanos; + if (bench_warm_up) { + printf("# (warm up) %s_%s\t%8" PRIu64 ".%06" PRIu64 " seconds\n", + proc_funcname, cc, nanos / 1000000000, (nanos % 1000000000) / 1000); + } else { + printf("Benchmark%s_%s\t%8" PRIu64 "\t%8" PRIu64 " ns/op\t%8d.%03d MB/s\n", + proc_funcname, cc, reps, nanos / reps, (int)(kb_per_s / 1000), + (int)(kb_per_s % 1000)); + } +} + +int main(int argc, char** argv) { + bool bench = false; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-b")) { + bench = true; } - tests_run++; + } + + int proc_reps = 1; + proc* procs = tests; + if (bench) { + proc_reps = 5 + 1; // +1 for the warm up run. + procs = benches; + printf( + "# The output format, including the redundant \"Benchmark\"s, is " + "compatible with\n# " + "https://godoc.org/golang.org/x/perf/cmd/benchstat\n"); + } + + for (int i = 0; i < proc_reps; i++) { + bench_warm_up = i == 0; + for (proc* t = procs; *t; t++) { + proc_funcname = "unknown_funcname"; + (*t)(); + if (fail_msg[0]) { + printf("%-16s%-8sFAIL %s: %s\n", proc_filename, cc, proc_funcname, + fail_msg); + return 1; + } + tests_run++; + } + } + if (!bench) { + printf("%-16s%-8sPASS (%d tests run)\n", proc_filename, cc, tests_run); } - printf("%-16s%-8sPASS (%d tests run)\n", test_filename, cc, tests_run); return 0; }