diff --git a/cmd/commonflags/commonflags.go b/cmd/commonflags/commonflags.go index 182e92f2..1dfe226f 100644 --- a/cmd/commonflags/commonflags.go +++ b/cmd/commonflags/commonflags.go @@ -32,6 +32,11 @@ const ( FocusDefault = "" FocusUsage = `comma-separated list of tests or benchmarks (name prefixes) to focus on, e.g. "wuffs_gif_decode"` + IterscaleDefault = 100 + IterscaleMin = 0 + IterscaleMax = 1000000 + IterscaleUsage = `a scaling factor for the number of iterations per benchmark` + MimicDefault = false MimicUsage = `whether to compare Wuffs' output with other libraries' output` diff --git a/cmd/wuffs-c/test.go b/cmd/wuffs-c/test.go index 5ce6ed11..bfadee28 100644 --- a/cmd/wuffs-c/test.go +++ b/cmd/wuffs-c/test.go @@ -34,6 +34,7 @@ func doBenchTest(args []string, bench bool) error { flags := flag.FlagSet{} ccompilersFlag := flags.String("ccompilers", cf.CcompilersDefault, cf.CcompilersUsage) focusFlag := flags.String("focus", cf.FocusDefault, cf.FocusUsage) + iterscaleFlag := flags.Int("iterscale", cf.RepsDefault, cf.RepsUsage) mimicFlag := flags.Bool("mimic", cf.MimicDefault, cf.MimicUsage) repsFlag := flags.Int("reps", cf.RepsDefault, cf.RepsUsage) @@ -47,15 +48,21 @@ func doBenchTest(args []string, bench bool) error { if !cf.IsAlphaNumericIsh(*focusFlag) { return fmt.Errorf("bad -focus flag value %q", *focusFlag) } + if *iterscaleFlag < cf.IterscaleMin || cf.IterscaleMax < *iterscaleFlag { + return fmt.Errorf("bad -iterscale flag value %d, outside the range [%d..%d]", + *iterscaleFlag, cf.IterscaleMin, cf.IterscaleMax) + } if *repsFlag < cf.RepsMin || cf.RepsMax < *repsFlag { - return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, cf.RepsMin, cf.RepsMax) + return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", + *repsFlag, cf.RepsMin, cf.RepsMax) } args = flags.Args() failed := false for _, arg := range args { - f, err := doBenchTest1(arg, bench, *ccompilersFlag, *focusFlag, *mimicFlag, *repsFlag) + f, err := doBenchTest1(arg, bench, + *ccompilersFlag, *focusFlag, *iterscaleFlag, *mimicFlag, *repsFlag) if err != nil { return err } @@ -71,7 +78,9 @@ func doBenchTest(args []string, bench bool) error { return nil } -func doBenchTest1(filename string, bench bool, ccompilers string, focus string, mimic bool, reps int) (failed bool, err error) { +func doBenchTest1(filename string, bench bool, ccompilers string, focus string, + iterscale int, mimic bool, reps int) (failed bool, err error) { + workDir, err := ioutil.TempDir("", "wuffs-c") if err != nil { return false, err @@ -112,7 +121,10 @@ func doBenchTest1(filename string, bench bool, ccompilers string, focus string, outArgs := []string(nil) if bench { - outArgs = append(outArgs, "-bench", fmt.Sprintf("-reps=%d", reps)) + outArgs = append(outArgs, "-bench", + fmt.Sprintf("-iterscale=%d", iterscale), + fmt.Sprintf("-reps=%d", reps), + ) } if focus != "" { outArgs = append(outArgs, fmt.Sprintf("-focus=%s", focus)) diff --git a/cmd/wuffs/test.go b/cmd/wuffs/test.go index 9abb88a4..a9105714 100644 --- a/cmd/wuffs/test.go +++ b/cmd/wuffs/test.go @@ -33,6 +33,7 @@ func doBenchTest(wuffsRoot string, args []string, bench bool) error { ccompilersFlag := flags.String("ccompilers", cf.CcompilersDefault, cf.CcompilersUsage) cformatterFlag := flags.String("cformatter", cf.CformatterDefault, cf.CformatterUsage) focusFlag := flags.String("focus", cf.FocusDefault, cf.FocusUsage) + iterscaleFlag := flags.Int("iterscale", cf.RepsDefault, cf.RepsUsage) langsFlag := flags.String("langs", langsDefault, langsUsage) mimicFlag := flags.Bool("mimic", cf.MimicDefault, cf.MimicUsage) repsFlag := flags.Int("reps", cf.RepsDefault, cf.RepsUsage) @@ -56,8 +57,13 @@ func doBenchTest(wuffsRoot string, args []string, bench bool) error { if !cf.IsAlphaNumericIsh(*focusFlag) { return fmt.Errorf("bad -focus flag value %q", *focusFlag) } + if *iterscaleFlag < cf.IterscaleMin || cf.IterscaleMax < *iterscaleFlag { + return fmt.Errorf("bad -iterscale flag value %d, outside the range [%d..%d]", + *iterscaleFlag, cf.IterscaleMin, cf.IterscaleMax) + } if *repsFlag < cf.RepsMin || cf.RepsMax < *repsFlag { - return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, cf.RepsMin, cf.RepsMax) + return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", + *repsFlag, cf.RepsMin, cf.RepsMax) } args = flags.Args() @@ -67,7 +73,10 @@ func doBenchTest(wuffsRoot string, args []string, bench bool) error { cmdArgs := []string(nil) if bench { - cmdArgs = append(cmdArgs, "bench", fmt.Sprintf("-reps=%d", *repsFlag)) + cmdArgs = append(cmdArgs, "bench", + fmt.Sprintf("-iterscale=%d", *iterscaleFlag), + fmt.Sprintf("-reps=%d", *repsFlag), + ) } else { cmdArgs = append(cmdArgs, "test") } diff --git a/test/c/std/adler32.c b/test/c/std/adler32.c index fac4b843..a110e85f 100644 --- a/test/c/std/adler32.c +++ b/test/c/std/adler32.c @@ -205,12 +205,12 @@ const char* wuffs_bench_adler32(wuffs_base__io_buffer* dst, void bench_wuffs_adler32_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(wuffs_bench_adler32, tc_src, &adler32_midsummer_gt, 0, 0, - 150000); + 1500); } void bench_wuffs_adler32_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(wuffs_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 15000); + do_bench_io_buffers(wuffs_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 150); } // ---------------- Mimic Benches @@ -220,12 +220,12 @@ void bench_wuffs_adler32_100k() { void bench_mimic_adler32_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(mimic_bench_adler32, tc_src, &adler32_midsummer_gt, 0, 0, - 150000); + 1500); } void bench_mimic_adler32_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(mimic_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 15000); + do_bench_io_buffers(mimic_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 150); } #endif // WUFFS_MIMIC diff --git a/test/c/std/crc32.c b/test/c/std/crc32.c index 4b566814..44c85bf6 100644 --- a/test/c/std/crc32.c +++ b/test/c/std/crc32.c @@ -205,13 +205,12 @@ const char* wuffs_bench_crc32_ieee(wuffs_base__io_buffer* dst, void bench_wuffs_crc32_ieee_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(wuffs_bench_crc32_ieee, tc_src, &crc32_midsummer_gt, 0, 0, - 150000); + 1500); } void bench_wuffs_crc32_ieee_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(wuffs_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0, - 15000); + do_bench_io_buffers(wuffs_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0, 150); } // ---------------- Mimic Benches @@ -221,13 +220,12 @@ void bench_wuffs_crc32_ieee_100k() { void bench_mimic_crc32_ieee_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(mimic_bench_crc32_ieee, tc_src, &crc32_midsummer_gt, 0, 0, - 150000); + 1500); } void bench_mimic_crc32_ieee_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(mimic_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0, - 15000); + do_bench_io_buffers(mimic_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0, 150); } #endif // WUFFS_MIMIC diff --git a/test/c/std/deflate.c b/test/c/std/deflate.c index 02a1c829..57e2a07b 100644 --- a/test/c/std/deflate.c +++ b/test/c/std/deflate.c @@ -582,18 +582,18 @@ void test_mimic_deflate_decode_romeo_fixed() { void bench_wuffs_deflate_decode_1k() { CHECK_FOCUS(__func__); do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_romeo_gt, 0, 0, - 200000); + 2000); } void bench_wuffs_deflate_decode_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_midsummer_gt, 0, 0, - 30000); + 300); } void bench_wuffs_deflate_decode_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 3000); + do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 30); } // ---------------- Mimic Benches @@ -603,18 +603,18 @@ void bench_wuffs_deflate_decode_100k() { void bench_mimic_deflate_decode_1k() { CHECK_FOCUS(__func__); do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_romeo_gt, 0, 0, - 200000); + 2000); } void bench_mimic_deflate_decode_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_midsummer_gt, 0, 0, - 30000); + 300); } void bench_mimic_deflate_decode_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 3000); + do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 30); } #endif // WUFFS_MIMIC diff --git a/test/c/std/gif.c b/test/c/std/gif.c index 56a8deef..4ede8b55 100644 --- a/test/c/std/gif.c +++ b/test/c/std/gif.c @@ -303,7 +303,7 @@ void test_wuffs_lzw_decode_pi() { // ---------------- LZW Benches -bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters) { +bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters_unscaled) { wuffs_base__io_buffer dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE}; wuffs_base__io_buffer src = {.ptr = global_src_buffer, .len = BUFFER_SIZE}; wuffs_base__io_writer dst_writer = wuffs_base__io_buffer__writer(&dst); @@ -325,6 +325,7 @@ bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters) { bench_start(); uint64_t n_bytes = 0; uint64_t i; + uint64_t iters = iters_unscaled * iterscale; for (i = 0; i < iters; i++) { dst.wi = 0; src.ri = 1; // Skip the literal width. @@ -344,12 +345,12 @@ bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters) { void bench_wuffs_lzw_decode_20k() { CHECK_FOCUS(__func__); - do_bench_wuffs_lzw_decode("../../data/bricks-gray.indexes.giflzw", 5000); + do_bench_wuffs_lzw_decode("../../data/bricks-gray.indexes.giflzw", 50); } void bench_wuffs_lzw_decode_100k() { CHECK_FOCUS(__func__); - do_bench_wuffs_lzw_decode("../../data/pi.txt.giflzw", 1000); + do_bench_wuffs_lzw_decode("../../data/pi.txt.giflzw", 10); } // ---------------- GIF Tests @@ -751,7 +752,7 @@ void test_mimic_gif_decode_pjw_thumbnail() { bool do_bench_gif_decode(const char* (*decode_func)(wuffs_base__io_buffer*, wuffs_base__io_buffer*), const char* filename, - uint64_t iters) { + uint64_t iters_unscaled) { wuffs_base__io_buffer dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE}; wuffs_base__io_buffer src = {.ptr = global_src_buffer, .len = BUFFER_SIZE}; @@ -762,6 +763,7 @@ bool do_bench_gif_decode(const char* (*decode_func)(wuffs_base__io_buffer*, bench_start(); uint64_t n_bytes = 0; uint64_t i; + uint64_t iters = iters_unscaled * iterscale; for (i = 0; i < iters; i++) { dst.wi = 0; src.ri = 0; @@ -778,22 +780,22 @@ bool do_bench_gif_decode(const char* (*decode_func)(wuffs_base__io_buffer*, void bench_wuffs_gif_decode_1k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(wuffs_gif_decode, "../../data/pjw-thumbnail.gif", 200000); + do_bench_gif_decode(wuffs_gif_decode, "../../data/pjw-thumbnail.gif", 2000); } void bench_wuffs_gif_decode_10k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(wuffs_gif_decode, "../../data/hat.gif", 10000); + do_bench_gif_decode(wuffs_gif_decode, "../../data/hat.gif", 100); } void bench_wuffs_gif_decode_100k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(wuffs_gif_decode, "../../data/hibiscus.gif", 1000); + do_bench_gif_decode(wuffs_gif_decode, "../../data/hibiscus.gif", 10); } void bench_wuffs_gif_decode_1000k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(wuffs_gif_decode, "../../data/harvesters.gif", 100); + do_bench_gif_decode(wuffs_gif_decode, "../../data/harvesters.gif", 1); } // ---------------- Mimic Benches @@ -802,22 +804,22 @@ void bench_wuffs_gif_decode_1000k() { void bench_mimic_gif_decode_1k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(mimic_gif_decode, "../../data/pjw-thumbnail.gif", 200000); + do_bench_gif_decode(mimic_gif_decode, "../../data/pjw-thumbnail.gif", 2000); } void bench_mimic_gif_decode_10k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(mimic_gif_decode, "../../data/hat.gif", 10000); + do_bench_gif_decode(mimic_gif_decode, "../../data/hat.gif", 100); } void bench_mimic_gif_decode_100k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(mimic_gif_decode, "../../data/hibiscus.gif", 1000); + do_bench_gif_decode(mimic_gif_decode, "../../data/hibiscus.gif", 10); } void bench_mimic_gif_decode_1000k() { CHECK_FOCUS(__func__); - do_bench_gif_decode(mimic_gif_decode, "../../data/harvesters.gif", 100); + do_bench_gif_decode(mimic_gif_decode, "../../data/harvesters.gif", 1); } #endif // WUFFS_MIMIC diff --git a/test/c/std/gzip.c b/test/c/std/gzip.c index d75756bd..3bd2c802 100644 --- a/test/c/std/gzip.c +++ b/test/c/std/gzip.c @@ -206,12 +206,12 @@ void test_mimic_gzip_decode_pi() { void bench_wuffs_gzip_decode_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(wuffs_gzip_decode, tc_dst, &gzip_midsummer_gt, 0, 0, - 30000); + 300); } void bench_wuffs_gzip_decode_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(wuffs_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 3000); + do_bench_io_buffers(wuffs_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 30); } // ---------------- Mimic Benches @@ -221,12 +221,12 @@ void bench_wuffs_gzip_decode_100k() { void bench_mimic_gzip_decode_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(mimic_gzip_decode, tc_dst, &gzip_midsummer_gt, 0, 0, - 30000); + 300); } void bench_mimic_gzip_decode_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(mimic_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 3000); + do_bench_io_buffers(mimic_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 30); } #endif // WUFFS_MIMIC diff --git a/test/c/std/zlib.c b/test/c/std/zlib.c index 0d05aeaa..dba1955c 100644 --- a/test/c/std/zlib.c +++ b/test/c/std/zlib.c @@ -215,12 +215,12 @@ void test_mimic_zlib_decode_pi() { void bench_wuffs_zlib_decode_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(wuffs_zlib_decode, tc_dst, &zlib_midsummer_gt, 0, 0, - 30000); + 300); } void bench_wuffs_zlib_decode_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(wuffs_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 3000); + do_bench_io_buffers(wuffs_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 30); } // ---------------- Mimic Benches @@ -230,12 +230,12 @@ void bench_wuffs_zlib_decode_100k() { void bench_mimic_zlib_decode_10k() { CHECK_FOCUS(__func__); do_bench_io_buffers(mimic_zlib_decode, tc_dst, &zlib_midsummer_gt, 0, 0, - 30000); + 300); } void bench_mimic_zlib_decode_100k() { CHECK_FOCUS(__func__); - do_bench_io_buffers(mimic_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 3000); + do_bench_io_buffers(mimic_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 30); } #endif // WUFFS_MIMIC diff --git a/test/c/testlib/testlib.c b/test/c/testlib/testlib.c index e7ff95b0..df41a31f 100644 --- a/test/c/testlib/testlib.c +++ b/test/c/testlib/testlib.c @@ -37,6 +37,8 @@ char fail_msg[65536] = {0}; int tests_run = 0; +uint64_t iterscale = 100; + const char* proc_filename = ""; const char* proc_funcname = ""; const char* focus = ""; @@ -196,6 +198,24 @@ int test_main(int argc, char** argv, proc* tests, proc* benches) { } else if ((arg_len >= 7) && !strncmp(arg, "-focus=", 7)) { focus = arg + 7; + } else if ((arg_len >= 11) && !strncmp(arg, "-iterscale=", 11)) { + arg += 11; + if (!*arg) { + fprintf(stderr, "missing -iterscale=N value\n"); + return 1; + } + char* end = NULL; + long int n = strtol(arg, &end, 10); + if (*end) { + fprintf(stderr, "invalid -iterscale=N value\n"); + return 1; + } + if ((n < 0) || (1000000 < n)) { + fprintf(stderr, "out-of-range -iterscale=N value\n"); + return 1; + } + iterscale = n; + } else if ((arg_len >= 6) && !strncmp(arg, "-reps=", 6)) { arg += 6; if (!*arg) { @@ -256,8 +276,10 @@ int test_main(int argc, char** argv, proc* tests, proc* benches) { } } if (bench) { - printf("# %-16s%-8s(%d benchmarks run, 1+%d reps per benchmark)\n", - proc_filename, cc, tests_run, reps - 1); + printf( + "# %-16s%-8s(%d benchmarks run, 1+%d reps per benchmark, " + "iterscale=%d)\n", + proc_filename, cc, tests_run, reps - 1, (int)(iterscale)); } else { printf("%-16s%-8sPASS (%d tests run)\n", proc_filename, cc, tests_run); } @@ -419,7 +441,7 @@ bool proc_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*, golden_test* gt, uint64_t wlimit, uint64_t rlimit, - uint64_t iters, + uint64_t iters_unscaled, bool bench) { if (!codec_func) { FAIL("NULL codec_func"); @@ -457,6 +479,7 @@ bool proc_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*, } uint64_t n_bytes = 0; uint64_t i; + uint64_t iters = iters_unscaled * iterscale; for (i = 0; i < iters; i++) { got.wi = 0; src.ri = gt->src_offset0; @@ -500,8 +523,9 @@ bool do_bench_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*, golden_test* gt, uint64_t wlimit, uint64_t rlimit, - uint64_t iters) { - return proc_io_buffers(codec_func, tc, gt, wlimit, rlimit, iters, true); + uint64_t iters_unscaled) { + return proc_io_buffers(codec_func, tc, gt, wlimit, rlimit, iters_unscaled, + true); } bool do_test_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*,