Add an iterscale flag

This commit is contained in:
Nigel Tao committed 2018-05-04 17:17:59 +10:00
1 parent bd71c2d9fd
commit ba98b04307
10 files changed
+97 -47

No files matched your search

+5
View File
@@ -32,6 +32,11 @@ const (
FocusDefault = ""
FocusUsage = `comma-separated list of tests or benchmarks (name prefixes) to focus on, e.g. "wuffs_gif_decode"`
IterscaleDefault = 100
IterscaleMin = 0
IterscaleMax = 1000000
IterscaleUsage = `a scaling factor for the number of iterations per benchmark`
MimicDefault = false
MimicUsage = `whether to compare Wuffs' output with other libraries' output`
+16 -4
View File
@@ -34,6 +34,7 @@ func doBenchTest(args []string, bench bool) error {
flags := flag.FlagSet{}
ccompilersFlag := flags.String("ccompilers", cf.CcompilersDefault, cf.CcompilersUsage)
focusFlag := flags.String("focus", cf.FocusDefault, cf.FocusUsage)
iterscaleFlag := flags.Int("iterscale", cf.RepsDefault, cf.RepsUsage)
mimicFlag := flags.Bool("mimic", cf.MimicDefault, cf.MimicUsage)
repsFlag := flags.Int("reps", cf.RepsDefault, cf.RepsUsage)
@@ -47,15 +48,21 @@ func doBenchTest(args []string, bench bool) error {
if !cf.IsAlphaNumericIsh(*focusFlag) {
return fmt.Errorf("bad -focus flag value %q", *focusFlag)
}
if *iterscaleFlag < cf.IterscaleMin || cf.IterscaleMax < *iterscaleFlag {
return fmt.Errorf("bad -iterscale flag value %d, outside the range [%d..%d]",
*iterscaleFlag, cf.IterscaleMin, cf.IterscaleMax)
}
if *repsFlag < cf.RepsMin || cf.RepsMax < *repsFlag {
return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, cf.RepsMin, cf.RepsMax)
return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]",
*repsFlag, cf.RepsMin, cf.RepsMax)
}
args = flags.Args()
failed := false
for _, arg := range args {
f, err := doBenchTest1(arg, bench, *ccompilersFlag, *focusFlag, *mimicFlag, *repsFlag)
f, err := doBenchTest1(arg, bench,
*ccompilersFlag, *focusFlag, *iterscaleFlag, *mimicFlag, *repsFlag)
if err != nil {
return err
}
@@ -71,7 +78,9 @@ func doBenchTest(args []string, bench bool) error {
return nil
}
func doBenchTest1(filename string, bench bool, ccompilers string, focus string, mimic bool, reps int) (failed bool, err error) {
func doBenchTest1(filename string, bench bool, ccompilers string, focus string,
iterscale int, mimic bool, reps int) (failed bool, err error) {
workDir, err := ioutil.TempDir("", "wuffs-c")
if err != nil {
return false, err
@@ -112,7 +121,10 @@ func doBenchTest1(filename string, bench bool, ccompilers string, focus string,
outArgs := []string(nil)
if bench {
outArgs = append(outArgs, "-bench", fmt.Sprintf("-reps=%d", reps))
outArgs = append(outArgs, "-bench",
fmt.Sprintf("-iterscale=%d", iterscale),
fmt.Sprintf("-reps=%d", reps),
)
}
if focus != "" {
outArgs = append(outArgs, fmt.Sprintf("-focus=%s", focus))
+11 -2
View File
@@ -33,6 +33,7 @@ func doBenchTest(wuffsRoot string, args []string, bench bool) error {
ccompilersFlag := flags.String("ccompilers", cf.CcompilersDefault, cf.CcompilersUsage)
cformatterFlag := flags.String("cformatter", cf.CformatterDefault, cf.CformatterUsage)
focusFlag := flags.String("focus", cf.FocusDefault, cf.FocusUsage)
iterscaleFlag := flags.Int("iterscale", cf.RepsDefault, cf.RepsUsage)
langsFlag := flags.String("langs", langsDefault, langsUsage)
mimicFlag := flags.Bool("mimic", cf.MimicDefault, cf.MimicUsage)
repsFlag := flags.Int("reps", cf.RepsDefault, cf.RepsUsage)
@@ -56,8 +57,13 @@ func doBenchTest(wuffsRoot string, args []string, bench bool) error {
if !cf.IsAlphaNumericIsh(*focusFlag) {
return fmt.Errorf("bad -focus flag value %q", *focusFlag)
}
if *iterscaleFlag < cf.IterscaleMin || cf.IterscaleMax < *iterscaleFlag {
return fmt.Errorf("bad -iterscale flag value %d, outside the range [%d..%d]",
*iterscaleFlag, cf.IterscaleMin, cf.IterscaleMax)
}
if *repsFlag < cf.RepsMin || cf.RepsMax < *repsFlag {
return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, cf.RepsMin, cf.RepsMax)
return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]",
*repsFlag, cf.RepsMin, cf.RepsMax)
}
args = flags.Args()
@@ -67,7 +73,10 @@ func doBenchTest(wuffsRoot string, args []string, bench bool) error {
cmdArgs := []string(nil)
if bench {
cmdArgs = append(cmdArgs, "bench", fmt.Sprintf("-reps=%d", *repsFlag))
cmdArgs = append(cmdArgs, "bench",
fmt.Sprintf("-iterscale=%d", *iterscaleFlag),
fmt.Sprintf("-reps=%d", *repsFlag),
)
} else {
cmdArgs = append(cmdArgs, "test")
}
+4 -4
View File
@@ -205,12 +205,12 @@ const char* wuffs_bench_adler32(wuffs_base__io_buffer* dst,
void bench_wuffs_adler32_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_bench_adler32, tc_src, &adler32_midsummer_gt, 0, 0,
150000);
1500);
}
void bench_wuffs_adler32_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 15000);
do_bench_io_buffers(wuffs_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 150);
}
// ---------------- Mimic Benches
@@ -220,12 +220,12 @@ void bench_wuffs_adler32_100k() {
void bench_mimic_adler32_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_bench_adler32, tc_src, &adler32_midsummer_gt, 0, 0,
150000);
1500);
}
void bench_mimic_adler32_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 15000);
do_bench_io_buffers(mimic_bench_adler32, tc_src, &adler32_pi_gt, 0, 0, 150);
}
#endif // WUFFS_MIMIC
+4 -6
View File
@@ -205,13 +205,12 @@ const char* wuffs_bench_crc32_ieee(wuffs_base__io_buffer* dst,
void bench_wuffs_crc32_ieee_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_bench_crc32_ieee, tc_src, &crc32_midsummer_gt, 0, 0,
150000);
1500);
}
void bench_wuffs_crc32_ieee_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0,
15000);
do_bench_io_buffers(wuffs_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0, 150);
}
// ---------------- Mimic Benches
@@ -221,13 +220,12 @@ void bench_wuffs_crc32_ieee_100k() {
void bench_mimic_crc32_ieee_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_bench_crc32_ieee, tc_src, &crc32_midsummer_gt, 0, 0,
150000);
1500);
}
void bench_mimic_crc32_ieee_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0,
15000);
do_bench_io_buffers(mimic_bench_crc32_ieee, tc_src, &crc32_pi_gt, 0, 0, 150);
}
#endif // WUFFS_MIMIC
+6 -6
View File
@@ -582,18 +582,18 @@ void test_mimic_deflate_decode_romeo_fixed() {
void bench_wuffs_deflate_decode_1k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_romeo_gt, 0, 0,
200000);
2000);
}
void bench_wuffs_deflate_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_midsummer_gt, 0, 0,
30000);
300);
}
void bench_wuffs_deflate_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 3000);
do_bench_io_buffers(wuffs_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 30);
}
// ---------------- Mimic Benches
@@ -603,18 +603,18 @@ void bench_wuffs_deflate_decode_100k() {
void bench_mimic_deflate_decode_1k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_romeo_gt, 0, 0,
200000);
2000);
}
void bench_mimic_deflate_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_midsummer_gt, 0, 0,
30000);
300);
}
void bench_mimic_deflate_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 3000);
do_bench_io_buffers(mimic_deflate_decode, tc_dst, &deflate_pi_gt, 0, 0, 30);
}
#endif // WUFFS_MIMIC
+14 -12
View File
@@ -303,7 +303,7 @@ void test_wuffs_lzw_decode_pi() {
// ---------------- LZW Benches
bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters) {
bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters_unscaled) {
wuffs_base__io_buffer dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE};
wuffs_base__io_buffer src = {.ptr = global_src_buffer, .len = BUFFER_SIZE};
wuffs_base__io_writer dst_writer = wuffs_base__io_buffer__writer(&dst);
@@ -325,6 +325,7 @@ bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters) {
bench_start();
uint64_t n_bytes = 0;
uint64_t i;
uint64_t iters = iters_unscaled * iterscale;
for (i = 0; i < iters; i++) {
dst.wi = 0;
src.ri = 1; // Skip the literal width.
@@ -344,12 +345,12 @@ bool do_bench_wuffs_lzw_decode(const char* filename, uint64_t iters) {
void bench_wuffs_lzw_decode_20k() {
CHECK_FOCUS(__func__);
do_bench_wuffs_lzw_decode("../../data/bricks-gray.indexes.giflzw", 5000);
do_bench_wuffs_lzw_decode("../../data/bricks-gray.indexes.giflzw", 50);
}
void bench_wuffs_lzw_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_wuffs_lzw_decode("../../data/pi.txt.giflzw", 1000);
do_bench_wuffs_lzw_decode("../../data/pi.txt.giflzw", 10);
}
// ---------------- GIF Tests
@@ -751,7 +752,7 @@ void test_mimic_gif_decode_pjw_thumbnail() {
bool do_bench_gif_decode(const char* (*decode_func)(wuffs_base__io_buffer*,
wuffs_base__io_buffer*),
const char* filename,
uint64_t iters) {
uint64_t iters_unscaled) {
wuffs_base__io_buffer dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE};
wuffs_base__io_buffer src = {.ptr = global_src_buffer, .len = BUFFER_SIZE};
@@ -762,6 +763,7 @@ bool do_bench_gif_decode(const char* (*decode_func)(wuffs_base__io_buffer*,
bench_start();
uint64_t n_bytes = 0;
uint64_t i;
uint64_t iters = iters_unscaled * iterscale;
for (i = 0; i < iters; i++) {
dst.wi = 0;
src.ri = 0;
@@ -778,22 +780,22 @@ bool do_bench_gif_decode(const char* (*decode_func)(wuffs_base__io_buffer*,
void bench_wuffs_gif_decode_1k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(wuffs_gif_decode, "../../data/pjw-thumbnail.gif", 200000);
do_bench_gif_decode(wuffs_gif_decode, "../../data/pjw-thumbnail.gif", 2000);
}
void bench_wuffs_gif_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(wuffs_gif_decode, "../../data/hat.gif", 10000);
do_bench_gif_decode(wuffs_gif_decode, "../../data/hat.gif", 100);
}
void bench_wuffs_gif_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(wuffs_gif_decode, "../../data/hibiscus.gif", 1000);
do_bench_gif_decode(wuffs_gif_decode, "../../data/hibiscus.gif", 10);
}
void bench_wuffs_gif_decode_1000k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(wuffs_gif_decode, "../../data/harvesters.gif", 100);
do_bench_gif_decode(wuffs_gif_decode, "../../data/harvesters.gif", 1);
}
// ---------------- Mimic Benches
@@ -802,22 +804,22 @@ void bench_wuffs_gif_decode_1000k() {
void bench_mimic_gif_decode_1k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(mimic_gif_decode, "../../data/pjw-thumbnail.gif", 200000);
do_bench_gif_decode(mimic_gif_decode, "../../data/pjw-thumbnail.gif", 2000);
}
void bench_mimic_gif_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(mimic_gif_decode, "../../data/hat.gif", 10000);
do_bench_gif_decode(mimic_gif_decode, "../../data/hat.gif", 100);
}
void bench_mimic_gif_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(mimic_gif_decode, "../../data/hibiscus.gif", 1000);
do_bench_gif_decode(mimic_gif_decode, "../../data/hibiscus.gif", 10);
}
void bench_mimic_gif_decode_1000k() {
CHECK_FOCUS(__func__);
do_bench_gif_decode(mimic_gif_decode, "../../data/harvesters.gif", 100);
do_bench_gif_decode(mimic_gif_decode, "../../data/harvesters.gif", 1);
}
#endif // WUFFS_MIMIC
+4 -4
View File
@@ -206,12 +206,12 @@ void test_mimic_gzip_decode_pi() {
void bench_wuffs_gzip_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_gzip_decode, tc_dst, &gzip_midsummer_gt, 0, 0,
30000);
300);
}
void bench_wuffs_gzip_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 3000);
do_bench_io_buffers(wuffs_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 30);
}
// ---------------- Mimic Benches
@@ -221,12 +221,12 @@ void bench_wuffs_gzip_decode_100k() {
void bench_mimic_gzip_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_gzip_decode, tc_dst, &gzip_midsummer_gt, 0, 0,
30000);
300);
}
void bench_mimic_gzip_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 3000);
do_bench_io_buffers(mimic_gzip_decode, tc_dst, &gzip_pi_gt, 0, 0, 30);
}
#endif // WUFFS_MIMIC
+4 -4
View File
@@ -215,12 +215,12 @@ void test_mimic_zlib_decode_pi() {
void bench_wuffs_zlib_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_zlib_decode, tc_dst, &zlib_midsummer_gt, 0, 0,
30000);
300);
}
void bench_wuffs_zlib_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(wuffs_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 3000);
do_bench_io_buffers(wuffs_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 30);
}
// ---------------- Mimic Benches
@@ -230,12 +230,12 @@ void bench_wuffs_zlib_decode_100k() {
void bench_mimic_zlib_decode_10k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_zlib_decode, tc_dst, &zlib_midsummer_gt, 0, 0,
30000);
300);
}
void bench_mimic_zlib_decode_100k() {
CHECK_FOCUS(__func__);
do_bench_io_buffers(mimic_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 3000);
do_bench_io_buffers(mimic_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 30);
}
#endif // WUFFS_MIMIC
+29 -5
View File
@@ -37,6 +37,8 @@ char fail_msg[65536] = {0};
int tests_run = 0;
uint64_t iterscale = 100;
const char* proc_filename = "";
const char* proc_funcname = "";
const char* focus = "";
@@ -196,6 +198,24 @@ int test_main(int argc, char** argv, proc* tests, proc* benches) {
} else if ((arg_len >= 7) && !strncmp(arg, "-focus=", 7)) {
focus = arg + 7;
} else if ((arg_len >= 11) && !strncmp(arg, "-iterscale=", 11)) {
arg += 11;
if (!*arg) {
fprintf(stderr, "missing -iterscale=N value\n");
return 1;
}
char* end = NULL;
long int n = strtol(arg, &end, 10);
if (*end) {
fprintf(stderr, "invalid -iterscale=N value\n");
return 1;
}
if ((n < 0) || (1000000 < n)) {
fprintf(stderr, "out-of-range -iterscale=N value\n");
return 1;
}
iterscale = n;
} else if ((arg_len >= 6) && !strncmp(arg, "-reps=", 6)) {
arg += 6;
if (!*arg) {
@@ -256,8 +276,10 @@ int test_main(int argc, char** argv, proc* tests, proc* benches) {
}
}
if (bench) {
printf("# %-16s%-8s(%d benchmarks run, 1+%d reps per benchmark)\n",
proc_filename, cc, tests_run, reps - 1);
printf(
"# %-16s%-8s(%d benchmarks run, 1+%d reps per benchmark, "
"iterscale=%d)\n",
proc_filename, cc, tests_run, reps - 1, (int)(iterscale));
} else {
printf("%-16s%-8sPASS (%d tests run)\n", proc_filename, cc, tests_run);
}
@@ -419,7 +441,7 @@ bool proc_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*,
golden_test* gt,
uint64_t wlimit,
uint64_t rlimit,
uint64_t iters,
uint64_t iters_unscaled,
bool bench) {
if (!codec_func) {
FAIL("NULL codec_func");
@@ -457,6 +479,7 @@ bool proc_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*,
}
uint64_t n_bytes = 0;
uint64_t i;
uint64_t iters = iters_unscaled * iterscale;
for (i = 0; i < iters; i++) {
got.wi = 0;
src.ri = gt->src_offset0;
@@ -500,8 +523,9 @@ bool do_bench_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*,
golden_test* gt,
uint64_t wlimit,
uint64_t rlimit,
uint64_t iters) {
return proc_io_buffers(codec_func, tc, gt, wlimit, rlimit, iters, true);
uint64_t iters_unscaled) {
return proc_io_buffers(codec_func, tc, gt, wlimit, rlimit, iters_unscaled,
true);
}
bool do_test_io_buffers(const char* (*codec_func)(wuffs_base__io_buffer*,