diff --git a/cmd/puffs-test-c/main.go b/cmd/puffs-test-c/main.go index 90c2dec9..5a6ec4e1 100644 --- a/cmd/puffs-test-c/main.go +++ b/cmd/puffs-test-c/main.go @@ -15,9 +15,20 @@ import ( "strings" ) +const ( + mimicDefault = false + mimicUsage = `whether to compare Puffs' output with other libraries' output` + + repsDefault = 5 + repsMin = 0 + repsMax = 1000000 + repsUsage = `the number of repetitions per benchmark` +) + var ( - bench = flag.Bool("bench", false, "whether to run benchmarks instead of regular tests") - mimic = flag.Bool("mimic", false, "whether to compare Puffs' output with other libraries' output") + benchFlag = flag.Bool("bench", false, "whether to run benchmarks instead of regular tests") + mimicFlag = flag.Bool("mimic", mimicDefault, mimicUsage) + repsFlag = flag.Int("reps", repsDefault, repsUsage) ) func main() { @@ -29,6 +40,10 @@ func main() { func main1() error { flag.Parse() + if *repsFlag < repsMin || repsMax < *repsFlag { + return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, repsMin, repsMax) + } + failed := false for _, arg := range flag.Args() { f, err := do(arg) @@ -39,7 +54,7 @@ func main1() error { } if failed { s := "tests" - if *bench { + if *benchFlag { s = "benchmarks" } return fmt.Errorf("%s: some %s failed", os.Args[0], s) @@ -58,14 +73,14 @@ func do(filename string) (failed bool, err error) { out := filepath.Join(workDir, "a.out") ccArgs := []string(nil) - if *bench { + if *benchFlag { ccArgs = append(ccArgs, "-O3") } else { // TODO: set these flags even if we pass -O3. ccArgs = append(ccArgs, "-Wall", "-Werror") } ccArgs = append(ccArgs, "-std=c99", "-o", out, in) - if *mimic { + if *mimicFlag { extra, err := findPuffsMimicCflags(in) if err != nil { return false, err @@ -82,8 +97,8 @@ func do(filename string) (failed bool, err error) { } outArgs := []string(nil) - if *bench { - outArgs = append(outArgs, "-bench") + if *benchFlag { + outArgs = append(outArgs, "-bench", fmt.Sprintf("-reps=%d", *repsFlag)) } outCmd := exec.Command(out, outArgs...) outCmd.Stdout = os.Stdout diff --git a/cmd/puffs/main.go b/cmd/puffs/main.go index 0a6726f9..23340eaa 100644 --- a/cmd/puffs/main.go +++ b/cmd/puffs/main.go @@ -110,6 +110,11 @@ const ( mimicDefault = false mimicUsage = `whether to compare Puffs' output with other libraries' output` + + repsDefault = 5 + repsMin = 0 + repsMax = 1000000 + repsUsage = `the number of repetitions per benchmark` ) func parseLangs(commaSeparated string) ([]string, error) { diff --git a/cmd/puffs/test.go b/cmd/puffs/test.go index 48da386b..6de3a1a7 100644 --- a/cmd/puffs/test.go +++ b/cmd/puffs/test.go @@ -19,6 +19,7 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error { flags := flag.NewFlagSet("test", flag.ExitOnError) langsFlag := flags.String("langs", langsDefault, langsUsage) mimicFlag := flags.Bool("mimic", mimicDefault, mimicUsage) + repsFlag := flags.Int("reps", repsDefault, repsUsage) if err := flags.Parse(args); err != nil { return err } @@ -26,6 +27,9 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error { if err != nil { return err } + if *repsFlag < repsMin || repsMax < *repsFlag { + return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, repsMin, repsMax) + } args = flags.Args() if len(args) == 0 { args = []string{"std/..."} @@ -36,7 +40,7 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error { cmdArgs = append(cmdArgs, "-mimic") } if bench { - cmdArgs = append(cmdArgs, "-bench") + cmdArgs = append(cmdArgs, "-bench", fmt.Sprintf("-reps=%d", *repsFlag)) } failed := false diff --git a/test/c/testlib/testlib.c b/test/c/testlib/testlib.c index 41db11ee..9eebe4ec 100644 --- a/test/c/testlib/testlib.c +++ b/test/c/testlib/testlib.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -95,20 +96,43 @@ void bench_finish(uint64_t reps, uint64_t n_bytes) { int main(int argc, char** argv) { bool bench = false; + int proc_reps = 5; + int i; for (i = 1; i < argc; i++) { - if (!strcmp(argv[i], "-bench")) { + const char* arg = argv[i]; + if (!strcmp(arg, "-bench")) { bench = true; + + } else if (!strncmp(arg, "-reps=", 6)) { + arg += 6; + if (!*arg) { + fprintf(stderr, "missing -reps=N value\n"); + return 1; + } + char* end = NULL; + long int n = strtol(arg, &end, 10); + if (*end) { + fprintf(stderr, "invalid -reps=N value\n"); + return 1; + } + if ((n < 0) || (1000000 < n)) { + fprintf(stderr, "out-of-range -reps=N value\n"); + return 1; + } + proc_reps = n; + } else { - fprintf(stderr, "unknown flag \"%s\"\n", argv[i]); + fprintf(stderr, "unknown flag \"%s\"\n", arg); return 1; } } - int proc_reps = 1; proc* procs = tests; - if (bench) { - proc_reps = 5 + 1; // +1 for the warm up run. + if (!bench) { + proc_reps = 1; + } else { + proc_reps++; // +1 for the warm up run. procs = benches; printf("# %s version %s\n#\n", cc, cc_version); printf( @@ -136,7 +160,8 @@ int main(int argc, char** argv) { } } if (bench) { - printf("# %-16s%-8s(%d benchmarks run)\n", proc_filename, cc, tests_run); + printf("# %-16s%-8s(%d benchmarks run, 1+%d reps per benchmark)\n", + proc_filename, cc, tests_run, proc_reps - 1); } else { printf("%-16s%-8sPASS (%d tests run)\n", proc_filename, cc, tests_run); }