Add a -reps=N flag to "puffs bench".

This commit is contained in:
Nigel Tao committed 2017-07-30 08:27:52 +10:00
1 parent 0a45eee9d0
commit e0f87fb2a7
4 files changed
+63 -14

No files matched your search

+22 -7
View File
@@ -15,9 +15,20 @@ import (
"strings"
)
const (
mimicDefault = false
mimicUsage = `whether to compare Puffs' output with other libraries' output`
repsDefault = 5
repsMin = 0
repsMax = 1000000
repsUsage = `the number of repetitions per benchmark`
)
var (
bench = flag.Bool("bench", false, "whether to run benchmarks instead of regular tests")
mimic = flag.Bool("mimic", false, "whether to compare Puffs' output with other libraries' output")
benchFlag = flag.Bool("bench", false, "whether to run benchmarks instead of regular tests")
mimicFlag = flag.Bool("mimic", mimicDefault, mimicUsage)
repsFlag = flag.Int("reps", repsDefault, repsUsage)
)
func main() {
@@ -29,6 +40,10 @@ func main() {
func main1() error {
flag.Parse()
if *repsFlag < repsMin || repsMax < *repsFlag {
return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, repsMin, repsMax)
}
failed := false
for _, arg := range flag.Args() {
f, err := do(arg)
@@ -39,7 +54,7 @@ func main1() error {
}
if failed {
s := "tests"
if *bench {
if *benchFlag {
s = "benchmarks"
}
return fmt.Errorf("%s: some %s failed", os.Args[0], s)
@@ -58,14 +73,14 @@ func do(filename string) (failed bool, err error) {
out := filepath.Join(workDir, "a.out")
ccArgs := []string(nil)
if *bench {
if *benchFlag {
ccArgs = append(ccArgs, "-O3")
} else {
// TODO: set these flags even if we pass -O3.
ccArgs = append(ccArgs, "-Wall", "-Werror")
}
ccArgs = append(ccArgs, "-std=c99", "-o", out, in)
if *mimic {
if *mimicFlag {
extra, err := findPuffsMimicCflags(in)
if err != nil {
return false, err
@@ -82,8 +97,8 @@ func do(filename string) (failed bool, err error) {
}
outArgs := []string(nil)
if *bench {
outArgs = append(outArgs, "-bench")
if *benchFlag {
outArgs = append(outArgs, "-bench", fmt.Sprintf("-reps=%d", *repsFlag))
}
outCmd := exec.Command(out, outArgs...)
outCmd.Stdout = os.Stdout
+5
View File
@@ -110,6 +110,11 @@ const (
mimicDefault = false
mimicUsage = `whether to compare Puffs' output with other libraries' output`
repsDefault = 5
repsMin = 0
repsMax = 1000000
repsUsage = `the number of repetitions per benchmark`
)
func parseLangs(commaSeparated string) ([]string, error) {
+5 -1
View File
@@ -19,6 +19,7 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error {
flags := flag.NewFlagSet("test", flag.ExitOnError)
langsFlag := flags.String("langs", langsDefault, langsUsage)
mimicFlag := flags.Bool("mimic", mimicDefault, mimicUsage)
repsFlag := flags.Int("reps", repsDefault, repsUsage)
if err := flags.Parse(args); err != nil {
return err
}
@@ -26,6 +27,9 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error {
if err != nil {
return err
}
if *repsFlag < repsMin || repsMax < *repsFlag {
return fmt.Errorf("bad -reps flag value %d, outside the range [%d..%d]", *repsFlag, repsMin, repsMax)
}
args = flags.Args()
if len(args) == 0 {
args = []string{"std/..."}
@@ -36,7 +40,7 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error {
cmdArgs = append(cmdArgs, "-mimic")
}
if bench {
cmdArgs = append(cmdArgs, "-bench")
cmdArgs = append(cmdArgs, "-bench", fmt.Sprintf("-reps=%d", *repsFlag))
}
failed := false
+31 -6
View File
@@ -4,6 +4,7 @@
#include <errno.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
@@ -95,20 +96,43 @@ void bench_finish(uint64_t reps, uint64_t n_bytes) {
int main(int argc, char** argv) {
bool bench = false;
int proc_reps = 5;
int i;
for (i = 1; i < argc; i++) {
if (!strcmp(argv[i], "-bench")) {
const char* arg = argv[i];
if (!strcmp(arg, "-bench")) {
bench = true;
} else if (!strncmp(arg, "-reps=", 6)) {
arg += 6;
if (!*arg) {
fprintf(stderr, "missing -reps=N value\n");
return 1;
}
char* end = NULL;
long int n = strtol(arg, &end, 10);
if (*end) {
fprintf(stderr, "invalid -reps=N value\n");
return 1;
}
if ((n < 0) || (1000000 < n)) {
fprintf(stderr, "out-of-range -reps=N value\n");
return 1;
}
proc_reps = n;
} else {
fprintf(stderr, "unknown flag \"%s\"\n", argv[i]);
fprintf(stderr, "unknown flag \"%s\"\n", arg);
return 1;
}
}
int proc_reps = 1;
proc* procs = tests;
if (bench) {
proc_reps = 5 + 1; // +1 for the warm up run.
if (!bench) {
proc_reps = 1;
} else {
proc_reps++; // +1 for the warm up run.
procs = benches;
printf("# %s version %s\n#\n", cc, cc_version);
printf(
@@ -136,7 +160,8 @@ int main(int argc, char** argv) {
}
}
if (bench) {
printf("# %-16s%-8s(%d benchmarks run)\n", proc_filename, cc, tests_run);
printf("# %-16s%-8s(%d benchmarks run, 1+%d reps per benchmark)\n",
proc_filename, cc, tests_run, proc_reps - 1);
} else {
printf("%-16s%-8sPASS (%d tests run)\n", proc_filename, cc, tests_run);
}