Add mimic tests.

This commit is contained in:
Nigel Tao committed 2017-07-02 11:19:23 +10:00
1 parent a58bfc67f1
commit a65e2bacf0
5 files changed
+220 -21

No files matched your search

+40 -10
View File
@@ -5,16 +5,19 @@
package main
import (
"bufio"
"flag"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
)
var (
bench = flag.Bool("bench", false, "whether to run benchmarks instead of regular tests")
mimic = flag.Bool("mimic", false, "whether to compare Puffs' output with other libraries' output")
)
func main() {
@@ -47,18 +50,26 @@ func do(filename string) (failed bool, err error) {
}
defer os.RemoveAll(workDir)
for _, cc := range []string{"clang", "gcc"} {
in := filename + ".c"
out := filepath.Join(workDir, cc+".out")
in := filename + ".c"
out := filepath.Join(workDir, "a.out")
ccArgs := []string(nil)
if *bench {
ccArgs = append(ccArgs, "-O3")
} else {
// TODO: set these flags even if we pass -O3.
ccArgs = append(ccArgs, "-Wall", "-Werror")
ccArgs := []string(nil)
if *bench {
ccArgs = append(ccArgs, "-O3")
} else {
// TODO: set these flags even if we pass -O3.
ccArgs = append(ccArgs, "-Wall", "-Werror")
}
ccArgs = append(ccArgs, "-std=c99", "-o", out, in)
if *mimic {
extra, err := findPuffsMimicCflags(in)
if err != nil {
return false, err
}
ccArgs = append(ccArgs, "-std=c99", "-o", out, in)
ccArgs = append(ccArgs, extra...)
}
for _, cc := range []string{"clang", "gcc"} {
ccCmd := exec.Command(cc, ccArgs...)
ccCmd.Stdout = os.Stdout
ccCmd.Stderr = os.Stderr
@@ -84,3 +95,22 @@ func do(filename string) (failed bool, err error) {
}
return failed, nil
}
func findPuffsMimicCflags(filename string) ([]string, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
s := bufio.NewScanner(f)
for s.Scan() {
t := s.Text()
const prefix = "// !! puffs mimic cflags:"
if strings.HasPrefix(t, prefix) {
t = strings.TrimSpace(t[len(prefix):])
return strings.Split(t, " "), nil
}
}
return nil, s.Err()
}
+3
View File
@@ -107,6 +107,9 @@ func listDir(puffsRoot string, dirname string, returnSubdirs bool) (filenames []
const (
langsDefault = "c"
langsUsage = `comma-separated list of target languages (file extensions), e.g. "c,go,rs"`
mimicDefault = false
mimicUsage = `whether to compare Puffs' output with other libraries' output`
)
func parseLangs(commaSeparated string) ([]string, error) {
+15 -9
View File
@@ -19,6 +19,7 @@ func doTest(puffsRoot string, args []string) error { return doBenchTest(puffsRo
func doBenchTest(puffsRoot string, args []string, bench bool) error {
flags := flag.NewFlagSet("test", flag.ExitOnError)
langsFlag := flags.String("langs", langsDefault, langsUsage)
mimicFlag := flags.Bool("mimic", mimicDefault, mimicUsage)
if err := flags.Parse(args); err != nil {
return err
}
@@ -31,6 +32,14 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error {
args = []string{"std/..."}
}
cmdArgs := []string(nil)
if *mimicFlag {
cmdArgs = append(cmdArgs, "-mimic")
}
if bench {
cmdArgs = append(cmdArgs, "-bench")
}
failed := false
for _, arg := range args {
recursive := strings.HasSuffix(arg, "/...")
@@ -44,7 +53,7 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error {
}
// Proceed with benching / testing the generated code.
f, err := benchTest(puffsRoot, arg, langs, bench, recursive)
f, err := benchTest(puffsRoot, arg, langs, cmdArgs, recursive)
if err != nil {
return err
}
@@ -56,13 +65,13 @@ func doBenchTest(puffsRoot string, args []string, bench bool) error {
return nil
}
func benchTest(puffsRoot, dirname string, langs []string, bench bool, recursive bool) (failed bool, err error) {
func benchTest(puffsRoot, dirname string, langs []string, cmdArgs []string, recursive bool) (failed bool, err error) {
filenames, dirnames, err := listDir(puffsRoot, dirname, recursive)
if err != nil {
return false, err
}
if len(filenames) > 0 {
f, err := benchTestDir(puffsRoot, dirname, langs, bench)
f, err := benchTestDir(puffsRoot, dirname, langs, cmdArgs)
if err != nil {
return false, err
}
@@ -70,7 +79,7 @@ func benchTest(puffsRoot, dirname string, langs []string, bench bool, recursive
}
if len(dirnames) > 0 {
for _, d := range dirnames {
f, err := benchTest(puffsRoot, filepath.Join(dirname, d), langs, bench, recursive)
f, err := benchTest(puffsRoot, filepath.Join(dirname, d), langs, cmdArgs, recursive)
if err != nil {
return false, err
}
@@ -80,17 +89,14 @@ func benchTest(puffsRoot, dirname string, langs []string, bench bool, recursive
return failed, nil
}
func benchTestDir(puffsRoot string, dirname string, langs []string, bench bool) (failed bool, err error) {
func benchTestDir(puffsRoot string, dirname string, langs []string, cmdArgs []string) (failed bool, err error) {
if packageName := filepath.Base(dirname); !validName(packageName) {
return false, fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName)
}
for _, lang := range langs {
command := "puffs-test-" + lang
args := []string(nil)
if bench {
args = append(args, "-bench")
}
args := append([]string(nil), cmdArgs...)
args = append(args, filepath.Join(puffsRoot, "test", lang, filepath.FromSlash(dirname)))
cmd := exec.Command(command, args...)
cmd.Stdout = os.Stdout