Add benchmarks.

This commit is contained in:
Nigel Tao committed 2017-06-29 14:13:23 +10:00
1 parent 04e78ad4e7
commit 480c509214
5 files changed
+241 -43

No files matched your search

+19 -3
View File
@@ -1,7 +1,7 @@
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
// puffs-test-c runs C test programs.
// puffs-test-c runs C benchmark and test programs.
package main
import (
@@ -13,6 +13,10 @@ import (
"path/filepath"
)
var (
bench = flag.Bool("b", false, "whether to run benchmarks instead of regular tests")
)
func main() {
if err := main1(); err != nil {
os.Stderr.WriteString(err.Error() + "\n")
@@ -47,14 +51,26 @@ func do(filename string) (failed bool, err error) {
in := filename + ".c"
out := filepath.Join(workDir, cc+".out")
ccCmd := exec.Command(cc, "-std=c99", "-Wall", "-Werror", "-o", out, in)
ccArgs := []string(nil)
if *bench {
ccArgs = append(ccArgs, "-O3")
} else {
// TODO: set these flags even if we pass -O3.
ccArgs = append(ccArgs, "-Wall", "-Werror")
}
ccArgs = append(ccArgs, "-std=c99", "-o", out, in)
ccCmd := exec.Command(cc, ccArgs...)
ccCmd.Stdout = os.Stdout
ccCmd.Stderr = os.Stderr
if err := ccCmd.Run(); err != nil {
return false, err
}
outCmd := exec.Command(out)
outArgs := []string(nil)
if *bench {
outArgs = append(outArgs, "-b")
}
outCmd := exec.Command(out, outArgs...)
outCmd.Stdout = os.Stdout
outCmd.Stderr = os.Stderr
outCmd.Dir = filepath.Dir(filename)
+1
View File
@@ -19,6 +19,7 @@ var commands = []struct {
name string
do func(puffsRoot string, args []string) error
}{
{"bench", doBench},
{"gen", doGen},
{"test", doTest},
}
+16 -8
View File
@@ -13,7 +13,10 @@ import (
"strings"
)
func doTest(puffsRoot string, args []string) error {
func doBench(puffsRoot string, args []string) error { return doBenchTest(puffsRoot, args, true) }
func doTest(puffsRoot string, args []string) error { return doBenchTest(puffsRoot, args, false) }
func doBenchTest(puffsRoot string, args []string, bench bool) error {
flags := flag.NewFlagSet("test", flag.ExitOnError)
langsFlag := flags.String("langs", langsDefault, langsUsage)
if err := flags.Parse(args); err != nil {
@@ -40,8 +43,8 @@ func doTest(puffsRoot string, args []string) error {
return err
}
// Proceed with testing the generated code.
f, err := test(puffsRoot, arg, langs, recursive)
// Proceed with benching / testing the generated code.
f, err := benchTest(puffsRoot, arg, langs, bench, recursive)
if err != nil {
return err
}
@@ -53,13 +56,13 @@ func doTest(puffsRoot string, args []string) error {
return nil
}
func test(puffsRoot, dirname string, langs []string, recursive bool) (failed bool, err error) {
func benchTest(puffsRoot, dirname string, langs []string, bench bool, recursive bool) (failed bool, err error) {
filenames, dirnames, err := listDir(puffsRoot, dirname, recursive)
if err != nil {
return false, err
}
if len(filenames) > 0 {
f, err := testDir(puffsRoot, dirname, langs)
f, err := benchTestDir(puffsRoot, dirname, langs, bench)
if err != nil {
return false, err
}
@@ -67,7 +70,7 @@ func test(puffsRoot, dirname string, langs []string, recursive bool) (failed boo
}
if len(dirnames) > 0 {
for _, d := range dirnames {
f, err := test(puffsRoot, filepath.Join(dirname, d), langs, recursive)
f, err := benchTest(puffsRoot, filepath.Join(dirname, d), langs, bench, recursive)
if err != nil {
return false, err
}
@@ -77,14 +80,19 @@ func test(puffsRoot, dirname string, langs []string, recursive bool) (failed boo
return failed, nil
}
func testDir(puffsRoot string, dirname string, langs []string) (failed bool, err error) {
func benchTestDir(puffsRoot string, dirname string, langs []string, bench bool) (failed bool, err error) {
if packageName := filepath.Base(dirname); !validName(packageName) {
return false, fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName)
}
for _, lang := range langs {
command := "puffs-test-" + lang
cmd := exec.Command(command, filepath.Join(puffsRoot, "test", lang, filepath.FromSlash(dirname)))
args := []string(nil)
if bench {
args = append(args, "-b")
}
args = append(args, filepath.Join(puffsRoot, "test", lang, filepath.FromSlash(dirname)))
cmd := exec.Command(command, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err == nil {
+133 -14
View File
@@ -15,7 +15,7 @@ Each edition should print "PASS", amongst other information, and exit(0).
#include "../../../gen/c/std/gif.c"
#include "../testlib/testlib.c"
const char* test_filename = "std/gif.c";
const char* proc_filename = "std/gif.c";
#define BUFFER_SIZE (1024 * 1024)
@@ -27,7 +27,7 @@ uint8_t global_palette_buffer[3 * 256];
// ---------------- Basic Tests
void test_basic_bad_argument_out_of_range() {
test_funcname = __func__;
proc_funcname = __func__;
puffs_gif_lzw_decoder dec;
puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0);
@@ -54,7 +54,7 @@ void test_basic_bad_argument_out_of_range() {
}
void test_basic_bad_receiver() {
test_funcname = __func__;
proc_funcname = __func__;
puffs_base_writer1 dst = {0};
puffs_base_reader1 src = {0};
puffs_gif_status status = puffs_gif_lzw_decoder_decode(NULL, dst, src);
@@ -64,7 +64,7 @@ void test_basic_bad_receiver() {
}
void test_basic_constructor_not_called() {
test_funcname = __func__;
proc_funcname = __func__;
puffs_gif_lzw_decoder dec = {{0}};
puffs_base_writer1 dst = {0};
puffs_base_reader1 src = {0};
@@ -76,7 +76,7 @@ void test_basic_constructor_not_called() {
}
void test_basic_puffs_version_bad() {
test_funcname = __func__;
proc_funcname = __func__;
puffs_gif_lzw_decoder dec;
puffs_gif_lzw_decoder_constructor(&dec, 0, 0); // 0 is not PUFFS_VERSION.
if (dec.private_impl.status != puffs_gif_error_bad_version) {
@@ -89,7 +89,7 @@ cleanup0:
}
void test_basic_puffs_version_good() {
test_funcname = __func__;
proc_funcname = __func__;
puffs_gif_lzw_decoder dec;
puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0);
if (dec.private_impl.magic != PUFFS_MAGIC) {
@@ -106,7 +106,7 @@ cleanup0:
}
void test_basic_status_is_error() {
test_funcname = __func__;
proc_funcname = __func__;
if (puffs_gif_status_is_error(puffs_gif_status_ok)) {
FAIL("is_error(ok) returned true");
return;
@@ -126,7 +126,7 @@ void test_basic_status_is_error() {
}
void test_basic_status_strings() {
test_funcname = __func__;
proc_funcname = __func__;
const char* s0 = puffs_gif_status_string(puffs_gif_status_ok);
const char* t0 = "gif: ok";
if (strcmp(s0, t0)) {
@@ -161,7 +161,7 @@ void test_basic_status_strings() {
}
void test_basic_sub_struct_constructor() {
test_funcname = __func__;
proc_funcname = __func__;
puffs_gif_decoder dec;
puffs_gif_decoder_constructor(&dec, PUFFS_VERSION, 0);
if (dec.private_impl.magic != PUFFS_MAGIC) {
@@ -272,15 +272,65 @@ cleanup0:;
}
void test_lzw_decode_many_small_inputs() {
test_funcname = __func__;
proc_funcname = __func__;
test_lzw_decode_xxx(100);
}
void test_lzw_decode_one_large_input() {
test_funcname = __func__;
proc_funcname = __func__;
test_lzw_decode_xxx(1 << 30);
}
// ---------------- LZW Benches
void bench_lzw_decode_xxx(const char* filename, uint64_t reps) {
puffs_base_buf1 dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE};
puffs_base_buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE};
puffs_base_writer1 dst_writer = {.buf = &dst};
puffs_base_reader1 src_reader = {.buf = &src};
if (!read_file(&src, filename)) {
return;
}
if (src.wi <= 0) {
FAIL("src size: got %d, want > 0", (int)(src.wi));
return;
}
uint8_t literal_width = src.ptr[0];
if (literal_width != 0x08) {
FAIL("LZW literal width: got %d, want %d", (int)(src.ptr[0]), 0x08);
return;
}
bench_start();
uint64_t n_bytes = 0;
for (uint64_t i = 0; i < reps; i++) {
dst.wi = 0;
src.ri = 1; // Skip the literal width.
puffs_gif_lzw_decoder dec;
puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0);
puffs_gif_status s =
puffs_gif_lzw_decoder_decode(&dec, dst_writer, src_reader);
puffs_gif_lzw_decoder_destructor(&dec);
if (s) {
FAIL("decode: %s", puffs_gif_status_string(s));
return;
}
n_bytes += dst.wi;
}
bench_finish(reps, n_bytes);
}
void bench_lzw_decode_20k() {
proc_funcname = __func__;
bench_lzw_decode_xxx("../../testdata/bricks-gray.giflzw", 5000);
}
void bench_lzw_decode_100k() {
proc_funcname = __func__;
bench_lzw_decode_xxx("../../testdata/pi.txt.giflzw", 1000);
}
// ---------------- GIF Tests
void test_gif_decode_input_is_a_xxx(const char* filename,
@@ -343,7 +393,7 @@ cleanup0:;
}
void test_gif_decode_input_is_a_gif() {
test_funcname = __func__;
proc_funcname = __func__;
test_gif_decode_input_is_a_xxx("../../testdata/bricks-dither.gif",
"../../testdata/bricks-dither.palette",
"../../testdata/bricks-dither.indexes",
@@ -351,15 +401,65 @@ void test_gif_decode_input_is_a_gif() {
}
void test_gif_decode_input_is_a_png() {
test_funcname = __func__;
proc_funcname = __func__;
test_gif_decode_input_is_a_xxx("../../testdata/bricks-dither.png", "", "",
puffs_gif_error_bad_gif_header);
}
// ---------------- GIF Benches
void bench_gif_decode_xxx(const char* filename, uint64_t reps) {
puffs_base_buf1 dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE};
puffs_base_buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE};
puffs_base_writer1 dst_writer = {.buf = &dst};
puffs_base_reader1 src_reader = {.buf = &src};
if (!read_file(&src, filename)) {
return;
}
bench_start();
uint64_t n_bytes = 0;
for (uint64_t i = 0; i < reps; i++) {
dst.wi = 0;
src.ri = 0;
puffs_gif_decoder dec;
puffs_gif_decoder_constructor(&dec, PUFFS_VERSION, 0);
puffs_gif_status s = puffs_gif_decoder_decode(&dec, dst_writer, src_reader);
puffs_gif_decoder_destructor(&dec);
if (s) {
FAIL("decode: %s", puffs_gif_status_string(s));
return;
}
n_bytes += dst.wi;
}
bench_finish(reps, n_bytes);
}
void bench_gif_decode_1k() {
proc_funcname = __func__;
bench_gif_decode_xxx("../../testdata/pjw-thumbnail.gif", 200000);
}
void bench_gif_decode_20k() {
proc_funcname = __func__;
bench_gif_decode_xxx("../../testdata/bricks-dither.gif", 5000);
}
void bench_gif_decode_100k() {
proc_funcname = __func__;
bench_gif_decode_xxx("../../testdata/hibiscus.gif", 1000);
}
void bench_gif_decode_1000k() {
proc_funcname = __func__;
bench_gif_decode_xxx("../../testdata/harvesters.gif", 100);
}
// ---------------- Manifest
// The empty comments forces clang-format to place one element per line.
test tests[] = {
proc tests[] = {
// Basic Tests
test_basic_bad_argument_out_of_range, //
test_basic_bad_receiver, //
@@ -369,12 +469,31 @@ test tests[] = {
test_basic_status_is_error, //
test_basic_status_strings, //
test_basic_sub_struct_constructor, //
// LZW Tests
test_lzw_decode_many_small_inputs, //
test_lzw_decode_one_large_input, //
// GIF Tests
test_gif_decode_input_is_a_gif, //
test_gif_decode_input_is_a_png, //
// End
NULL,
};
// The empty comments forces clang-format to place one element per line.
proc benches[] = {
// LZW Benches
bench_lzw_decode_20k, //
bench_lzw_decode_100k, //
// GIF Benches
bench_gif_decode_1k, //
bench_gif_decode_20k, //
bench_gif_decode_100k, //
bench_gif_decode_1000k, //
// End
NULL,
};
+72 -18
View File
@@ -2,8 +2,10 @@
// in the LICENSE file.
#include <errno.h>
#include <inttypes.h>
#include <stdio.h>
#include <string.h>
#include <sys/time.h>
char fail_msg[65536] = {0};
@@ -13,36 +15,88 @@ char fail_msg[65536] = {0};
int tests_run = 0;
typedef void (*test)();
typedef void (*proc)();
test tests[];
proc benches[];
proc tests[];
const char* test_filename;
const char* test_funcname = "";
const char* proc_filename;
const char* proc_funcname = "";
int main(int argc, char** argv) {
// The order matters here. Clang also defines "__GNUC__".
#if defined(__clang__)
const char* cc = "clang";
const char* cc = "clang";
#elif defined(__GNUC__)
const char* cc = "gcc";
const char* cc = "gcc";
#elif defined(_MSC_VER)
const char* cc = "cl";
const char* cc = "cl";
#else
const char* cc = "cc";
const char* cc = "cc";
#endif
for (test* t = tests; *t; t++) {
test_funcname = "unknown_test_funcname";
(*t)();
if (fail_msg[0]) {
printf("%-16s%-8sFAIL %s: %s\n", test_filename, cc, test_funcname,
fail_msg);
return 1;
bool bench_warm_up;
struct timeval bench_start_tv;
void bench_start() {
gettimeofday(&bench_start_tv, NULL);
}
void bench_finish(uint64_t reps, uint64_t n_bytes) {
struct timeval bench_finish_tv;
gettimeofday(&bench_finish_tv, NULL);
int64_t micros =
(int64_t)(bench_finish_tv.tv_sec - bench_start_tv.tv_sec) * 1000000 +
(int64_t)(bench_finish_tv.tv_usec - bench_start_tv.tv_usec);
uint64_t nanos = 1;
if (micros > 0) {
nanos = (uint64_t)(micros)*1000;
}
uint64_t kb_per_s = n_bytes * 1000000 / nanos;
if (bench_warm_up) {
printf("# (warm up) %s_%s\t%8" PRIu64 ".%06" PRIu64 " seconds\n",
proc_funcname, cc, nanos / 1000000000, (nanos % 1000000000) / 1000);
} else {
printf("Benchmark%s_%s\t%8" PRIu64 "\t%8" PRIu64 " ns/op\t%8d.%03d MB/s\n",
proc_funcname, cc, reps, nanos / reps, (int)(kb_per_s / 1000),
(int)(kb_per_s % 1000));
}
}
int main(int argc, char** argv) {
bool bench = false;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "-b")) {
bench = true;
}
tests_run++;
}
int proc_reps = 1;
proc* procs = tests;
if (bench) {
proc_reps = 5 + 1; // +1 for the warm up run.
procs = benches;
printf(
"# The output format, including the redundant \"Benchmark\"s, is "
"compatible with\n# "
"https://godoc.org/golang.org/x/perf/cmd/benchstat\n");
}
for (int i = 0; i < proc_reps; i++) {
bench_warm_up = i == 0;
for (proc* t = procs; *t; t++) {
proc_funcname = "unknown_funcname";
(*t)();
if (fail_msg[0]) {
printf("%-16s%-8sFAIL %s: %s\n", proc_filename, cc, proc_funcname,
fail_msg);
return 1;
}
tests_run++;
}
}
if (!bench) {
printf("%-16s%-8sPASS (%d tests run)\n", proc_filename, cc, tests_run);
}
printf("%-16s%-8sPASS (%d tests run)\n", test_filename, cc, tests_run);
return 0;
}