mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Add benchmarks.
This commit is contained in:
1 parent
04e78ad4e7
commit
480c509214
5 files changed
+241
-43
No files matched your search
@@ -1,7 +1,7 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be found
|
||||
// in the LICENSE file.
|
||||
|
||||
// puffs-test-c runs C test programs.
|
||||
// puffs-test-c runs C benchmark and test programs.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -13,6 +13,10 @@ import (
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var (
|
||||
bench = flag.Bool("b", false, "whether to run benchmarks instead of regular tests")
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := main1(); err != nil {
|
||||
os.Stderr.WriteString(err.Error() + "\n")
|
||||
@@ -47,14 +51,26 @@ func do(filename string) (failed bool, err error) {
|
||||
in := filename + ".c"
|
||||
out := filepath.Join(workDir, cc+".out")
|
||||
|
||||
ccCmd := exec.Command(cc, "-std=c99", "-Wall", "-Werror", "-o", out, in)
|
||||
ccArgs := []string(nil)
|
||||
if *bench {
|
||||
ccArgs = append(ccArgs, "-O3")
|
||||
} else {
|
||||
// TODO: set these flags even if we pass -O3.
|
||||
ccArgs = append(ccArgs, "-Wall", "-Werror")
|
||||
}
|
||||
ccArgs = append(ccArgs, "-std=c99", "-o", out, in)
|
||||
ccCmd := exec.Command(cc, ccArgs...)
|
||||
ccCmd.Stdout = os.Stdout
|
||||
ccCmd.Stderr = os.Stderr
|
||||
if err := ccCmd.Run(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
outCmd := exec.Command(out)
|
||||
outArgs := []string(nil)
|
||||
if *bench {
|
||||
outArgs = append(outArgs, "-b")
|
||||
}
|
||||
outCmd := exec.Command(out, outArgs...)
|
||||
outCmd.Stdout = os.Stdout
|
||||
outCmd.Stderr = os.Stderr
|
||||
outCmd.Dir = filepath.Dir(filename)
|
||||
|
||||
@@ -19,6 +19,7 @@ var commands = []struct {
|
||||
name string
|
||||
do func(puffsRoot string, args []string) error
|
||||
}{
|
||||
{"bench", doBench},
|
||||
{"gen", doGen},
|
||||
{"test", doTest},
|
||||
}
|
||||
|
||||
+16
-8
@@ -13,7 +13,10 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
func doTest(puffsRoot string, args []string) error {
|
||||
func doBench(puffsRoot string, args []string) error { return doBenchTest(puffsRoot, args, true) }
|
||||
func doTest(puffsRoot string, args []string) error { return doBenchTest(puffsRoot, args, false) }
|
||||
|
||||
func doBenchTest(puffsRoot string, args []string, bench bool) error {
|
||||
flags := flag.NewFlagSet("test", flag.ExitOnError)
|
||||
langsFlag := flags.String("langs", langsDefault, langsUsage)
|
||||
if err := flags.Parse(args); err != nil {
|
||||
@@ -40,8 +43,8 @@ func doTest(puffsRoot string, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Proceed with testing the generated code.
|
||||
f, err := test(puffsRoot, arg, langs, recursive)
|
||||
// Proceed with benching / testing the generated code.
|
||||
f, err := benchTest(puffsRoot, arg, langs, bench, recursive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -53,13 +56,13 @@ func doTest(puffsRoot string, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func test(puffsRoot, dirname string, langs []string, recursive bool) (failed bool, err error) {
|
||||
func benchTest(puffsRoot, dirname string, langs []string, bench bool, recursive bool) (failed bool, err error) {
|
||||
filenames, dirnames, err := listDir(puffsRoot, dirname, recursive)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(filenames) > 0 {
|
||||
f, err := testDir(puffsRoot, dirname, langs)
|
||||
f, err := benchTestDir(puffsRoot, dirname, langs, bench)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -67,7 +70,7 @@ func test(puffsRoot, dirname string, langs []string, recursive bool) (failed boo
|
||||
}
|
||||
if len(dirnames) > 0 {
|
||||
for _, d := range dirnames {
|
||||
f, err := test(puffsRoot, filepath.Join(dirname, d), langs, recursive)
|
||||
f, err := benchTest(puffsRoot, filepath.Join(dirname, d), langs, bench, recursive)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -77,14 +80,19 @@ func test(puffsRoot, dirname string, langs []string, recursive bool) (failed boo
|
||||
return failed, nil
|
||||
}
|
||||
|
||||
func testDir(puffsRoot string, dirname string, langs []string) (failed bool, err error) {
|
||||
func benchTestDir(puffsRoot string, dirname string, langs []string, bench bool) (failed bool, err error) {
|
||||
if packageName := filepath.Base(dirname); !validName(packageName) {
|
||||
return false, fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName)
|
||||
}
|
||||
|
||||
for _, lang := range langs {
|
||||
command := "puffs-test-" + lang
|
||||
cmd := exec.Command(command, filepath.Join(puffsRoot, "test", lang, filepath.FromSlash(dirname)))
|
||||
args := []string(nil)
|
||||
if bench {
|
||||
args = append(args, "-b")
|
||||
}
|
||||
args = append(args, filepath.Join(puffsRoot, "test", lang, filepath.FromSlash(dirname)))
|
||||
cmd := exec.Command(command, args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err == nil {
|
||||
|
||||
+133
-14
@@ -15,7 +15,7 @@ Each edition should print "PASS", amongst other information, and exit(0).
|
||||
#include "../../../gen/c/std/gif.c"
|
||||
#include "../testlib/testlib.c"
|
||||
|
||||
const char* test_filename = "std/gif.c";
|
||||
const char* proc_filename = "std/gif.c";
|
||||
|
||||
#define BUFFER_SIZE (1024 * 1024)
|
||||
|
||||
@@ -27,7 +27,7 @@ uint8_t global_palette_buffer[3 * 256];
|
||||
// ---------------- Basic Tests
|
||||
|
||||
void test_basic_bad_argument_out_of_range() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
puffs_gif_lzw_decoder dec;
|
||||
puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0);
|
||||
|
||||
@@ -54,7 +54,7 @@ void test_basic_bad_argument_out_of_range() {
|
||||
}
|
||||
|
||||
void test_basic_bad_receiver() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
puffs_base_writer1 dst = {0};
|
||||
puffs_base_reader1 src = {0};
|
||||
puffs_gif_status status = puffs_gif_lzw_decoder_decode(NULL, dst, src);
|
||||
@@ -64,7 +64,7 @@ void test_basic_bad_receiver() {
|
||||
}
|
||||
|
||||
void test_basic_constructor_not_called() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
puffs_gif_lzw_decoder dec = {{0}};
|
||||
puffs_base_writer1 dst = {0};
|
||||
puffs_base_reader1 src = {0};
|
||||
@@ -76,7 +76,7 @@ void test_basic_constructor_not_called() {
|
||||
}
|
||||
|
||||
void test_basic_puffs_version_bad() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
puffs_gif_lzw_decoder dec;
|
||||
puffs_gif_lzw_decoder_constructor(&dec, 0, 0); // 0 is not PUFFS_VERSION.
|
||||
if (dec.private_impl.status != puffs_gif_error_bad_version) {
|
||||
@@ -89,7 +89,7 @@ cleanup0:
|
||||
}
|
||||
|
||||
void test_basic_puffs_version_good() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
puffs_gif_lzw_decoder dec;
|
||||
puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0);
|
||||
if (dec.private_impl.magic != PUFFS_MAGIC) {
|
||||
@@ -106,7 +106,7 @@ cleanup0:
|
||||
}
|
||||
|
||||
void test_basic_status_is_error() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
if (puffs_gif_status_is_error(puffs_gif_status_ok)) {
|
||||
FAIL("is_error(ok) returned true");
|
||||
return;
|
||||
@@ -126,7 +126,7 @@ void test_basic_status_is_error() {
|
||||
}
|
||||
|
||||
void test_basic_status_strings() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
const char* s0 = puffs_gif_status_string(puffs_gif_status_ok);
|
||||
const char* t0 = "gif: ok";
|
||||
if (strcmp(s0, t0)) {
|
||||
@@ -161,7 +161,7 @@ void test_basic_status_strings() {
|
||||
}
|
||||
|
||||
void test_basic_sub_struct_constructor() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
puffs_gif_decoder dec;
|
||||
puffs_gif_decoder_constructor(&dec, PUFFS_VERSION, 0);
|
||||
if (dec.private_impl.magic != PUFFS_MAGIC) {
|
||||
@@ -272,15 +272,65 @@ cleanup0:;
|
||||
}
|
||||
|
||||
void test_lzw_decode_many_small_inputs() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
test_lzw_decode_xxx(100);
|
||||
}
|
||||
|
||||
void test_lzw_decode_one_large_input() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
test_lzw_decode_xxx(1 << 30);
|
||||
}
|
||||
|
||||
// ---------------- LZW Benches
|
||||
|
||||
void bench_lzw_decode_xxx(const char* filename, uint64_t reps) {
|
||||
puffs_base_buf1 dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE};
|
||||
puffs_base_buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE};
|
||||
puffs_base_writer1 dst_writer = {.buf = &dst};
|
||||
puffs_base_reader1 src_reader = {.buf = &src};
|
||||
|
||||
if (!read_file(&src, filename)) {
|
||||
return;
|
||||
}
|
||||
if (src.wi <= 0) {
|
||||
FAIL("src size: got %d, want > 0", (int)(src.wi));
|
||||
return;
|
||||
}
|
||||
uint8_t literal_width = src.ptr[0];
|
||||
if (literal_width != 0x08) {
|
||||
FAIL("LZW literal width: got %d, want %d", (int)(src.ptr[0]), 0x08);
|
||||
return;
|
||||
}
|
||||
|
||||
bench_start();
|
||||
uint64_t n_bytes = 0;
|
||||
for (uint64_t i = 0; i < reps; i++) {
|
||||
dst.wi = 0;
|
||||
src.ri = 1; // Skip the literal width.
|
||||
puffs_gif_lzw_decoder dec;
|
||||
puffs_gif_lzw_decoder_constructor(&dec, PUFFS_VERSION, 0);
|
||||
puffs_gif_status s =
|
||||
puffs_gif_lzw_decoder_decode(&dec, dst_writer, src_reader);
|
||||
puffs_gif_lzw_decoder_destructor(&dec);
|
||||
if (s) {
|
||||
FAIL("decode: %s", puffs_gif_status_string(s));
|
||||
return;
|
||||
}
|
||||
n_bytes += dst.wi;
|
||||
}
|
||||
bench_finish(reps, n_bytes);
|
||||
}
|
||||
|
||||
void bench_lzw_decode_20k() {
|
||||
proc_funcname = __func__;
|
||||
bench_lzw_decode_xxx("../../testdata/bricks-gray.giflzw", 5000);
|
||||
}
|
||||
|
||||
void bench_lzw_decode_100k() {
|
||||
proc_funcname = __func__;
|
||||
bench_lzw_decode_xxx("../../testdata/pi.txt.giflzw", 1000);
|
||||
}
|
||||
|
||||
// ---------------- GIF Tests
|
||||
|
||||
void test_gif_decode_input_is_a_xxx(const char* filename,
|
||||
@@ -343,7 +393,7 @@ cleanup0:;
|
||||
}
|
||||
|
||||
void test_gif_decode_input_is_a_gif() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
test_gif_decode_input_is_a_xxx("../../testdata/bricks-dither.gif",
|
||||
"../../testdata/bricks-dither.palette",
|
||||
"../../testdata/bricks-dither.indexes",
|
||||
@@ -351,15 +401,65 @@ void test_gif_decode_input_is_a_gif() {
|
||||
}
|
||||
|
||||
void test_gif_decode_input_is_a_png() {
|
||||
test_funcname = __func__;
|
||||
proc_funcname = __func__;
|
||||
test_gif_decode_input_is_a_xxx("../../testdata/bricks-dither.png", "", "",
|
||||
puffs_gif_error_bad_gif_header);
|
||||
}
|
||||
|
||||
// ---------------- GIF Benches
|
||||
|
||||
void bench_gif_decode_xxx(const char* filename, uint64_t reps) {
|
||||
puffs_base_buf1 dst = {.ptr = global_got_buffer, .len = BUFFER_SIZE};
|
||||
puffs_base_buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE};
|
||||
puffs_base_writer1 dst_writer = {.buf = &dst};
|
||||
puffs_base_reader1 src_reader = {.buf = &src};
|
||||
|
||||
if (!read_file(&src, filename)) {
|
||||
return;
|
||||
}
|
||||
|
||||
bench_start();
|
||||
uint64_t n_bytes = 0;
|
||||
for (uint64_t i = 0; i < reps; i++) {
|
||||
dst.wi = 0;
|
||||
src.ri = 0;
|
||||
puffs_gif_decoder dec;
|
||||
puffs_gif_decoder_constructor(&dec, PUFFS_VERSION, 0);
|
||||
puffs_gif_status s = puffs_gif_decoder_decode(&dec, dst_writer, src_reader);
|
||||
puffs_gif_decoder_destructor(&dec);
|
||||
if (s) {
|
||||
FAIL("decode: %s", puffs_gif_status_string(s));
|
||||
return;
|
||||
}
|
||||
n_bytes += dst.wi;
|
||||
}
|
||||
bench_finish(reps, n_bytes);
|
||||
}
|
||||
|
||||
void bench_gif_decode_1k() {
|
||||
proc_funcname = __func__;
|
||||
bench_gif_decode_xxx("../../testdata/pjw-thumbnail.gif", 200000);
|
||||
}
|
||||
|
||||
void bench_gif_decode_20k() {
|
||||
proc_funcname = __func__;
|
||||
bench_gif_decode_xxx("../../testdata/bricks-dither.gif", 5000);
|
||||
}
|
||||
|
||||
void bench_gif_decode_100k() {
|
||||
proc_funcname = __func__;
|
||||
bench_gif_decode_xxx("../../testdata/hibiscus.gif", 1000);
|
||||
}
|
||||
|
||||
void bench_gif_decode_1000k() {
|
||||
proc_funcname = __func__;
|
||||
bench_gif_decode_xxx("../../testdata/harvesters.gif", 100);
|
||||
}
|
||||
|
||||
// ---------------- Manifest
|
||||
|
||||
// The empty comments forces clang-format to place one element per line.
|
||||
test tests[] = {
|
||||
proc tests[] = {
|
||||
// Basic Tests
|
||||
test_basic_bad_argument_out_of_range, //
|
||||
test_basic_bad_receiver, //
|
||||
@@ -369,12 +469,31 @@ test tests[] = {
|
||||
test_basic_status_is_error, //
|
||||
test_basic_status_strings, //
|
||||
test_basic_sub_struct_constructor, //
|
||||
|
||||
// LZW Tests
|
||||
test_lzw_decode_many_small_inputs, //
|
||||
test_lzw_decode_one_large_input, //
|
||||
|
||||
// GIF Tests
|
||||
test_gif_decode_input_is_a_gif, //
|
||||
test_gif_decode_input_is_a_png, //
|
||||
|
||||
// End
|
||||
NULL,
|
||||
};
|
||||
|
||||
// The empty comments forces clang-format to place one element per line.
|
||||
proc benches[] = {
|
||||
// LZW Benches
|
||||
bench_lzw_decode_20k, //
|
||||
bench_lzw_decode_100k, //
|
||||
|
||||
// GIF Benches
|
||||
bench_gif_decode_1k, //
|
||||
bench_gif_decode_20k, //
|
||||
bench_gif_decode_100k, //
|
||||
bench_gif_decode_1000k, //
|
||||
|
||||
// End
|
||||
NULL,
|
||||
};
|
||||
+72
-18
@@ -2,8 +2,10 @@
|
||||
// in the LICENSE file.
|
||||
|
||||
#include <errno.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
char fail_msg[65536] = {0};
|
||||
|
||||
@@ -13,36 +15,88 @@ char fail_msg[65536] = {0};
|
||||
|
||||
int tests_run = 0;
|
||||
|
||||
typedef void (*test)();
|
||||
typedef void (*proc)();
|
||||
|
||||
test tests[];
|
||||
proc benches[];
|
||||
proc tests[];
|
||||
|
||||
const char* test_filename;
|
||||
const char* test_funcname = "";
|
||||
const char* proc_filename;
|
||||
const char* proc_funcname = "";
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
// The order matters here. Clang also defines "__GNUC__".
|
||||
#if defined(__clang__)
|
||||
const char* cc = "clang";
|
||||
const char* cc = "clang";
|
||||
#elif defined(__GNUC__)
|
||||
const char* cc = "gcc";
|
||||
const char* cc = "gcc";
|
||||
#elif defined(_MSC_VER)
|
||||
const char* cc = "cl";
|
||||
const char* cc = "cl";
|
||||
#else
|
||||
const char* cc = "cc";
|
||||
const char* cc = "cc";
|
||||
#endif
|
||||
|
||||
for (test* t = tests; *t; t++) {
|
||||
test_funcname = "unknown_test_funcname";
|
||||
(*t)();
|
||||
if (fail_msg[0]) {
|
||||
printf("%-16s%-8sFAIL %s: %s\n", test_filename, cc, test_funcname,
|
||||
fail_msg);
|
||||
return 1;
|
||||
bool bench_warm_up;
|
||||
struct timeval bench_start_tv;
|
||||
|
||||
void bench_start() {
|
||||
gettimeofday(&bench_start_tv, NULL);
|
||||
}
|
||||
|
||||
void bench_finish(uint64_t reps, uint64_t n_bytes) {
|
||||
struct timeval bench_finish_tv;
|
||||
gettimeofday(&bench_finish_tv, NULL);
|
||||
int64_t micros =
|
||||
(int64_t)(bench_finish_tv.tv_sec - bench_start_tv.tv_sec) * 1000000 +
|
||||
(int64_t)(bench_finish_tv.tv_usec - bench_start_tv.tv_usec);
|
||||
uint64_t nanos = 1;
|
||||
if (micros > 0) {
|
||||
nanos = (uint64_t)(micros)*1000;
|
||||
}
|
||||
uint64_t kb_per_s = n_bytes * 1000000 / nanos;
|
||||
if (bench_warm_up) {
|
||||
printf("# (warm up) %s_%s\t%8" PRIu64 ".%06" PRIu64 " seconds\n",
|
||||
proc_funcname, cc, nanos / 1000000000, (nanos % 1000000000) / 1000);
|
||||
} else {
|
||||
printf("Benchmark%s_%s\t%8" PRIu64 "\t%8" PRIu64 " ns/op\t%8d.%03d MB/s\n",
|
||||
proc_funcname, cc, reps, nanos / reps, (int)(kb_per_s / 1000),
|
||||
(int)(kb_per_s % 1000));
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
bool bench = false;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (!strcmp(argv[i], "-b")) {
|
||||
bench = true;
|
||||
}
|
||||
tests_run++;
|
||||
}
|
||||
|
||||
int proc_reps = 1;
|
||||
proc* procs = tests;
|
||||
if (bench) {
|
||||
proc_reps = 5 + 1; // +1 for the warm up run.
|
||||
procs = benches;
|
||||
printf(
|
||||
"# The output format, including the redundant \"Benchmark\"s, is "
|
||||
"compatible with\n# "
|
||||
"https://godoc.org/golang.org/x/perf/cmd/benchstat\n");
|
||||
}
|
||||
|
||||
for (int i = 0; i < proc_reps; i++) {
|
||||
bench_warm_up = i == 0;
|
||||
for (proc* t = procs; *t; t++) {
|
||||
proc_funcname = "unknown_funcname";
|
||||
(*t)();
|
||||
if (fail_msg[0]) {
|
||||
printf("%-16s%-8sFAIL %s: %s\n", proc_filename, cc, proc_funcname,
|
||||
fail_msg);
|
||||
return 1;
|
||||
}
|
||||
tests_run++;
|
||||
}
|
||||
}
|
||||
if (!bench) {
|
||||
printf("%-16s%-8sPASS (%d tests run)\n", proc_filename, cc, tests_run);
|
||||
}
|
||||
printf("%-16s%-8sPASS (%d tests run)\n", test_filename, cc, tests_run);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user