Add zlib (not just raw flate) benchmark.

This commit is contained in:
Nigel Tao committed 2017-07-14 12:00:00 +10:00
1 parent 6c8414099a
commit 03572374dd
5 files changed
+142 -12

No files matched your search

+73 -6
View File
@@ -9,14 +9,24 @@ package main
// flate-compressed data wrapped in a .gz file.
//
// Usage: go run extract-flate-offsets.go foo.gz bar.gz
//
// Alternatively: go run extract-flate-offsets.go -write-zlib foo.gz
import (
"bytes"
"compress/flate"
"compress/zlib"
"crypto/md5"
"flag"
"fmt"
"hash/adler32"
"io/ioutil"
"os"
"strings"
)
var (
writeZlib = flag.Bool("write-zlib", false, "whether to convert gzip to zlib")
)
// GZIP wraps a header and footer around flate data. The format is described in
@@ -37,7 +47,8 @@ func main() {
}
func main1() error {
for _, a := range os.Args[1:] {
flag.Parse()
for _, a := range flag.Args() {
if err := decode(a); err != nil {
return err
}
@@ -86,12 +97,55 @@ func decode(filename string) error {
}
// As a sanity check, the result should be valid flate.
hash, err := checkFlate(src[i:])
uncompressed, err := checkFlate(src[i:])
if err != nil {
return err
}
fmt.Printf("%7d %7d %x %s\n", i, len(src), hash, filename)
if *writeZlib {
return doWriteZlib(src[i:], uncompressed, filename)
}
fmt.Printf("%7d %7d %x %s\n", i, len(src), md5.Sum(uncompressed), filename)
return nil
}
func doWriteZlib(flateCompressed []byte, uncompressed []byte, filename string) error {
buf := bytes.NewBuffer(nil)
// The ZLIB header (as per https://www.ietf.org/rfc/rfc1950.txt) is 2
// bytes.
//
// The first byte's low 4 bits is the compression method: 8 means flate.
// The first byte's high 4 bits is the compression info: 7 means a 32KiB
// flate window size.
//
// The second byte's low 5 bits are a parity check. The 5th bit (0 in this
// case) indicates a preset dictionary. The high 2 bits (2 in this case)
// means the default compression algorithm.
buf.WriteString("\x78\x9c")
// Write the payload.
buf.Write(flateCompressed)
// The ZLIB footer is 4 bytes: a big-endian checksum.
checksum := adler32.Checksum(uncompressed)
buf.WriteByte(uint8(checksum >> 24))
buf.WriteByte(uint8(checksum >> 16))
buf.WriteByte(uint8(checksum >> 8))
buf.WriteByte(uint8(checksum >> 0))
asZlib := buf.Bytes()
// As a sanity check, the result should be valid zlib.
if _, err := checkZlib(asZlib); err != nil {
return err
}
if strings.HasSuffix(filename, ".gz") {
filename = filename[:len(filename)-3]
}
filename += ".zlib"
if err := ioutil.WriteFile(filename, asZlib, 0666); err != nil {
return err
}
fmt.Printf("wrote %s\n", filename)
return nil
}
@@ -107,12 +161,25 @@ func readString(src []byte, i int) (int, error) {
}
}
func checkFlate(x []byte) ([md5.Size]byte, error) {
func checkFlate(x []byte) ([]byte, error) {
rc := flate.NewReader(bytes.NewReader(x))
defer rc.Close()
x, err := ioutil.ReadAll(rc)
if err != nil {
return [md5.Size]byte{}, fmt.Errorf("data is not valid flate: %v", err)
return nil, fmt.Errorf("data is not valid flate: %v", err)
}
return md5.Sum(x), nil
return x, nil
}
func checkZlib(x []byte) ([]byte, error) {
rc, err := zlib.NewReader(bytes.NewReader(x))
if err != nil {
return nil, fmt.Errorf("data is not valid zlib: %v", err)
}
defer rc.Close()
x, err = ioutil.ReadAll(rc)
if err != nil {
return nil, fmt.Errorf("data is not valid zlib: %v", err)
}
return x, nil
}
+18 -6
View File
@@ -57,15 +57,19 @@ cleanup0:;
return ret;
}
const char* mimic_gzip_decode(puffs_base_buf1* dst, puffs_base_buf1* src) {
const char* mimic_gzip_zlib_decode(puffs_base_buf1* dst,
puffs_base_buf1* src,
bool gzip_instead_of_zlib) {
const char* ret = NULL;
z_stream z = {0};
const int window_bits = 15;
// See inflateInit2 in the zlib manual, or in zlib.h, for details about this
// magic gzip_instead_of_zlib constant.
const int gzip_instead_of_zlib = 16;
int ii2_err = inflateInit2(&z, window_bits | gzip_instead_of_zlib);
int window_bits = 15;
if (gzip_instead_of_zlib) {
// See inflateInit2 in the zlib manual, or in zlib.h, for details about
// this magic constant.
window_bits |= 16;
}
int ii2_err = inflateInit2(&z, window_bits);
if (ii2_err != Z_OK) {
ret = "inflateInit2 failed";
goto cleanup0;
@@ -107,3 +111,11 @@ cleanup1:;
cleanup0:;
return ret;
}
const char* mimic_gzip_decode(puffs_base_buf1* dst, puffs_base_buf1* src) {
return mimic_gzip_zlib_decode(dst, src, true);
}
const char* mimic_zlib_decode(puffs_base_buf1* dst, puffs_base_buf1* src) {
return mimic_gzip_zlib_decode(dst, src, false);
}
+51
View File
@@ -67,11 +67,26 @@ golden_test flate_romeo_gt = {
.src_offset1 = 550, //
};
golden_test gzip_midsummer_gt = {
.want_filename = "../../testdata/midsummer.txt", //
.src_filename = "../../testdata/midsummer.txt.gz", //
};
golden_test gzip_pi_gt = {
.want_filename = "../../testdata/pi.txt", //
.src_filename = "../../testdata/pi.txt.gz", //
};
golden_test zlib_midsummer_gt = {
.want_filename = "../../testdata/midsummer.txt", //
.src_filename = "../../testdata/midsummer.txt.zlib", //
};
golden_test zlib_pi_gt = {
.want_filename = "../../testdata/pi.txt", //
.src_filename = "../../testdata/pi.txt.zlib", //
};
// ---------------- Flate Tests
const char* puffs_flate_decode(puffs_base_buf1* dst, puffs_base_buf1* src) {
@@ -200,11 +215,26 @@ void test_mimic_flate_decode_romeo() {
test_buf1_buf1(mimic_flate_decode, &flate_romeo_gt);
}
void test_mimic_gzip_decode_midsummer() {
proc_funcname = __func__;
test_buf1_buf1(mimic_gzip_decode, &gzip_midsummer_gt);
}
void test_mimic_gzip_decode_pi() {
proc_funcname = __func__;
test_buf1_buf1(mimic_gzip_decode, &gzip_pi_gt);
}
void test_mimic_zlib_decode_midsummer() {
proc_funcname = __func__;
test_buf1_buf1(mimic_zlib_decode, &zlib_midsummer_gt);
}
void test_mimic_zlib_decode_pi() {
proc_funcname = __func__;
test_buf1_buf1(mimic_zlib_decode, &zlib_pi_gt);
}
#endif // PUFFS_MIMIC
// ---------------- Flate Benches
@@ -243,11 +273,26 @@ void bench_mimic_flate_decode_100k() {
bench_buf1_buf1(mimic_flate_decode, &flate_pi_gt, 3000);
}
void bench_mimic_gzip_decode_10k() {
proc_funcname = __func__;
bench_buf1_buf1(mimic_gzip_decode, &gzip_midsummer_gt, 30000);
}
void bench_mimic_gzip_decode_100k() {
proc_funcname = __func__;
bench_buf1_buf1(mimic_gzip_decode, &gzip_pi_gt, 3000);
}
void bench_mimic_zlib_decode_10k() {
proc_funcname = __func__;
bench_buf1_buf1(mimic_zlib_decode, &zlib_midsummer_gt, 30000);
}
void bench_mimic_zlib_decode_100k() {
proc_funcname = __func__;
bench_buf1_buf1(mimic_zlib_decode, &zlib_pi_gt, 3000);
}
#endif // PUFFS_MIMIC
// ---------------- Manifest
@@ -270,7 +315,10 @@ proc tests[] = {
test_mimic_flate_decode_midsummer, //
test_mimic_flate_decode_pi, //
test_mimic_flate_decode_romeo, //
test_mimic_gzip_decode_midsummer, //
test_mimic_gzip_decode_pi, //
test_mimic_zlib_decode_midsummer, //
test_mimic_zlib_decode_pi, //
#endif // PUFFS_MIMIC
@@ -292,7 +340,10 @@ proc benches[] = {
bench_mimic_flate_decode_1k, //
bench_mimic_flate_decode_10k, //
bench_mimic_flate_decode_100k, //
bench_mimic_gzip_decode_10k, //
bench_mimic_gzip_decode_100k, //
bench_mimic_zlib_decode_10k, //
bench_mimic_zlib_decode_100k, //
#endif // PUFFS_MIMIC
Binary file not shown.
BIN
View File
Binary file not shown.