diff --git a/script/extract-flate-offsets.go b/script/extract-flate-offsets.go index f11d2ac2..da070b2c 100644 --- a/script/extract-flate-offsets.go +++ b/script/extract-flate-offsets.go @@ -9,14 +9,24 @@ package main // flate-compressed data wrapped in a .gz file. // // Usage: go run extract-flate-offsets.go foo.gz bar.gz +// +// Alternatively: go run extract-flate-offsets.go -write-zlib foo.gz import ( "bytes" "compress/flate" + "compress/zlib" "crypto/md5" + "flag" "fmt" + "hash/adler32" "io/ioutil" "os" + "strings" +) + +var ( + writeZlib = flag.Bool("write-zlib", false, "whether to convert gzip to zlib") ) // GZIP wraps a header and footer around flate data. The format is described in @@ -37,7 +47,8 @@ func main() { } func main1() error { - for _, a := range os.Args[1:] { + flag.Parse() + for _, a := range flag.Args() { if err := decode(a); err != nil { return err } @@ -86,12 +97,55 @@ func decode(filename string) error { } // As a sanity check, the result should be valid flate. - hash, err := checkFlate(src[i:]) + uncompressed, err := checkFlate(src[i:]) if err != nil { return err } - fmt.Printf("%7d %7d %x %s\n", i, len(src), hash, filename) + if *writeZlib { + return doWriteZlib(src[i:], uncompressed, filename) + } + fmt.Printf("%7d %7d %x %s\n", i, len(src), md5.Sum(uncompressed), filename) + return nil +} + +func doWriteZlib(flateCompressed []byte, uncompressed []byte, filename string) error { + buf := bytes.NewBuffer(nil) + // The ZLIB header (as per https://www.ietf.org/rfc/rfc1950.txt) is 2 + // bytes. + // + // The first byte's low 4 bits is the compression method: 8 means flate. + // The first byte's high 4 bits is the compression info: 7 means a 32KiB + // flate window size. + // + // The second byte's low 5 bits are a parity check. The 5th bit (0 in this + // case) indicates a preset dictionary. The high 2 bits (2 in this case) + // means the default compression algorithm. + buf.WriteString("\x78\x9c") + // Write the payload. + buf.Write(flateCompressed) + // The ZLIB footer is 4 bytes: a big-endian checksum. + checksum := adler32.Checksum(uncompressed) + buf.WriteByte(uint8(checksum >> 24)) + buf.WriteByte(uint8(checksum >> 16)) + buf.WriteByte(uint8(checksum >> 8)) + buf.WriteByte(uint8(checksum >> 0)) + + asZlib := buf.Bytes() + + // As a sanity check, the result should be valid zlib. + if _, err := checkZlib(asZlib); err != nil { + return err + } + + if strings.HasSuffix(filename, ".gz") { + filename = filename[:len(filename)-3] + } + filename += ".zlib" + if err := ioutil.WriteFile(filename, asZlib, 0666); err != nil { + return err + } + fmt.Printf("wrote %s\n", filename) return nil } @@ -107,12 +161,25 @@ func readString(src []byte, i int) (int, error) { } } -func checkFlate(x []byte) ([md5.Size]byte, error) { +func checkFlate(x []byte) ([]byte, error) { rc := flate.NewReader(bytes.NewReader(x)) defer rc.Close() x, err := ioutil.ReadAll(rc) if err != nil { - return [md5.Size]byte{}, fmt.Errorf("data is not valid flate: %v", err) + return nil, fmt.Errorf("data is not valid flate: %v", err) } - return md5.Sum(x), nil + return x, nil +} + +func checkZlib(x []byte) ([]byte, error) { + rc, err := zlib.NewReader(bytes.NewReader(x)) + if err != nil { + return nil, fmt.Errorf("data is not valid zlib: %v", err) + } + defer rc.Close() + x, err = ioutil.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("data is not valid zlib: %v", err) + } + return x, nil } diff --git a/test/c/mimiclib/flate.c b/test/c/mimiclib/flate.c index 92fb12e2..f824e3f6 100644 --- a/test/c/mimiclib/flate.c +++ b/test/c/mimiclib/flate.c @@ -57,15 +57,19 @@ cleanup0:; return ret; } -const char* mimic_gzip_decode(puffs_base_buf1* dst, puffs_base_buf1* src) { +const char* mimic_gzip_zlib_decode(puffs_base_buf1* dst, + puffs_base_buf1* src, + bool gzip_instead_of_zlib) { const char* ret = NULL; z_stream z = {0}; - const int window_bits = 15; - // See inflateInit2 in the zlib manual, or in zlib.h, for details about this - // magic gzip_instead_of_zlib constant. - const int gzip_instead_of_zlib = 16; - int ii2_err = inflateInit2(&z, window_bits | gzip_instead_of_zlib); + int window_bits = 15; + if (gzip_instead_of_zlib) { + // See inflateInit2 in the zlib manual, or in zlib.h, for details about + // this magic constant. + window_bits |= 16; + } + int ii2_err = inflateInit2(&z, window_bits); if (ii2_err != Z_OK) { ret = "inflateInit2 failed"; goto cleanup0; @@ -107,3 +111,11 @@ cleanup1:; cleanup0:; return ret; } + +const char* mimic_gzip_decode(puffs_base_buf1* dst, puffs_base_buf1* src) { + return mimic_gzip_zlib_decode(dst, src, true); +} + +const char* mimic_zlib_decode(puffs_base_buf1* dst, puffs_base_buf1* src) { + return mimic_gzip_zlib_decode(dst, src, false); +} diff --git a/test/c/std/flate.c b/test/c/std/flate.c index 4c46a6d8..7f33ad81 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -67,11 +67,26 @@ golden_test flate_romeo_gt = { .src_offset1 = 550, // }; +golden_test gzip_midsummer_gt = { + .want_filename = "../../testdata/midsummer.txt", // + .src_filename = "../../testdata/midsummer.txt.gz", // +}; + golden_test gzip_pi_gt = { .want_filename = "../../testdata/pi.txt", // .src_filename = "../../testdata/pi.txt.gz", // }; +golden_test zlib_midsummer_gt = { + .want_filename = "../../testdata/midsummer.txt", // + .src_filename = "../../testdata/midsummer.txt.zlib", // +}; + +golden_test zlib_pi_gt = { + .want_filename = "../../testdata/pi.txt", // + .src_filename = "../../testdata/pi.txt.zlib", // +}; + // ---------------- Flate Tests const char* puffs_flate_decode(puffs_base_buf1* dst, puffs_base_buf1* src) { @@ -200,11 +215,26 @@ void test_mimic_flate_decode_romeo() { test_buf1_buf1(mimic_flate_decode, &flate_romeo_gt); } +void test_mimic_gzip_decode_midsummer() { + proc_funcname = __func__; + test_buf1_buf1(mimic_gzip_decode, &gzip_midsummer_gt); +} + void test_mimic_gzip_decode_pi() { proc_funcname = __func__; test_buf1_buf1(mimic_gzip_decode, &gzip_pi_gt); } +void test_mimic_zlib_decode_midsummer() { + proc_funcname = __func__; + test_buf1_buf1(mimic_zlib_decode, &zlib_midsummer_gt); +} + +void test_mimic_zlib_decode_pi() { + proc_funcname = __func__; + test_buf1_buf1(mimic_zlib_decode, &zlib_pi_gt); +} + #endif // PUFFS_MIMIC // ---------------- Flate Benches @@ -243,11 +273,26 @@ void bench_mimic_flate_decode_100k() { bench_buf1_buf1(mimic_flate_decode, &flate_pi_gt, 3000); } +void bench_mimic_gzip_decode_10k() { + proc_funcname = __func__; + bench_buf1_buf1(mimic_gzip_decode, &gzip_midsummer_gt, 30000); +} + void bench_mimic_gzip_decode_100k() { proc_funcname = __func__; bench_buf1_buf1(mimic_gzip_decode, &gzip_pi_gt, 3000); } +void bench_mimic_zlib_decode_10k() { + proc_funcname = __func__; + bench_buf1_buf1(mimic_zlib_decode, &zlib_midsummer_gt, 30000); +} + +void bench_mimic_zlib_decode_100k() { + proc_funcname = __func__; + bench_buf1_buf1(mimic_zlib_decode, &zlib_pi_gt, 3000); +} + #endif // PUFFS_MIMIC // ---------------- Manifest @@ -270,7 +315,10 @@ proc tests[] = { test_mimic_flate_decode_midsummer, // test_mimic_flate_decode_pi, // test_mimic_flate_decode_romeo, // + test_mimic_gzip_decode_midsummer, // test_mimic_gzip_decode_pi, // + test_mimic_zlib_decode_midsummer, // + test_mimic_zlib_decode_pi, // #endif // PUFFS_MIMIC @@ -292,7 +340,10 @@ proc benches[] = { bench_mimic_flate_decode_1k, // bench_mimic_flate_decode_10k, // bench_mimic_flate_decode_100k, // + bench_mimic_gzip_decode_10k, // bench_mimic_gzip_decode_100k, // + bench_mimic_zlib_decode_10k, // + bench_mimic_zlib_decode_100k, // #endif // PUFFS_MIMIC diff --git a/test/testdata/midsummer.txt.zlib b/test/testdata/midsummer.txt.zlib new file mode 100644 index 00000000..1077b60e Binary files /dev/null and b/test/testdata/midsummer.txt.zlib differ diff --git a/test/testdata/pi.txt.zlib b/test/testdata/pi.txt.zlib new file mode 100644 index 00000000..3bd9df6c Binary files /dev/null and b/test/testdata/pi.txt.zlib differ