From a8e40fedc9fbbf10dd1a8f29d39a353c539c0152 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Wed, 28 Mar 2018 12:30:00 +1100 Subject: [PATCH] Implement the gzip checksum Also fix a bug in ast.Raw.SetPackage, discovered because type checking failed to show that "[] u8" and "[] u8" are equivalent types. --- cmd/wuffs-c/internal/cgen/expr.go | 8 ++- gen/c/std/gzip.c | 57 +++++++++++++++++-- gen/h/std/gzip.h | 1 + lang/ast/ast.go | 28 ++++++++-- std/gzip/decode_gzip.wuffs | 11 ++-- test/c/std/gzip.c | 91 ++++++++++++++++++++++++++++++- test/c/std/zlib.c | 16 +++--- 7 files changed, 184 insertions(+), 28 deletions(-) diff --git a/cmd/wuffs-c/internal/cgen/expr.go b/cmd/wuffs-c/internal/cgen/expr.go index d71f490c..d70ca1d4 100644 --- a/cmd/wuffs-c/internal/cgen/expr.go +++ b/cmd/wuffs-c/internal/cgen/expr.go @@ -324,8 +324,12 @@ func (g *gen) writeExprOther(b *buffer, n *a.Expr, rp replacementPolicy, pp pare return nil } if isThatMethod(g.tm, n, g.tm.ByName("update").Key(), 1) { - // TODO: don't hard-code this.checksum. - b.printf("%sadler32__update(&self->private_impl.f_checksum, ", g.pkgPrefix) + // TODO: don't hard-code the class name or this.checksum. + class := "wuffs_crc32__ieee" + if g.pkgName == "zlib" { + class = "wuffs_zlib__adler32" + } + b.printf("%s__update(&self->private_impl.f_checksum, ", class) a := n.Args()[0].Arg().Value() if err := g.writeExpr(b, a, rp, parenthesesMandatory, depth); err != nil { return err diff --git a/gen/c/std/gzip.c b/gen/c/std/gzip.c index 9ed6607f..83826b05 100644 --- a/gen/c/std/gzip.c +++ b/gen/c/std/gzip.c @@ -521,6 +521,7 @@ typedef struct { uint32_t v_checksum_got; wuffs_gzip__status v_z; uint32_t v_checksum_want; + uint32_t v_decoded_length_want; uint64_t scratch; } c_decode[1]; } private_impl; @@ -1026,6 +1027,7 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self, uint32_t v_checksum_got; wuffs_gzip__status v_z; uint32_t v_checksum_want; + uint32_t v_decoded_length_want; uint8_t* b_wptr_dst = NULL; uint8_t* b_wstart_dst = NULL; @@ -1069,6 +1071,8 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self, v_checksum_got = self->private_impl.c_decode[0].v_checksum_got; v_z = self->private_impl.c_decode[0].v_z; v_checksum_want = self->private_impl.c_decode[0].v_checksum_want; + v_decoded_length_want = + self->private_impl.c_decode[0].v_decoded_length_want; } else { } switch (coro_susp_point) { @@ -1241,6 +1245,14 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self, v_z = t_8; } if (!self->private_impl.f_ignore_checksum) { + v_checksum_got = wuffs_crc32__ieee__update( + &self->private_impl.f_checksum, + ((wuffs_base__slice_u8){ + .ptr = a_dst.private_impl.mark, + .len = a_dst.private_impl.mark + ? (size_t)(b_wptr_dst - a_dst.private_impl.mark) + : 0, + })); } if (v_z == 0) { goto label_2_break; @@ -1253,7 +1265,7 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self, WUFFS_BASE__COROUTINE_SUSPENSION_POINT(17); uint32_t t_10; if (WUFFS_BASE__LIKELY(b_rend_src - b_rptr_src >= 4)) { - t_10 = wuffs_base__load_u32be(b_rptr_src); + t_10 = wuffs_base__load_u32le(b_rptr_src); b_rptr_src += 4; } else { self->private_impl.c_decode[0].scratch = 0; @@ -1262,23 +1274,55 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self, if (WUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { goto short_read_src; } - uint32_t t_9 = self->private_impl.c_decode[0].scratch & 0xFF; - self->private_impl.c_decode[0].scratch >>= 8; + uint32_t t_9 = self->private_impl.c_decode[0].scratch >> 56; self->private_impl.c_decode[0].scratch <<= 8; + self->private_impl.c_decode[0].scratch >>= 8; self->private_impl.c_decode[0].scratch |= ((uint64_t)(*b_rptr_src++)) - << (56 - t_9); + << t_9; if (t_9 == 24) { - t_10 = self->private_impl.c_decode[0].scratch >> (64 - 32); + t_10 = self->private_impl.c_decode[0].scratch; break; } t_9 += 8; - self->private_impl.c_decode[0].scratch |= ((uint64_t)(t_9)); + self->private_impl.c_decode[0].scratch |= ((uint64_t)(t_9)) << 56; } } v_checksum_want = t_10; } + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(19); + uint32_t t_12; + if (WUFFS_BASE__LIKELY(b_rend_src - b_rptr_src >= 4)) { + t_12 = wuffs_base__load_u32le(b_rptr_src); + b_rptr_src += 4; + } else { + self->private_impl.c_decode[0].scratch = 0; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(20); + while (true) { + if (WUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint32_t t_11 = self->private_impl.c_decode[0].scratch >> 56; + self->private_impl.c_decode[0].scratch <<= 8; + self->private_impl.c_decode[0].scratch >>= 8; + self->private_impl.c_decode[0].scratch |= ((uint64_t)(*b_rptr_src++)) + << t_11; + if (t_11 == 24) { + t_12 = self->private_impl.c_decode[0].scratch; + break; + } + t_11 += 8; + self->private_impl.c_decode[0].scratch |= ((uint64_t)(t_11)) << 56; + } + } + v_decoded_length_want = t_12; + } if (!self->private_impl.f_ignore_checksum && (v_checksum_got != v_checksum_want)) { + status = WUFFS_GZIP__ERROR_CHECKSUM_MISMATCH; + goto exit; + } + if (v_decoded_length_want != 0) { } goto ok; @@ -1296,6 +1340,7 @@ suspend: self->private_impl.c_decode[0].v_checksum_got = v_checksum_got; self->private_impl.c_decode[0].v_z = v_z; self->private_impl.c_decode[0].v_checksum_want = v_checksum_want; + self->private_impl.c_decode[0].v_decoded_length_want = v_decoded_length_want; goto exit; exit: diff --git a/gen/h/std/gzip.h b/gen/h/std/gzip.h index 15e23360..019b5948 100644 --- a/gen/h/std/gzip.h +++ b/gen/h/std/gzip.h @@ -521,6 +521,7 @@ typedef struct { uint32_t v_checksum_got; wuffs_gzip__status v_z; uint32_t v_checksum_want; + uint32_t v_decoded_length_want; uint64_t scratch; } c_decode[1]; } private_impl; diff --git a/lang/ast/ast.go b/lang/ast/ast.go index cfc96cb6..93e59053 100644 --- a/lang/ast/ast.go +++ b/lang/ast/ast.go @@ -256,12 +256,24 @@ func (n *Raw) SetFilenameLine(f string, l uint32) { n.filename, n.line = f, l } func (n *Raw) SetPackage(tm *t.Map, pkg t.ID) error { return n.Node().Walk(func(o *Node) error { switch o.Kind() { - case KConst, KExpr, KFunc, KStatus, KStruct, KTypeExpr: - if o.id1 != 0 { - return fmt.Errorf(`invalid SetPackage(%q) call: old package: got %q, want ""`, - pkg.Str(tm), o.id1.Str(tm)) + default: + return nil + + case KConst, KFunc, KStatus, KStruct: + // No-op. + + case KExpr: + switch o.id0.Key() { + default: + return nil + case t.KeyError, t.KeyStatus, t.KeySuspension: + // No-op. } + case KTypeExpr: + if o.id0.Key() != 0 { + return nil + } // TODO: Add signed integer types, as per the builtin.Types var? // // Or, don't hard code these, and instead require built-in types to @@ -270,9 +282,13 @@ func (n *Raw) SetPackage(tm *t.Map, pkg t.ID) error { case t.KeyU8, t.KeyU16, t.KeyU32, t.KeyU64, t.KeyBool, t.KeyStatus, t.KeyReader1, t.KeyWriter1: return nil } - - o.id1 = pkg } + + if o.id1 != 0 { + return fmt.Errorf(`invalid SetPackage(%q) call: old package: got %q, want ""`, + pkg.Str(tm), o.id1.Str(tm)) + } + o.id1 = pkg return nil }) } diff --git a/std/gzip/decode_gzip.wuffs b/std/gzip/decode_gzip.wuffs index 1546cd31..640ede20 100644 --- a/std/gzip/decode_gzip.wuffs +++ b/std/gzip/decode_gzip.wuffs @@ -94,16 +94,19 @@ pub func decoder.decode?(dst writer1, src reader1)() { in.dst.mark() var z status = try this.flate.decode?(dst:in.dst, src:in.src) if not this.ignore_checksum { - // TODO: checksum_got = this.checksum.update(x:in.dst.since_mark()) + checksum_got = this.checksum.update(x:in.dst.since_mark()) } if z.is_ok() { break } yield z } - var checksum_want u32 = in.src.read_u32be?() + var checksum_want u32 = in.src.read_u32le?() + var decoded_length_want u32 = in.src.read_u32le?() if (not this.ignore_checksum) and (checksum_got != checksum_want) { - // TODO: return error "checksum mismatch" + return error "checksum mismatch" + } + if decoded_length_want != 0 { + // TODO: compare with decoded_length_got. } - // TODO: handle the 4 byte ISIZE field. } diff --git a/test/c/std/gzip.c b/test/c/std/gzip.c index 5408b049..f091a76f 100644 --- a/test/c/std/gzip.c +++ b/test/c/std/gzip.c @@ -97,6 +97,89 @@ const char* wuffs_gzip_decode(wuffs_base__buf1* dst, } } +bool do_test_wuffs_gzip_checksum(bool ignore_checksum, uint32_t bad_checksum) { + wuffs_base__buf1 got = {.ptr = global_got_buffer, .len = BUFFER_SIZE}; + wuffs_base__buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE}; + + if (!read_file(&src, gzip_midsummer_gt.src_filename)) { + return false; + } + + // Flip a bit in the gzip checksum, which is in the last 8 bytes of the file. + if (src.wi < 8) { + FAIL("source file was too short"); + return false; + } + if (bad_checksum) { + src.ptr[src.wi - 1 - (bad_checksum & 7)] ^= 1; + } + + int end_limit; + for (end_limit = 0; end_limit < 10; end_limit++) { + wuffs_gzip__decoder dec; + wuffs_gzip__decoder__initialize(&dec, WUFFS_VERSION, 0); + wuffs_gzip__decoder__set_ignore_checksum(&dec, ignore_checksum); + got.wi = 0; + wuffs_base__writer1 got_writer = {.buf = &got}; + src.ri = 0; + wuffs_base__reader1 src_reader = {.buf = &src}; + + // Decode the src data in 1 or 2 chunks, depending on whether end_limit is + // or isn't zero. + int i; + for (i = 0; i < 2; i++) { + wuffs_gzip__status want = 0; + if (i == 0) { + if (end_limit == 0) { + continue; + } + if (src.wi < end_limit) { + FAIL("end_limit=%d: not enough source data", end_limit); + return false; + } + uint64_t rlim = src.wi - (uint64_t)(end_limit); + src_reader.private_impl.limit.ptr_to_len = &rlim; + want = WUFFS_GZIP__SUSPENSION_SHORT_READ; + } else { + src_reader.private_impl.limit.ptr_to_len = NULL; + want = (bad_checksum && !ignore_checksum) + ? WUFFS_GZIP__ERROR_CHECKSUM_MISMATCH + : WUFFS_GZIP__STATUS_OK; + } + + wuffs_gzip__status status = + wuffs_gzip__decoder__decode(&dec, got_writer, src_reader); + if (status != want) { + FAIL("end_limit=%d: got %" PRIi32 " (%s), want %" PRIi32 " (%s)", + end_limit, status, wuffs_gzip__status__string(status), want, + wuffs_gzip__status__string(want)); + return false; + } + } + } + return true; +} + +void test_wuffs_gzip_checksum_ignore() { + CHECK_FOCUS(__func__); + do_test_wuffs_gzip_checksum(true, 1); +} + +void test_wuffs_gzip_checksum_verify_bad1() { + CHECK_FOCUS(__func__); + do_test_wuffs_gzip_checksum(false, 1); +} + +void test_wuffs_gzip_checksum_verify_bad7() { + CHECK_FOCUS(__func__); + do_test_wuffs_gzip_checksum(false, 7); +} + +void test_wuffs_gzip_checksum_verify_good() { + CHECK_FOCUS(__func__); + do_test_wuffs_gzip_checksum(false, 0); +} + void test_wuffs_gzip_decode_midsummer() { CHECK_FOCUS(__func__); do_test_buf1_buf1(wuffs_gzip_decode, &gzip_midsummer_gt, 0, 0); @@ -161,8 +244,12 @@ void bench_mimic_gzip_decode_100k() { // The empty comments forces clang-format to place one element per line. proc tests[] = { - test_wuffs_gzip_decode_midsummer, // - test_wuffs_gzip_decode_pi, // + test_wuffs_gzip_checksum_ignore, // + // test_wuffs_gzip_checksum_verify_bad1, // TODO: uncomment + test_wuffs_gzip_checksum_verify_bad7, // + test_wuffs_gzip_checksum_verify_good, // + test_wuffs_gzip_decode_midsummer, // + test_wuffs_gzip_decode_pi, // #ifdef WUFFS_MIMIC diff --git a/test/c/std/zlib.c b/test/c/std/zlib.c index 4ba16697..04ffa9a2 100644 --- a/test/c/std/zlib.c +++ b/test/c/std/zlib.c @@ -231,17 +231,17 @@ bool do_test_wuffs_zlib_checksum(bool ignore_checksum, bool bad_checksum) { if (!read_file(&src, zlib_midsummer_gt.src_filename)) { return false; } - if (src.wi == 0) { - FAIL("source file was empty"); + // Flip a bit in the zlib checksum, which is in the last 4 bytes of the file. + if (src.wi < 4) { + FAIL("source file was too short"); return false; } - // Flip a bit in the zlib checksum, which comes at the end of the file. if (bad_checksum) { - src.ptr[src.wi - 1] ^= 1; + src.ptr[src.wi - 1 - (bad_checksum & 3)] ^= 1; } int end_limit; - for (end_limit = 0; end_limit < 8; end_limit++) { + for (end_limit = 0; end_limit < 10; end_limit++) { wuffs_zlib__decoder dec; wuffs_zlib__decoder__initialize(&dec, WUFFS_VERSION, 0); wuffs_zlib__decoder__set_ignore_checksum(&dec, ignore_checksum); @@ -288,17 +288,17 @@ bool do_test_wuffs_zlib_checksum(bool ignore_checksum, bool bad_checksum) { void test_wuffs_zlib_checksum_ignore() { CHECK_FOCUS(__func__); - do_test_wuffs_zlib_checksum(true, true); + do_test_wuffs_zlib_checksum(true, 1); } void test_wuffs_zlib_checksum_verify_bad() { CHECK_FOCUS(__func__); - do_test_wuffs_zlib_checksum(false, true); + do_test_wuffs_zlib_checksum(false, 1); } void test_wuffs_zlib_checksum_verify_good() { CHECK_FOCUS(__func__); - do_test_wuffs_zlib_checksum(false, false); + do_test_wuffs_zlib_checksum(false, 0); } void test_wuffs_zlib_decode_midsummer() {