From 1c75edb32eab6878b60cd2f5fea370e44c2a6539 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 21 Oct 2017 12:36:08 +1100 Subject: [PATCH] Calculate the zlib checksum. --- cmd/puffs-c/internal/cgen/expr.go | 10 ++++ gen/c/std/flate.c | 99 +++++++++++++++++++++++++------ gen/h/std/flate.h | 23 +++---- lang/check/bounds.go | 18 +----- lang/check/type.go | 16 +++++ std/flate/decode_flate.puffs | 1 + std/flate/decode_zlib.puffs | 8 ++- test/c/std/flate.c | 45 ++++++++++++++ 8 files changed, 174 insertions(+), 46 deletions(-) diff --git a/cmd/puffs-c/internal/cgen/expr.go b/cmd/puffs-c/internal/cgen/expr.go index fb2d32bc..10dad959 100644 --- a/cmd/puffs-c/internal/cgen/expr.go +++ b/cmd/puffs-c/internal/cgen/expr.go @@ -265,6 +265,16 @@ func (g *gen) writeExprOther(b *buffer, n *a.Expr, rp replacementPolicy, pp pare } return nil } + if isThatMethod(g.tm, n, g.tm.ByName("update").Key(), 1) { + // TODO: don't hard-code this.adler. + b.printf("%sadler32_update(&self->private_impl.f_adler, ", g.pkgPrefix) + a := n.Args()[0].Arg().Value() + if err := g.writeExpr(b, a, rp, parenthesesMandatory, depth); err != nil { + return err + } + b.writes(")\n") + return nil + } if isThatMethod(g.tm, n, g.tm.ByName("set_literal_width").Key(), 1) { // TODO: don't hard-code lzw. b.printf("%slzw_decoder_set_literal_width(&self->private_impl.f_lzw, ", g.pkgPrefix) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 3babd88f..f1598067 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -134,22 +134,23 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040121 // 0xbb08f807 #define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040120 // 0xbb08f808 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040119 // 0xbb08f809 + -1157040119 // 0xbb08f809 +#define PUFFS_FLATE_ERROR_CHECKSUM_MISMATCH -1157040118 // 0xbb08f80a #define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ - -1157040118 // 0xbb08f80a -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ -1157040117 // 0xbb08f80b +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040116 // 0xbb08f80c #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040116 // 0xbb08f80c -#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040115 // 0xbb08f80d -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040114 // 0xbb08f80e + -1157040115 // 0xbb08f80d +#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040114 // 0xbb08f80e +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040113 // 0xbb08f80f #define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_METHOD \ - -1157040113 // 0xbb08f80f + -1157040112 // 0xbb08f810 #define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_WINDOW_SIZE \ - -1157040112 // 0xbb08f810 -#define PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK -1157040111 // 0xbb08f811 + -1157040111 // 0xbb08f811 +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK -1157040110 // 0xbb08f812 #define PUFFS_FLATE_ERROR_TODO_UNSUPPORTED_ZLIB_PRESET_DICTIONARY \ - -1157040110 // 0xbb08f812 + -1157040109 // 0xbb08f813 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -269,8 +270,8 @@ typedef struct { struct { uint32_t coro_susp_point; uint16_t v_x; - puffs_flate_status v_z; uint32_t v_checksum; + puffs_flate_status v_z; uint64_t scratch; } c_decode[1]; } private_impl; @@ -582,7 +583,7 @@ const char* puffs_flate_status_strings0[9] = { "flate: short write", }; -const char* puffs_flate_status_strings1[19] = { +const char* puffs_flate_status_strings1[20] = { "flate: bad Huffman code (over-subscribed)", "flate: bad Huffman code (under-subscribed)", "flate: bad Huffman code length count", @@ -593,6 +594,7 @@ const char* puffs_flate_status_strings1[19] = { "flate: bad distance code count", "flate: bad flate block", "flate: bad literal/length code count", + "flate: checksum mismatch", "flate: inconsistent stored block length", "flate: internal error: inconsistent Huffman decoder state", "flate: internal error: inconsistent n_bits", @@ -614,7 +616,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) { break; case puffs_flate_packageid: a = puffs_flate_status_strings1; - n = 19; + n = 20; break; } uint32_t i = s & 0xff; @@ -2261,9 +2263,27 @@ puffs_flate_status puffs_flate_zlib_decoder_decode( puffs_flate_status status = PUFFS_FLATE_STATUS_OK; uint16_t v_x; - puffs_flate_status v_z; uint32_t v_checksum; + puffs_flate_status v_z; + uint8_t* b_wptr_dst = NULL; + uint8_t* b_wstart_dst = NULL; + uint8_t* b_wend_dst = NULL; + if (a_dst.buf) { + b_wptr_dst = a_dst.buf->ptr + a_dst.buf->wi; + b_wstart_dst = b_wptr_dst; + b_wend_dst = b_wptr_dst; + if (!a_dst.buf->closed) { + size_t len = a_dst.buf->len - a_dst.buf->wi; + puffs_base_limit1* lim; + for (lim = &a_dst.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_wend_dst += len; + } + } uint8_t* b_rptr_src = NULL; uint8_t* b_rstart_src = NULL; uint8_t* b_rend_src = NULL; @@ -2283,8 +2303,8 @@ puffs_flate_status puffs_flate_zlib_decoder_decode( uint32_t coro_susp_point = self->private_impl.c_decode[0].coro_susp_point; if (coro_susp_point) { v_x = self->private_impl.c_decode[0].v_x; - v_z = self->private_impl.c_decode[0].v_z; v_checksum = self->private_impl.c_decode[0].v_checksum; + v_z = self->private_impl.c_decode[0].v_z; } switch (coro_susp_point) { PUFFS_COROUTINE_SUSPENSION_POINT(0); @@ -2331,8 +2351,19 @@ puffs_flate_status puffs_flate_zlib_decoder_decode( status = PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK; goto exit; } + v_checksum = 0; while (true) { PUFFS_COROUTINE_SUSPENSION_POINT(3); + if (a_dst.buf) { + size_t n = b_wptr_dst - (a_dst.buf->ptr + a_dst.buf->wi); + a_dst.buf->wi += n; + puffs_base_limit1* lim; + for (lim = &a_dst.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + } if (a_src.buf) { size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); a_src.buf->ri += n; @@ -2345,6 +2376,20 @@ puffs_flate_status puffs_flate_zlib_decoder_decode( } puffs_flate_status t_2 = puffs_flate_decoder_decode(&self->private_impl.f_dec, a_dst, a_src); + if (a_dst.buf) { + b_wptr_dst = a_dst.buf->ptr + a_dst.buf->wi; + b_wend_dst = b_wptr_dst; + if (!a_dst.buf->closed) { + size_t len = a_dst.buf->len - a_dst.buf->wi; + puffs_base_limit1* lim; + for (lim = &a_dst.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_wend_dst += len; + } + } if (a_src.buf) { b_rptr_src = a_src.buf->ptr + a_src.buf->ri; size_t len = a_src.buf->wi - a_src.buf->ri; @@ -2357,6 +2402,11 @@ puffs_flate_status puffs_flate_zlib_decoder_decode( b_rend_src = b_rptr_src + len; } v_z = t_2; + v_checksum = puffs_flate_adler32_update( + &self->private_impl.f_adler, ((puffs_base_slice_u8){ + .ptr = b_wstart_dst, + .len = b_wptr_dst - b_wstart_dst, + })); if (v_z == 0) { goto label_0_break; } @@ -2389,7 +2439,10 @@ puffs_flate_status puffs_flate_zlib_decoder_decode( self->private_impl.c_decode[0].scratch |= ((uint64_t)(t_3)); } } - v_checksum = t_4; + if (v_checksum != t_4) { + status = PUFFS_FLATE_ERROR_CHECKSUM_MISMATCH; + goto exit; + } goto ok; ok: @@ -2401,10 +2454,22 @@ puffs_flate_status puffs_flate_zlib_decoder_decode( suspend: self->private_impl.c_decode[0].coro_susp_point = coro_susp_point; self->private_impl.c_decode[0].v_x = v_x; - self->private_impl.c_decode[0].v_z = v_z; self->private_impl.c_decode[0].v_checksum = v_checksum; + self->private_impl.c_decode[0].v_z = v_z; exit: + if (a_dst.buf) { + size_t n = b_wptr_dst - (a_dst.buf->ptr + a_dst.buf->wi); + a_dst.buf->wi += n; + puffs_base_limit1* lim; + for (lim = &a_dst.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + PUFFS_IGNORE_POTENTIALLY_UNUSED_VARIABLE(b_wstart_dst); + PUFFS_IGNORE_POTENTIALLY_UNUSED_VARIABLE(b_wend_dst); + } if (a_src.buf) { size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); a_src.buf->ri += n; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index ece55099..2173a134 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -134,22 +134,23 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040121 // 0xbb08f807 #define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040120 // 0xbb08f808 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040119 // 0xbb08f809 + -1157040119 // 0xbb08f809 +#define PUFFS_FLATE_ERROR_CHECKSUM_MISMATCH -1157040118 // 0xbb08f80a #define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ - -1157040118 // 0xbb08f80a -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ -1157040117 // 0xbb08f80b +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040116 // 0xbb08f80c #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040116 // 0xbb08f80c -#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040115 // 0xbb08f80d -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040114 // 0xbb08f80e + -1157040115 // 0xbb08f80d +#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040114 // 0xbb08f80e +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040113 // 0xbb08f80f #define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_METHOD \ - -1157040113 // 0xbb08f80f + -1157040112 // 0xbb08f810 #define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_WINDOW_SIZE \ - -1157040112 // 0xbb08f810 -#define PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK -1157040111 // 0xbb08f811 + -1157040111 // 0xbb08f811 +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK -1157040110 // 0xbb08f812 #define PUFFS_FLATE_ERROR_TODO_UNSUPPORTED_ZLIB_PRESET_DICTIONARY \ - -1157040110 // 0xbb08f812 + -1157040109 // 0xbb08f813 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -269,8 +270,8 @@ typedef struct { struct { uint32_t coro_susp_point; uint16_t v_x; - puffs_flate_status v_z; uint32_t v_checksum; + puffs_flate_status v_z; uint64_t scratch; } c_decode[1]; } private_impl; diff --git a/lang/check/bounds.go b/lang/check/bounds.go index bb2be5f3..18ff1f32 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -762,6 +762,7 @@ func (q *checker) bcheckExprOther(n *a.Expr, depth uint32) (*big.Int, *big.Int, // TODO: delete this hack that only matches "foo.is_suspension(etc)". if isThatMethod(q.tm, n, t.KeyIsError, 0) || isThatMethod(q.tm, n, t.KeyIsOK, 0) || isThatMethod(q.tm, n, t.KeyIsSuspension, 0) || isThatMethod(q.tm, n, t.KeySuffix, 1) { + // TODO: should return be break? Ditto below. return nil, nil, nil } // TODO: delete this hack that only matches "foo.set_literal_width(etc)". @@ -777,21 +778,8 @@ func (q *checker) bcheckExprOther(n *a.Expr, depth uint32) (*big.Int, *big.Int, return nil, nil, nil } // TODO: delete this hack that only matches "foo.decode(etc)". - if isThatMethod(q.tm, n, q.tm.ByName("decode").Key(), 2) { - for _, o := range n.Args() { - a := o.Arg().Value() - aMin, aMax, err := q.bcheckExpr(a, depth) - if err != nil { - return nil, nil, err - } - // TODO: check that aMin and aMax is within the function's - // declared arg bounds. - _, _ = aMin, aMax - } - return nil, nil, nil - } - // TODO: delete this hack that only matches "foo.copy_from_slice(etc)". - if isThatMethod(q.tm, n, t.KeyCopyFromSlice, 1) { + if isThatMethod(q.tm, n, q.tm.ByName("decode").Key(), 2) || isThatMethod(q.tm, n, t.KeyCopyFromSlice, 1) || + isThatMethod(q.tm, n, q.tm.ByName("update").Key(), 1) { for _, o := range n.Args() { a := o.Arg().Value() aMin, aMax, err := q.bcheckExpr(a, depth) diff --git a/lang/check/type.go b/lang/check/type.go index a9ee3f67..38ee5990 100644 --- a/lang/check/type.go +++ b/lang/check/type.go @@ -553,6 +553,22 @@ func (q *checker) tcheckExprOther(n *a.Expr, depth uint32) error { n.SetMType(typeExprU64) return nil } + // TODO: delete this hack that only matches "foo.update(etc)". + if isThatMethod(q.tm, n, q.tm.ByName("update").Key(), 1) { + foo := n.LHS().Expr().LHS().Expr() + if err := q.tcheckExpr(foo, depth); err != nil { + return err + } + n.LHS().SetTypeChecked() + n.LHS().Expr().SetMType(typeExprPlaceholder) // HACK. + for _, o := range n.Args() { + if err := q.tcheckArg(o.Arg(), depth); err != nil { + return err + } + } + n.SetMType(typeExprU32) + return nil + } case t.KeyOpenBracket: // n is an index. diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 1ea8a8e8..61799f3a 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -24,6 +24,7 @@ pub error "bad distance" pub error "bad distance code count" pub error "bad flate block" pub error "bad literal/length code count" +pub error "checksum mismatch" pub error "inconsistent stored block length" pub error "internal error: inconsistent Huffman decoder state" pub error "internal error: inconsistent n_bits" diff --git a/std/flate/decode_zlib.puffs b/std/flate/decode_zlib.puffs index 0354f8c1..d8efeaf5 100644 --- a/std/flate/decode_zlib.puffs +++ b/std/flate/decode_zlib.puffs @@ -37,16 +37,18 @@ pub func zlib_decoder.decode?(dst writer1, src reader1)() { if (x % 31) != 0 { return error "invalid zlib parity check" } + var checksum u32 while true { var z status = try this.dec.decode?(dst:in.dst, src:in.src) - // TODO: calculate the checksum. + checksum = this.adler.update(x:in.dst.slice()) if z.is_ok() { break } return z } - var checksum u32 = in.src.read_u32be?() - // TODO: verify the checksum. + if checksum != in.src.read_u32be?() { + return error "checksum mismatch" + } } // TODO: drop the '?' but still generate puffs_flate_adler_initialize? diff --git a/test/c/std/flate.c b/test/c/std/flate.c index 09d9d3a7..2306d989 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -622,6 +622,37 @@ void test_puffs_flate_table_redirect() { } } +void test_puffs_zlib_checksum_mismatch() { + proc_funcname = __func__; + + puffs_base_buf1 got = {.ptr = global_got_buffer, .len = BUFFER_SIZE}; + puffs_base_buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE}; + + if (!read_file(&src, zlib_midsummer_gt.src_filename)) { + return; + } + if (src.wi == 0) { + FAIL("source file was empty"); + return; + } + // Flip a bit in the zlib checksum, which comes at the end of the file. + src.ptr[src.wi - 1] ^= 1; + + puffs_flate_zlib_decoder dec; + puffs_flate_zlib_decoder_initialize(&dec, PUFFS_VERSION, 0); + puffs_base_writer1 got_writer = {.buf = &got}; + puffs_base_reader1 src_reader = {.buf = &src}; + + puffs_flate_status status = + puffs_flate_zlib_decoder_decode(&dec, got_writer, src_reader); + if (status != PUFFS_FLATE_ERROR_CHECKSUM_MISMATCH) { + FAIL("status: got %" PRIi32 " (%s), want %" PRIi32 " (%s)", status, + puffs_flate_status_string(status), PUFFS_FLATE_ERROR_CHECKSUM_MISMATCH, + puffs_flate_status_string(PUFFS_FLATE_ERROR_CHECKSUM_MISMATCH)); + return; + } +} + void test_puffs_zlib_decode_midsummer() { proc_funcname = __func__; do_test_buf1_buf1(puffs_zlib_decode, &zlib_midsummer_gt, 0, 0); @@ -734,6 +765,17 @@ void bench_puffs_flate_decode_100k() { do_bench_buf1_buf1(puffs_flate_decode, tc_dst, &flate_pi_gt, 0, 0, 3000); } +void bench_puffs_zlib_decode_10k() { + proc_funcname = __func__; + do_bench_buf1_buf1(puffs_zlib_decode, tc_dst, &zlib_midsummer_gt, 0, 0, + 30000); +} + +void bench_puffs_zlib_decode_100k() { + proc_funcname = __func__; + do_bench_buf1_buf1(puffs_zlib_decode, tc_dst, &zlib_pi_gt, 0, 0, 3000); +} + // ---------------- Mimic Benches #ifdef PUFFS_MIMIC @@ -818,6 +860,7 @@ proc tests[] = { test_puffs_flate_history_full, // test_puffs_flate_history_partial, // test_puffs_flate_table_redirect, // + test_puffs_zlib_checksum_mismatch, // test_puffs_zlib_decode_midsummer, // test_puffs_zlib_decode_pi, // @@ -847,6 +890,8 @@ proc benches[] = { bench_puffs_flate_decode_1k, // bench_puffs_flate_decode_10k, // bench_puffs_flate_decode_100k, // + bench_puffs_zlib_decode_10k, // + bench_puffs_zlib_decode_100k, // #ifdef PUFFS_MIMIC