From f7fb991619346ad8c691e820344f03526c1cb8c5 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 21 Jul 2017 15:55:49 +1000 Subject: [PATCH] Implement init_huffman. --- gen/c/std/flate.c | 78 +++++++++++++++++++++----- gen/h/std/flate.h | 17 ++++-- lang/check/assert.go | 46 ++++++++++----- std/flate/decode_flate.puffs | 105 +++++++++++++++++++++++++++++++---- test/c/std/flate.c | 59 ++++++++++++++++---- 5 files changed, 250 insertions(+), 55 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 7ac9f93b..87aa2d42 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -105,14 +105,15 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800 #define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040126 // 0xbb08f802 -#define PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE -1157040125 // 0xbb08f803 + -1157040126 // 0xbb08f802 #define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ + -1157040125 // 0xbb08f803 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFFMAN_STATE \ -1157040124 // 0xbb08f804 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040122 // 0xbb08f806 -#define PUFFS_FLATE_ERROR_TODO_DYNAMIC_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 + -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -132,6 +133,7 @@ typedef struct { // It is a struct, not a struct*, so that it can be stack allocated. struct { uint16_t f_counts[16]; + uint16_t f_symbols[290]; } private_impl; } puffs_flate_huffman_decoder; @@ -150,7 +152,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; - puffs_flate_huffman_decoder f_huff[2]; + puffs_flate_huffman_decoder f_huffs[2]; uint8_t f_code_lengths[289]; uint32_t f_wip0; uint32_t f_wip1; @@ -176,6 +178,9 @@ typedef struct { struct { uint32_t coro_susp_point; uint32_t v_i; + uint16_t v_offsets[16]; + uint16_t v_total; + uint16_t v_count; } c_init_huffman[1]; } private_impl; } puffs_flate_decoder; @@ -263,11 +268,11 @@ const char* puffs_flate_status_strings1[8] = { "flate: bad distance code count", "flate: bad flate block", "flate: bad literal/length code count", - "flate: inconsistent Huffman table", "flate: inconsistent stored block length", + "flate: internal error: inconsistent init_huffman state", "flate: internal error: inconsistent n_bits", + "flate: no Huffman codes", "flate: TODO: fixed Huffman blocks", - "flate: TODO: dynamic Huffman blocks", }; const char* puffs_flate_status_string(puffs_flate_status s) { @@ -792,38 +797,80 @@ puffs_flate_status puffs_flate_decoder_init_huffman(puffs_flate_decoder* self, puffs_flate_status status = PUFFS_FLATE_STATUS_OK; uint32_t v_i; + uint16_t v_offsets[16]; + uint16_t v_total; + uint16_t v_count; uint32_t coro_susp_point = self->private_impl.c_init_huffman[0].coro_susp_point; if (coro_susp_point) { v_i = self->private_impl.c_init_huffman[0].v_i; + memcpy(v_offsets, self->private_impl.c_init_huffman[0].v_offsets, 2 * 16); + v_total = self->private_impl.c_init_huffman[0].v_total; + v_count = self->private_impl.c_init_huffman[0].v_count; } switch (coro_susp_point) { PUFFS_COROUTINE_SUSPENSION_POINT(0); v_i = 0; while (v_i < 16) { - self->private_impl.f_huff[a_which].private_impl.f_counts[v_i] = 0; + self->private_impl.f_huffs[a_which].private_impl.f_counts[v_i] = 0; v_i += 1; } v_i = 0; while (v_i < a_n_codes) { - if (self->private_impl.f_huff[a_which] + if (self->private_impl.f_huffs[a_which] .private_impl.f_counts[self->private_impl.f_code_lengths[v_i]] >= 289) { - status = PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE; + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFFMAN_STATE; goto exit; } - self->private_impl.f_huff[a_which] + self->private_impl.f_huffs[a_which] .private_impl.f_counts[self->private_impl.f_code_lengths[v_i]] += 1; v_i += 1; } if (((uint32_t)( - self->private_impl.f_huff[a_which].private_impl.f_counts[0])) == + self->private_impl.f_huffs[a_which].private_impl.f_counts[0])) == a_n_codes) { - status = PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE; + status = PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES; goto exit; } + { + size_t i; + for (i = 0; i < 16; i++) { + v_offsets[i] = 0; + } + }; + v_total = 0; + v_i = 1; + while (v_i <= 15) { + v_offsets[v_i] = v_total; + v_count = self->private_impl.f_huffs[a_which].private_impl.f_counts[v_i]; + if (v_total > (289 - v_count)) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFFMAN_STATE; + goto exit; + } + v_total = (v_total + v_count); + v_i += 1; + } + v_i = 0; + while (v_i < a_n_codes) { + if (self->private_impl.f_code_lengths[v_i] != 0) { + self->private_impl.f_huffs[a_which] + .private_impl + .f_symbols[v_offsets[self->private_impl.f_code_lengths[v_i]]] = + ((uint16_t)(v_i)); + if (v_offsets[self->private_impl.f_code_lengths[v_i]] >= 289) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFFMAN_STATE; + goto exit; + } + v_offsets[self->private_impl.f_code_lengths[v_i]] += 1; + } + v_i += 1; + } self->private_impl.c_init_huffman[0].coro_susp_point = 0; goto exit; } @@ -832,6 +879,9 @@ puffs_flate_status puffs_flate_decoder_init_huffman(puffs_flate_decoder* self, suspend: self->private_impl.c_init_huffman[0].coro_susp_point = coro_susp_point; self->private_impl.c_init_huffman[0].v_i = v_i; + memcpy(self->private_impl.c_init_huffman[0].v_offsets, v_offsets, 2 * 16); + self->private_impl.c_init_huffman[0].v_total = v_total; + self->private_impl.c_init_huffman[0].v_count = v_count; exit: return status; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index b56c3bf5..0774aa7a 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -105,14 +105,15 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800 #define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040126 // 0xbb08f802 -#define PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE -1157040125 // 0xbb08f803 + -1157040126 // 0xbb08f802 #define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ + -1157040125 // 0xbb08f803 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFFMAN_STATE \ -1157040124 // 0xbb08f804 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040122 // 0xbb08f806 -#define PUFFS_FLATE_ERROR_TODO_DYNAMIC_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 + -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -132,6 +133,7 @@ typedef struct { // It is a struct, not a struct*, so that it can be stack allocated. struct { uint16_t f_counts[16]; + uint16_t f_symbols[290]; } private_impl; } puffs_flate_huffman_decoder; @@ -150,7 +152,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; - puffs_flate_huffman_decoder f_huff[2]; + puffs_flate_huffman_decoder f_huffs[2]; uint8_t f_code_lengths[289]; uint32_t f_wip0; uint32_t f_wip1; @@ -176,6 +178,9 @@ typedef struct { struct { uint32_t coro_susp_point; uint32_t v_i; + uint16_t v_offsets[16]; + uint16_t v_total; + uint16_t v_count; } c_init_huffman[1]; } private_impl; } puffs_flate_decoder; diff --git a/lang/check/assert.go b/lang/check/assert.go index 985dff3a..cf1e2168 100644 --- a/lang/check/assert.go +++ b/lang/check/assert.go @@ -275,6 +275,11 @@ func errFailedOrNil(ok bool) error { var errFailed = errors.New("failed") +func binOpReasonError(tm *t.Map, op t.ID, lhs *a.Expr, rhs *a.Expr, err error) error { + n := a.NewExpr(a.FlagsTypeChecked, op, 0, lhs.Node(), nil, rhs.Node(), nil) + return fmt.Errorf("cannot prove %q: %v", n.String(tm), err) +} + type reason func(q *checker, n *a.Assert) error type reasonMap map[t.Key]reason @@ -284,36 +289,51 @@ var reasons = [...]struct { r reason }{ {`"a < (b + c): a < c; 0 <= b"`, func(q *checker, n *a.Assert) error { - op, a, bc := parseBinaryOp(n.Condition()) + op, xa, bc := parseBinaryOp(n.Condition()) if op.Key() != t.KeyXBinaryLessThan { return errFailed } - op, b, c := parseBinaryOp(bc) + op, xb, xc := parseBinaryOp(bc) if op.Key() != t.KeyXBinaryPlus { return errFailed } - if err := q.proveBinaryOp(t.KeyXBinaryLessThan, a, c); err != nil { - return fmt.Errorf("cannot prove \"%s < %s\": %v", a.String(q.tm), c.String(q.tm), err) + if err := q.proveBinaryOp(t.KeyXBinaryLessThan, xa, xc); err != nil { + return binOpReasonError(q.tm, t.IDXBinaryLessThan, xa, xc, err) } - if err := q.proveBinaryOp(t.KeyXBinaryLessEq, zeroExpr, b); err != nil { - return fmt.Errorf("cannot prove \"%s <= %s\": %v", zeroExpr.String(q.tm), b.String(q.tm), err) + if err := q.proveBinaryOp(t.KeyXBinaryLessEq, zeroExpr, xb); err != nil { + return binOpReasonError(q.tm, t.IDXBinaryLessEq, zeroExpr, xb, err) + } + return nil + }}, + {`"(a + b) <= c: a <= (c - b)"`, func(q *checker, n *a.Assert) error { + op, ab, xc := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessEq { + return errFailed + } + op, xa, xb := parseBinaryOp(ab) + if op.Key() != t.KeyXBinaryPlus { + return errFailed + } + sub := a.NewExpr(a.FlagsTypeChecked, t.IDXBinaryMinus, 0, xc.Node(), nil, xb.Node(), nil) + if err := q.proveBinaryOp(t.KeyXBinaryLessEq, xa, sub); err != nil { + return binOpReasonError(q.tm, t.IDXBinaryLessEq, xa, sub, err) } return nil }}, {`"a < b: a < c; c <= b"`, func(q *checker, n *a.Assert) error { - c := argValue(q.tm, n.Args(), "c") - if c == nil { + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { return errFailed } - op, a, b := parseBinaryOp(n.Condition()) + op, xa, xb := parseBinaryOp(n.Condition()) if op.Key() != t.KeyXBinaryLessThan { return errFailed } - if err := q.proveBinaryOp(t.KeyXBinaryLessThan, a, c); err != nil { - return fmt.Errorf("cannot prove \"%s < %s\": %v", a.String(q.tm), c.String(q.tm), err) + if err := q.proveBinaryOp(t.KeyXBinaryLessThan, xa, xc); err != nil { + return binOpReasonError(q.tm, t.IDXBinaryLessThan, xa, xc, err) } - if err := q.proveBinaryOp(t.KeyXBinaryLessEq, c, b); err != nil { - return fmt.Errorf("cannot prove \"%s <= %s\": %v", c.String(q.tm), b.String(q.tm), err) + if err := q.proveBinaryOp(t.KeyXBinaryLessEq, xc, xb); err != nil { + return binOpReasonError(q.tm, t.IDXBinaryLessEq, xc, xb, err) } return nil }}, diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 5c1f4090..cb828647 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -8,19 +8,19 @@ packageid "flte" pub error "bad distance code count" pub error "bad flate block" pub error "bad literal/length code count" -pub error "inconsistent Huffman table" pub error "inconsistent stored block length" +pub error "internal error: inconsistent init_huffman state" pub error "internal error: inconsistent n_bits" +pub error "no Huffman codes" pub error "TODO: fixed Huffman blocks" -pub error "TODO: dynamic Huffman blocks" pub struct decoder?( // These fields yield src's bits in Least Significant Bits order. bits u32, n_bits u32, - huff[2] huffman_decoder, + huffs[2] huffman_decoder, code_lengths[289] u8[..15], // These temporary fields are used to debug work-in-progress code. @@ -133,9 +133,31 @@ pri func decoder.decode_dynamic?(dst writer1, src reader1)() { // and its callers, as it can only return errors, not suspensions. pri func decoder.init_huffman?(which u32[..1], n_codes u32[..289])() { + // For the clcode example in this package's README.md: + // - n_codes = 19 + // - code_lengths[ 0] = 3 + // - code_lengths[ 1] = 0 + // - code_lengths[ 2] = 0 + // - code_lengths[ 3] = 5 + // - code_lengths[ 4] = 3 + // - code_lengths[ 5] = 3 + // - code_lengths[ 6] = 3 + // - code_lengths[ 7] = 3 + // - code_lengths[ 8] = 3 + // - code_lengths[ 9] = 3 + // - code_lengths[10] = 0 + // - code_lengths[11] = 0 + // - code_lengths[12] = 0 + // - code_lengths[13] = 0 + // - code_lengths[14] = 0 + // - code_lengths[15] = 0 + // - code_lengths[16] = 0 + // - code_lengths[17] = 4 + // - code_lengths[18] = 5 + var i u32 while i < 16 { - this.huff[in.which].counts[i] = 0 + this.huffs[in.which].counts[i] = 0 i += 1 } i = 0 @@ -143,26 +165,85 @@ pri func decoder.init_huffman?(which u32[..1], n_codes u32[..289])() { assert i < 289 via "a < b: a < c; c <= b"(c:in.n_codes) // TODO: this if should be unnecessary. Have some way to assert that, // for all j, counts[j] <= i, and thus counts[j]++ will not overflow. - if this.huff[in.which].counts[this.code_lengths[i]] >= 289 { - return error "inconsistent Huffman table" + if this.huffs[in.which].counts[this.code_lengths[i]] >= 289 { + return error "internal error: inconsistent init_huffman state" } - this.huff[in.which].counts[this.code_lengths[i]] += 1 + this.huffs[in.which].counts[this.code_lengths[i]] += 1 i += 1 } - if (this.huff[in.which].counts[0] as u32) == in.n_codes { - // There are no codes. - return error "inconsistent Huffman table" + if (this.huffs[in.which].counts[0] as u32) == in.n_codes { + return error "no Huffman codes" } - // For the clcode_lengths example in this package's README.md: + // For the clcode example in this package's README.md: + // - counts[0] = 9 + // - counts[1] = 0 + // - counts[2] = 0 // - counts[3] = 7 // - counts[4] = 1 // - counts[5] = 2 - // - all other counts elements are zero. + // - all other counts elements are 0. // TODO: check for the total code_lengths being too short or too long. + + var offsets[16] u16[..289] + var total u16[..289] + i = 1 + while i <= 15 { + offsets[i] = total + var count u16[..289] = this.huffs[in.which].counts[i] + if total > (289 - count) { + return error "internal error: inconsistent init_huffman state" + } + assert (total + count) <= 289 via "(a + b) <= c: a <= (c - b)"() + // TODO: change this to total += count, once the proof engine's bounds + // checking can handle it. + total = total + count + i += 1 + } + + // For the clcode example in this package's README.md: + // - offsets[0] = 0 + // - offsets[1] = 0 + // - offsets[2] = 0 + // - offsets[3] = 0 + // - offsets[4] = 7 + // - offsets[5] = 8 + // - offsets[6] = 10 + // - all other offsets elements are 10. + + i = 0 + while i < in.n_codes { + assert i < 289 via "a < b: a < c; c <= b"(c:in.n_codes) + if this.code_lengths[i] != 0 { + this.huffs[in.which].symbols[offsets[this.code_lengths[i]]] = i as u16 + if offsets[this.code_lengths[i]] >= 289 { + return error "internal error: inconsistent init_huffman state" + } + offsets[this.code_lengths[i]] += 1 + } + i += 1 + } + + // For the clcode example in this package's README.md: + // - symbols[0] = 0 + // - symbols[1] = 4 + // - symbols[2] = 5 + // - symbols[3] = 6 + // - symbols[4] = 7 + // - symbols[5] = 8 + // - symbols[6] = 9 + // - symbols[7] = 17 + // - symbols[8] = 3 + // - symbols[9] = 18 + // + // As a (local variable) side effect, offsets' values have been updated: + // - offsets[3] = 7, formerly 0 + // - offsets[4] = 8, formerly 7 + // - offsets[5] = 10, formerly 8 } pri struct huffman_decoder( counts[16] u16[..289], + symbols[290] u16[..289], ) diff --git a/test/c/std/flate.c b/test/c/std/flate.c index ba8d2348..d1a122c1 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -229,32 +229,71 @@ void test_puffs_flate_work_in_progress() { } if (dec.private_impl.f_wip0 != 266) { - FAIL("wip0: got %" PRIu32 ", want %" PRIu32, dec.private_impl.f_wip0, 266); + FAIL("wip0: got %" PRIu32 ", want %" PRIu32, dec.private_impl.f_wip0, + (uint32_t)(266)); return; } if (dec.private_impl.f_wip1 != 20) { - FAIL("wip1: got %" PRIu32 ", want %" PRIu32, dec.private_impl.f_wip1, 20); + FAIL("wip1: got %" PRIu32 ", want %" PRIu32, dec.private_impl.f_wip1, + (uint32_t)(20)); return; } if (dec.private_impl.f_code_lengths[16] != 0) { - FAIL("clcl[16]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[16], 0); + FAIL("cl[16]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[16], (uint8_t)(0)); return; } if (dec.private_impl.f_code_lengths[17] != 4) { - FAIL("clcl[17]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[17], 4); + FAIL("cl[17]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[17], (uint8_t)(4)); return; } if (dec.private_impl.f_code_lengths[18] != 5) { - FAIL("clcl[18]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[18], 5); + FAIL("cl[18]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[18], (uint8_t)(5)); return; } if (dec.private_impl.f_code_lengths[3] != 5) { - FAIL("clcl[3]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[3], 5); + FAIL("cl[3]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[3], (uint8_t)(5)); + return; + } + + if (dec.private_impl.f_huffs[0].private_impl.f_counts[3] != 7) { + FAIL("c[3]: got %" PRIu16 ", want %" PRIu16, + dec.private_impl.f_huffs[0].private_impl.f_counts[3], (uint16_t)(7)); + return; + } + if (dec.private_impl.f_huffs[0].private_impl.f_counts[4] != 1) { + FAIL("c[4]: got %" PRIu16 ", want %" PRIu16, + dec.private_impl.f_huffs[0].private_impl.f_counts[4], (uint16_t)(1)); + return; + } + if (dec.private_impl.f_huffs[0].private_impl.f_counts[5] != 2) { + FAIL("c[5]: got %" PRIu16 ", want %" PRIu16, + dec.private_impl.f_huffs[0].private_impl.f_counts[5], (uint16_t)(2)); + return; + } + + if (dec.private_impl.f_huffs[0].private_impl.f_symbols[6] != 9) { + FAIL("s[6]: got %" PRIu16 ", want %" PRIu16, + dec.private_impl.f_huffs[0].private_impl.f_symbols[6], (uint16_t)(9)); + return; + } + if (dec.private_impl.f_huffs[0].private_impl.f_symbols[7] != 17) { + FAIL("s[7]: got %" PRIu16 ", want %" PRIu16, + dec.private_impl.f_huffs[0].private_impl.f_symbols[7], (uint16_t)(17)); + return; + } + if (dec.private_impl.f_huffs[0].private_impl.f_symbols[8] != 3) { + FAIL("s[8]: got %" PRIu16 ", want %" PRIu16, + dec.private_impl.f_huffs[0].private_impl.f_symbols[8], (uint16_t)(3)); + return; + } + if (dec.private_impl.f_huffs[0].private_impl.f_symbols[9] != 18) { + FAIL("s[9]: got %" PRIu16 ", want %" PRIu16, + dec.private_impl.f_huffs[0].private_impl.f_symbols[9], (uint16_t)(18)); return; } }