From fd07fa1f27b8fe452a482232373aefd9fd564f5e Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 21 Jul 2017 17:47:58 +1000 Subject: [PATCH] Decode flate's clcode symbols. --- gen/c/std/flate.c | 151 +++++++++++++++++++++++++++++++---- gen/h/std/flate.h | 24 ++++-- lang/check/assert.go | 5 +- lang/check/bounds.go | 18 ++++- std/flate/decode_flate.puffs | 131 ++++++++++++++++++++++++++++-- 5 files changed, 299 insertions(+), 30 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 89e01ed5..0b02f425 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -102,18 +102,19 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_SUSPENSION_LIMITED_READ 9 // 0x00000009 #define PUFFS_FLATE_SUSPENSION_LIMITED_WRITE 10 // 0x0000000a -#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800 -#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040128 // 0xbb08f800 +#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040127 // 0xbb08f801 +#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040126 // 0xbb08f802 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040126 // 0xbb08f802 -#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040125 // 0xbb08f803 -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE \ +#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040124 // 0xbb08f804 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040123 // 0xbb08f805 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040122 // 0xbb08f806 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 + -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040121 // 0xbb08f807 +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040120 // 0xbb08f808 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -177,6 +178,13 @@ typedef struct { uint32_t v_n_dist; uint32_t v_n_clen; uint32_t v_i; + uint32_t v_symbol; + uint32_t v_code; + uint32_t v_count; + uint32_t v_first; + uint32_t v_index; + uint32_t v_length; + uint32_t v_n_extra_bits; } c_init_huffs[1]; struct { uint32_t coro_susp_point; @@ -267,12 +275,13 @@ const char* puffs_flate_status_strings0[11] = { "flate: limited write", }; -const char* puffs_flate_status_strings1[8] = { +const char* puffs_flate_status_strings1[9] = { + "flate: bad Huffman code", "flate: bad distance code count", "flate: bad flate block", "flate: bad literal/length code count", "flate: inconsistent stored block length", - "flate: internal error: inconsistent init_huff state", + "flate: internal error: inconsistent Huffman decoder state", "flate: internal error: inconsistent n_bits", "flate: no Huffman codes", "flate: TODO: fixed Huffman blocks", @@ -288,7 +297,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) { break; case puffs_flate_packageid: a = puffs_flate_status_strings1; - n = 8; + n = 9; break; } uint32_t i = s & 0xff; @@ -701,6 +710,13 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, uint32_t v_n_dist; uint32_t v_n_clen; uint32_t v_i; + uint32_t v_symbol; + uint32_t v_code; + uint32_t v_count; + uint32_t v_first; + uint32_t v_index; + uint32_t v_length; + uint32_t v_n_extra_bits; uint8_t* b_rptr_src = NULL; uint8_t* b_rend_src = NULL; @@ -724,6 +740,13 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, v_n_dist = self->private_impl.c_init_huffs[0].v_n_dist; v_n_clen = self->private_impl.c_init_huffs[0].v_n_clen; v_i = self->private_impl.c_init_huffs[0].v_i; + v_symbol = self->private_impl.c_init_huffs[0].v_symbol; + v_code = self->private_impl.c_init_huffs[0].v_code; + v_count = self->private_impl.c_init_huffs[0].v_count; + v_first = self->private_impl.c_init_huffs[0].v_first; + v_index = self->private_impl.c_init_huffs[0].v_index; + v_length = self->private_impl.c_init_huffs[0].v_length; + v_n_extra_bits = self->private_impl.c_init_huffs[0].v_n_extra_bits; } switch (coro_susp_point) { PUFFS_COROUTINE_SUSPENSION_POINT(0); @@ -780,6 +803,97 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, if (status) { goto suspend; } + v_i = 0; + while (v_i < 31) { + v_symbol = 0; + v_code = 0; + v_count = 0; + v_first = 0; + v_index = 0; + v_length = 1; + while (true) { + if (v_length > 15) { + status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE; + goto exit; + } + while (v_n_bits <= 0) { + PUFFS_COROUTINE_SUSPENSION_POINT(4); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_2 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_2)) << v_n_bits); + v_n_bits += 8; + } + v_code |= (v_bits & 1); + if (v_code > 65535) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_bits >>= 1; + v_n_bits -= 1; + v_count = ((uint32_t)( + self->private_impl.f_huffs[0].private_impl.f_counts[v_length])); + if (v_count > 65535) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + if (v_code < (v_first + v_count)) { + v_index += v_code; + if (v_index < v_first) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_index -= v_first; + if (v_index >= 290) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_symbol = ((uint32_t)( + self->private_impl.f_huffs[0].private_impl.f_symbols[v_index])); + goto label_0_break; + } + v_index += v_count; + if (v_index > 65535) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_first = ((v_first + v_count) << 1); + if (v_first > 65535) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_code <<= 1; + v_length += 1; + } + label_0_break:; + v_n_extra_bits = 0; + if (v_symbol == 16) { + v_n_extra_bits = 2; + } else if (v_symbol == 17) { + v_n_extra_bits = 3; + } else if (v_symbol == 18) { + v_n_extra_bits = 7; + } + while (v_n_bits < v_n_extra_bits) { + PUFFS_COROUTINE_SUSPENSION_POINT(5); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_3 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_3)) << v_n_bits); + v_n_bits += 8; + } + v_bits >>= v_n_extra_bits; + v_n_bits -= v_n_extra_bits; + v_i += 1; + } self->private_impl.f_wip0 = v_n_lit; self->private_impl.f_wip1 = v_n_dist; self->private_impl.f_bits = v_bits; @@ -797,6 +911,13 @@ suspend: self->private_impl.c_init_huffs[0].v_n_dist = v_n_dist; self->private_impl.c_init_huffs[0].v_n_clen = v_n_clen; self->private_impl.c_init_huffs[0].v_i = v_i; + self->private_impl.c_init_huffs[0].v_symbol = v_symbol; + self->private_impl.c_init_huffs[0].v_code = v_code; + self->private_impl.c_init_huffs[0].v_count = v_count; + self->private_impl.c_init_huffs[0].v_first = v_first; + self->private_impl.c_init_huffs[0].v_index = v_index; + self->private_impl.c_init_huffs[0].v_length = v_length; + self->private_impl.c_init_huffs[0].v_n_extra_bits = v_n_extra_bits; exit: if (a_src.buf) { @@ -854,7 +975,8 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, if (self->private_impl.f_huffs[a_which] .private_impl.f_counts[self->private_impl.f_code_lengths[v_i]] >= 289) { - status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE; + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } self->private_impl.f_huffs[a_which] @@ -879,7 +1001,8 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_offsets[v_i] = v_total; v_count = self->private_impl.f_huffs[a_which].private_impl.f_counts[v_i]; if (v_total > (289 - v_count)) { - status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE; + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } v_total = (v_total + v_count); @@ -894,7 +1017,7 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, ((uint16_t)(v_i)); if (v_offsets[self->private_impl.f_code_lengths[v_i]] >= 289) { status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE; + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } v_offsets[self->private_impl.f_code_lengths[v_i]] += 1; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index a06f8818..a7708519 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -102,18 +102,19 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_SUSPENSION_LIMITED_READ 9 // 0x00000009 #define PUFFS_FLATE_SUSPENSION_LIMITED_WRITE 10 // 0x0000000a -#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800 -#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040128 // 0xbb08f800 +#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040127 // 0xbb08f801 +#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040126 // 0xbb08f802 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040126 // 0xbb08f802 -#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040125 // 0xbb08f803 -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE \ +#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040124 // 0xbb08f804 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040123 // 0xbb08f805 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040122 // 0xbb08f806 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 + -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040121 // 0xbb08f807 +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040120 // 0xbb08f808 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -177,6 +178,13 @@ typedef struct { uint32_t v_n_dist; uint32_t v_n_clen; uint32_t v_i; + uint32_t v_symbol; + uint32_t v_code; + uint32_t v_count; + uint32_t v_first; + uint32_t v_index; + uint32_t v_length; + uint32_t v_n_extra_bits; } c_init_huffs[1]; struct { uint32_t coro_susp_point; diff --git a/lang/check/assert.go b/lang/check/assert.go index cf1e2168..62cf2236 100644 --- a/lang/check/assert.go +++ b/lang/check/assert.go @@ -67,7 +67,10 @@ func (z *facts) appendFact(fact *a.Expr) { z.appendFact(fact.RHS().Expr()) return case t.KeyXAssociativeAnd: - // TODO. + for _, a := range fact.Args() { + z.appendFact(a.Expr()) + } + return } *z = append(*z, fact) diff --git a/lang/check/bounds.go b/lang/check/bounds.go index 2acfbc01..541ba7e1 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -101,7 +101,7 @@ func invert(tm *t.Map, n *a.Expr) (*a.Expr, error) { if cv := n.ConstValue(); cv != nil { return nil, fmt.Errorf("check: invert(%q) called on constant expression", n.String(tm)) } - id0, lhs, rhs := n.ID0(), n.LHS().Expr(), n.RHS().Expr() + id0, lhs, rhs, args := n.ID0(), n.LHS().Expr(), n.RHS().Expr(), []*a.Node(nil) switch id0.Key() { case t.KeyXUnaryNot: return rhs, nil @@ -132,10 +132,24 @@ func invert(tm *t.Map, n *a.Expr) (*a.Expr, error) { } else { id0 = t.IDXBinaryAnd } + case t.KeyXAssociativeAnd, t.KeyXAssociativeOr: + args = make([]*a.Node, 0, len(n.Args())) + for _, a := range n.Args() { + v, err := invert(tm, a.Expr()) + if err != nil { + return nil, err + } + args = append(args, v.Node()) + } + if id0.Key() == t.KeyXAssociativeAnd { + id0 = t.IDXAssociativeOr + } else { + id0 = t.IDXAssociativeAnd + } default: id0, lhs, rhs = t.IDXUnaryNot, nil, n } - o := a.NewExpr(n.Node().Raw().Flags(), id0, 0, lhs.Node(), nil, rhs.Node(), nil) + o := a.NewExpr(n.Node().Raw().Flags(), id0, 0, lhs.Node(), nil, rhs.Node(), args) o.SetMType(n.MType()) return o, nil } diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 6e238a91..84804b50 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -5,11 +5,12 @@ packageid "flte" +pub error "bad Huffman code" pub error "bad distance code count" pub error "bad flate block" pub error "bad literal/length code count" pub error "inconsistent stored block length" -pub error "internal error: inconsistent init_huff state" +pub error "internal error: inconsistent Huffman decoder state" pub error "internal error: inconsistent n_bits" pub error "no Huffman codes" @@ -125,7 +126,127 @@ pri func decoder.init_huffs?(src reader1)() { } this.init_huff?(which:0, n_codes:19) - // TODO: implement. + i = 0 + while i < 31 { // TODO: replace 31 with n_lit + n_dist. + // Decode a clcode symbol from H-CL. + var symbol u32[..289] + var code u32 + var count u32 + var first u32 + var index u32 + var length u32 = 1 + while true, + pre first <= 0xFFFF, + pre index <= 0xFFFF, + inv i < 31, + { + if length > 15 { + return error "bad Huffman code" + } + + // Read the next bit. + while n_bits <= 0, + inv first <= 0xFFFF, + inv index <= 0xFFFF, + inv i < 31, + inv length <= 15, + post n_bits > 0, + { + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + code |= bits & 1 + // TODO: improve the proof engine's bounds checking so that any + // check returning "internal error: inconsistent Huffman decoder + // state" can be removed. There is one here and more below. + if code > 0xFFFF { + return error "internal error: inconsistent Huffman decoder state" + } + bits >>= 1 + n_bits -= 1 + + count = this.huffs[0].counts[length] as u32 + if count > 0xFFFF { + return error "internal error: inconsistent Huffman decoder state" + } + + // Compare the code so far to the counts. + // + // For the clcode example in this package's README.md: + // - counts[1] = 0 + // - counts[2] = 0 + // - counts[3] = 7 + // - counts[4] = 1 + // - counts[5] = 2 + // + // Recall that the map from bits to clcode symbols is: + // bits clcode + // 0b000 0 + // 0b001 4 + // 0b010 5 + // 0b011 6 + // 0b100 7 + // 0b101 8 + // 0b110 9 + // 0b1110 17 + // 0b11110 3 + // 0b11111 18 + // + // The loop iterations at this point have: + // - length=1, count=0, first= 0, index=0, code in [0.. 1]. + // - length=2, count=0, first= 0, index=0, code in [0.. 3]. + // - length=3, count=7, first= 0, index=0, code in [0.. 7]. + // - length=4, count=1, first=14, index=7, code in [0..15]. + // - length=5, count=2, first=30, index=8, code in [0..31]. + if code < (first + count) { + index += code + if index < first { + return error "internal error: inconsistent Huffman decoder state" + } + index -= first + if index >= 290 { + return error "internal error: inconsistent Huffman decoder state" + } + symbol = this.huffs[0].symbols[index] as u32 + break + } + + index += count + if index > 0xFFFF { + return error "internal error: inconsistent Huffman decoder state" + } + first = (first + count) << 1 + if first > 0xFFFF { + return error "internal error: inconsistent Huffman decoder state" + } + code <<= 1 + + length += 1 + } + + // Read any extra bits, as per the RFC 1951 section 3.2.7. + var n_extra_bits u32[..7] + if symbol == 16 { + n_extra_bits = 2 + } else if symbol == 17 { + n_extra_bits = 3 + } else if symbol == 18 { + n_extra_bits = 7 + } + while n_bits < n_extra_bits, + inv i < 31, + post n_bits >= n_extra_bits, + { + assert n_bits < 7 via "a < b: a < c; c <= b"(c:n_extra_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + // TODO: do something with those extra bits. + bits >>= n_extra_bits + n_bits -= n_extra_bits + + i += 1 + } this.wip0 = n_lit this.wip1 = n_dist @@ -176,7 +297,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() { // TODO: this if should be unnecessary. Have some way to assert that, // for all j, counts[j] <= i, and thus counts[j]++ will not overflow. if this.huffs[in.which].counts[this.code_lengths[i]] >= 289 { - return error "internal error: inconsistent init_huff state" + return error "internal error: inconsistent Huffman decoder state" } this.huffs[in.which].counts[this.code_lengths[i]] += 1 i += 1 @@ -203,7 +324,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() { offsets[i] = total var count u16[..289] = this.huffs[in.which].counts[i] if total > (289 - count) { - return error "internal error: inconsistent init_huff state" + return error "internal error: inconsistent Huffman decoder state" } assert (total + count) <= 289 via "(a + b) <= c: a <= (c - b)"() // TODO: change this to total += count, once the proof engine's bounds @@ -228,7 +349,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() { if this.code_lengths[i] != 0 { this.huffs[in.which].symbols[offsets[this.code_lengths[i]]] = i as u16 if offsets[this.code_lengths[i]] >= 289 { - return error "internal error: inconsistent init_huff state" + return error "internal error: inconsistent Huffman decoder state" } offsets[this.code_lengths[i]] += 1 }