From e772352036364f053d2d473790bf9f7689486086 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 22 Jul 2017 16:45:54 +1000 Subject: [PATCH] Decode the lcode and dcode code lengths. --- gen/c/std/flate.c | 89 +++++++++++++++++++++++++++--------- gen/h/std/flate.h | 31 +++++++------ std/flate/decode_flate.puffs | 59 +++++++++++++++++++----- test/c/std/flate.c | 82 +++++++++++---------------------- 4 files changed, 160 insertions(+), 101 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 99b2445b..f7457d24 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -102,19 +102,24 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_SUSPENSION_LIMITED_READ 9 // 0x00000009 #define PUFFS_FLATE_SUSPENSION_LIMITED_WRITE 10 // 0x0000000a -#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040128 // 0xbb08f800 -#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040127 // 0xbb08f801 -#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040126 // 0xbb08f802 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_COUNT \ + -1157040128 // 0xbb08f800 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_REPETITION \ + -1157040127 // 0xbb08f801 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040126 // 0xbb08f802 +#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040125 // 0xbb08f803 +#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040124 // 0xbb08f804 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040125 // 0xbb08f803 -#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ - -1157040124 // 0xbb08f804 -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ + -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040121 // 0xbb08f807 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040122 // 0xbb08f806 -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040121 // 0xbb08f807 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040120 // 0xbb08f808 + -1157040120 // 0xbb08f808 +#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040119 // 0xbb08f809 +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040118 // 0xbb08f80a +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040117 // 0xbb08f80b bool puffs_flate_status_is_error(puffs_flate_status s); @@ -154,9 +159,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; puffs_flate_huffman_decoder f_huffs[2]; - uint8_t f_code_lengths[289]; - uint32_t f_wip0; - uint32_t f_wip1; + uint8_t f_code_lengths[316]; struct { uint32_t coro_susp_point; @@ -185,6 +188,8 @@ typedef struct { uint32_t v_index; uint32_t v_length; uint32_t v_n_extra_bits; + uint8_t v_rep_symbol; + uint32_t v_rep_count; } c_init_huffs[1]; struct { uint32_t coro_susp_point; @@ -275,7 +280,9 @@ const char* puffs_flate_status_strings0[11] = { "flate: limited write", }; -const char* puffs_flate_status_strings1[9] = { +const char* puffs_flate_status_strings1[12] = { + "flate: bad Huffman code length count", + "flate: bad Huffman code length repetition", "flate: bad Huffman code", "flate: bad distance code count", "flate: bad flate block", @@ -283,6 +290,7 @@ const char* puffs_flate_status_strings1[9] = { "flate: inconsistent stored block length", "flate: internal error: inconsistent Huffman decoder state", "flate: internal error: inconsistent n_bits", + "flate: missing end-of-block code", "flate: no Huffman codes", "flate: TODO: fixed Huffman blocks", }; @@ -297,7 +305,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) { break; case puffs_flate_packageid: a = puffs_flate_status_strings1; - n = 9; + n = 12; break; } uint32_t i = s & 0xff; @@ -717,6 +725,8 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, uint32_t v_index; uint32_t v_length; uint32_t v_n_extra_bits; + uint8_t v_rep_symbol; + uint32_t v_rep_count; uint8_t* b_rptr_src = NULL; uint8_t* b_rend_src = NULL; @@ -747,6 +757,8 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, v_index = self->private_impl.c_init_huffs[0].v_index; v_length = self->private_impl.c_init_huffs[0].v_length; v_n_extra_bits = self->private_impl.c_init_huffs[0].v_n_extra_bits; + v_rep_symbol = self->private_impl.c_init_huffs[0].v_rep_symbol; + v_rep_count = self->private_impl.c_init_huffs[0].v_rep_count; } switch (coro_susp_point) { PUFFS_COROUTINE_SUSPENSION_POINT(0); @@ -804,6 +816,7 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, goto suspend; } v_i = 0; + label_0_continue:; while (v_i < (v_n_lit + v_n_dist)) { v_symbol = 0; v_code = 0; @@ -855,7 +868,7 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, } v_symbol = ((uint32_t)( self->private_impl.f_huffs[0].private_impl.f_symbols[v_index])); - goto label_0_break; + goto label_1_break; } v_index += v_count; if (v_index > 65535) { @@ -872,14 +885,31 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, v_code <<= 1; v_length += 1; } - label_0_break:; + label_1_break:; + if (v_symbol < 16) { + self->private_impl.f_code_lengths[v_i] = ((uint8_t)(v_symbol)); + v_i += 1; + goto label_0_continue; + } v_n_extra_bits = 0; + v_rep_symbol = 0; + v_rep_count = 0; if (v_symbol == 16) { v_n_extra_bits = 2; + if (v_i <= 0) { + status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_REPETITION; + goto exit; + } + v_rep_symbol = self->private_impl.f_code_lengths[v_i - 1]; + v_rep_count = 3; } else if (v_symbol == 17) { v_n_extra_bits = 3; - } else if (v_symbol == 18) { + v_rep_symbol = 0; + v_rep_count = 3; + } else { v_n_extra_bits = 7; + v_rep_symbol = 0; + v_rep_count = 11; } while (v_n_bits < v_n_extra_bits) { PUFFS_COROUTINE_SUSPENSION_POINT(5); @@ -890,12 +920,27 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, v_bits |= (((uint32_t)(t_3)) << v_n_bits); v_n_bits += 8; } + v_rep_count += ((v_bits) & ((1 << (v_n_extra_bits)) - 1)); v_bits >>= v_n_extra_bits; v_n_bits -= v_n_extra_bits; - v_i += 1; + while (v_rep_count > 0) { + if (v_i >= (v_n_lit + v_n_dist)) { + status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_COUNT; + goto exit; + } + self->private_impl.f_code_lengths[v_i] = v_rep_symbol; + v_i += 1; + v_rep_count -= 1; + } + } + if (v_i != (v_n_lit + v_n_dist)) { + status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_COUNT; + goto exit; + } + if (self->private_impl.f_code_lengths[256] == 0) { + status = PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE; + goto exit; } - self->private_impl.f_wip0 = v_n_lit; - self->private_impl.f_wip1 = v_n_dist; self->private_impl.f_bits = v_bits; self->private_impl.f_n_bits = v_n_bits; self->private_impl.c_init_huffs[0].coro_susp_point = 0; @@ -918,6 +963,8 @@ suspend: self->private_impl.c_init_huffs[0].v_index = v_index; self->private_impl.c_init_huffs[0].v_length = v_length; self->private_impl.c_init_huffs[0].v_n_extra_bits = v_n_extra_bits; + self->private_impl.c_init_huffs[0].v_rep_symbol = v_rep_symbol; + self->private_impl.c_init_huffs[0].v_rep_count = v_rep_count; exit: if (a_src.buf) { diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index a7708519..37d35afe 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -102,19 +102,24 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_SUSPENSION_LIMITED_READ 9 // 0x00000009 #define PUFFS_FLATE_SUSPENSION_LIMITED_WRITE 10 // 0x0000000a -#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040128 // 0xbb08f800 -#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040127 // 0xbb08f801 -#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040126 // 0xbb08f802 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_COUNT \ + -1157040128 // 0xbb08f800 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_REPETITION \ + -1157040127 // 0xbb08f801 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040126 // 0xbb08f802 +#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040125 // 0xbb08f803 +#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040124 // 0xbb08f804 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040125 // 0xbb08f803 -#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ - -1157040124 // 0xbb08f804 -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ + -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040121 // 0xbb08f807 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040122 // 0xbb08f806 -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040121 // 0xbb08f807 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040120 // 0xbb08f808 + -1157040120 // 0xbb08f808 +#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040119 // 0xbb08f809 +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040118 // 0xbb08f80a +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040117 // 0xbb08f80b bool puffs_flate_status_is_error(puffs_flate_status s); @@ -154,9 +159,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; puffs_flate_huffman_decoder f_huffs[2]; - uint8_t f_code_lengths[289]; - uint32_t f_wip0; - uint32_t f_wip1; + uint8_t f_code_lengths[316]; struct { uint32_t coro_susp_point; @@ -185,6 +188,8 @@ typedef struct { uint32_t v_index; uint32_t v_length; uint32_t v_n_extra_bits; + uint8_t v_rep_symbol; + uint32_t v_rep_count; } c_init_huffs[1]; struct { uint32_t coro_susp_point; diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 4cb1d81c..4b452e7f 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -5,6 +5,8 @@ packageid "flte" +pub error "bad Huffman code length count" +pub error "bad Huffman code length repetition" pub error "bad Huffman code" pub error "bad distance code count" pub error "bad flate block" @@ -12,6 +14,7 @@ pub error "bad literal/length code count" pub error "inconsistent stored block length" pub error "internal error: inconsistent Huffman decoder state" pub error "internal error: inconsistent n_bits" +pub error "missing end-of-block code" pub error "no Huffman codes" pub error "TODO: fixed Huffman blocks" @@ -27,11 +30,7 @@ pub struct decoder?( // - huffs[1] is used for dcode. huffs[2] huffman_decoder, - code_lengths[289] u8[..15], - - // These temporary fields are used to debug work-in-progress code. - wip0 u32, - wip1 u32, + code_lengths[316] u8[..15], ) pub func decoder.decode?(dst writer1, src reader1)() { @@ -242,17 +241,39 @@ pri func decoder.init_huffs?(src reader1)() { length += 1 } - // Read any extra bits, as per the RFC 1951 section 3.2.7. + // Write a literal code length. + if symbol < 16 { + this.code_lengths[i] = symbol as u8 + i += 1 + continue + } + + // Write a repeated code length. var n_extra_bits u32[..7] + var rep_symbol u8[..15] + var rep_count u32 if symbol == 16 { n_extra_bits = 2 + if i <= 0 { + return error "bad Huffman code length repetition" + } + rep_symbol = this.code_lengths[i - 1] + rep_count = 3 + assert rep_count <= 11 } else if symbol == 17 { n_extra_bits = 3 - } else if symbol == 18 { + rep_symbol = 0 + rep_count = 3 + assert rep_count <= 11 + } else { n_extra_bits = 7 + rep_symbol = 0 + rep_count = 11 + assert rep_count <= 11 } while n_bits < n_extra_bits, inv i < 316, + inv rep_count <= 11, inv n_dist <= 30, inv n_lit <= 286, post n_bits >= n_extra_bits, @@ -261,15 +282,31 @@ pri func decoder.init_huffs?(src reader1)() { bits |= (in.src.read_u8?() as u32) << n_bits n_bits += 8 } - // TODO: do something with those extra bits. + rep_count += bits.low_bits(n:n_extra_bits) bits >>= n_extra_bits n_bits -= n_extra_bits - i += 1 + while rep_count > 0, + inv n_dist <= 30, + inv n_lit <= 286, + { + // TODO: hoist this check up one level? + if i >= (n_lit + n_dist) { + return error "bad Huffman code length count" + } + assert i < (286 + 30) via "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"(b0:n_lit, c0:n_dist) + this.code_lengths[i] = rep_symbol + i += 1 + rep_count -= 1 + } } - this.wip0 = n_lit - this.wip1 = n_dist + if i != (n_lit + n_dist) { + return error "bad Huffman code length count" + } + if this.code_lengths[256] == 0 { + return error "missing end-of-block code" + } this.bits = bits this.n_bits = n_bits diff --git a/test/c/std/flate.c b/test/c/std/flate.c index d1a122c1..d64163f2 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -228,72 +228,42 @@ void test_puffs_flate_work_in_progress() { return; } - if (dec.private_impl.f_wip0 != 266) { - FAIL("wip0: got %" PRIu32 ", want %" PRIu32, dec.private_impl.f_wip0, - (uint32_t)(266)); + // As per the package's README.md: + // lcode_lengths[31] = 0 + // lcode_lengths[32] = 4 + // lcode_lengths[33] = 8 + // lcode_lengths[34] = 0 + // etc + // dcode_lengths[18] = 3 + // dcode_lengths[19] = 6 + if (dec.private_impl.f_code_lengths[31] != 0) { + FAIL("cl[31]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[31], (uint8_t)(0)); return; } - if (dec.private_impl.f_wip1 != 20) { - FAIL("wip1: got %" PRIu32 ", want %" PRIu32, dec.private_impl.f_wip1, - (uint32_t)(20)); + if (dec.private_impl.f_code_lengths[32] != 4) { + FAIL("cl[32]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[32], (uint8_t)(4)); return; } - - if (dec.private_impl.f_code_lengths[16] != 0) { - FAIL("cl[16]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[16], (uint8_t)(0)); + if (dec.private_impl.f_code_lengths[33] != 8) { + FAIL("cl[33]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[33], (uint8_t)(8)); return; } - if (dec.private_impl.f_code_lengths[17] != 4) { - FAIL("cl[17]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[17], (uint8_t)(4)); + if (dec.private_impl.f_code_lengths[34] != 0) { + FAIL("cl[34]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[34], (uint8_t)(0)); return; } - if (dec.private_impl.f_code_lengths[18] != 5) { - FAIL("cl[18]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[18], (uint8_t)(5)); + if (dec.private_impl.f_code_lengths[266 + 18] != 3) { + FAIL("cl[266 + 18]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[266 + 18], (uint8_t)(3)); return; } - if (dec.private_impl.f_code_lengths[3] != 5) { - FAIL("cl[3]: got %" PRIu8 ", want %" PRIu8, - dec.private_impl.f_code_lengths[3], (uint8_t)(5)); - return; - } - - if (dec.private_impl.f_huffs[0].private_impl.f_counts[3] != 7) { - FAIL("c[3]: got %" PRIu16 ", want %" PRIu16, - dec.private_impl.f_huffs[0].private_impl.f_counts[3], (uint16_t)(7)); - return; - } - if (dec.private_impl.f_huffs[0].private_impl.f_counts[4] != 1) { - FAIL("c[4]: got %" PRIu16 ", want %" PRIu16, - dec.private_impl.f_huffs[0].private_impl.f_counts[4], (uint16_t)(1)); - return; - } - if (dec.private_impl.f_huffs[0].private_impl.f_counts[5] != 2) { - FAIL("c[5]: got %" PRIu16 ", want %" PRIu16, - dec.private_impl.f_huffs[0].private_impl.f_counts[5], (uint16_t)(2)); - return; - } - - if (dec.private_impl.f_huffs[0].private_impl.f_symbols[6] != 9) { - FAIL("s[6]: got %" PRIu16 ", want %" PRIu16, - dec.private_impl.f_huffs[0].private_impl.f_symbols[6], (uint16_t)(9)); - return; - } - if (dec.private_impl.f_huffs[0].private_impl.f_symbols[7] != 17) { - FAIL("s[7]: got %" PRIu16 ", want %" PRIu16, - dec.private_impl.f_huffs[0].private_impl.f_symbols[7], (uint16_t)(17)); - return; - } - if (dec.private_impl.f_huffs[0].private_impl.f_symbols[8] != 3) { - FAIL("s[8]: got %" PRIu16 ", want %" PRIu16, - dec.private_impl.f_huffs[0].private_impl.f_symbols[8], (uint16_t)(3)); - return; - } - if (dec.private_impl.f_huffs[0].private_impl.f_symbols[9] != 18) { - FAIL("s[9]: got %" PRIu16 ", want %" PRIu16, - dec.private_impl.f_huffs[0].private_impl.f_symbols[9], (uint16_t)(18)); + if (dec.private_impl.f_code_lengths[266 + 19] != 6) { + FAIL("cl[266 + 19]: got %" PRIu8 ", want %" PRIu8, + dec.private_impl.f_code_lengths[266 + 19], (uint8_t)(6)); return; } }