From ea935e1f8ccf85f6ada3ce5fee045d80595173e2 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 27 Jul 2017 14:18:32 +1000 Subject: [PATCH] Use array-of-u32 Huffman decoder. --- gen/c/std/flate.c | 132 ++++++++++++----------------------- gen/h/std/flate.h | 9 +-- std/flate/decode_flate.puffs | 118 ++++++------------------------- 3 files changed, 69 insertions(+), 190 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 421ab304..fefacc04 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -172,12 +172,9 @@ typedef struct { uint32_t v_n_dist; uint32_t v_n_clen; uint32_t v_i; - uint32_t v_symbol; - uint32_t v_code; - uint32_t v_count; - uint32_t v_first; - uint32_t v_index; - uint32_t v_length; + uint32_t v_mask; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; uint32_t v_n_extra_bits; uint8_t v_rep_symbol; uint32_t v_rep_count; @@ -738,12 +735,9 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, uint32_t v_n_dist; uint32_t v_n_clen; uint32_t v_i; - uint32_t v_symbol; - uint32_t v_code; - uint32_t v_count; - uint32_t v_first; - uint32_t v_index; - uint32_t v_length; + uint32_t v_mask; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; uint32_t v_n_extra_bits; uint8_t v_rep_symbol; uint32_t v_rep_count; @@ -770,12 +764,10 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, v_n_dist = self->private_impl.c_init_huffs[0].v_n_dist; v_n_clen = self->private_impl.c_init_huffs[0].v_n_clen; v_i = self->private_impl.c_init_huffs[0].v_i; - v_symbol = self->private_impl.c_init_huffs[0].v_symbol; - v_code = self->private_impl.c_init_huffs[0].v_code; - v_count = self->private_impl.c_init_huffs[0].v_count; - v_first = self->private_impl.c_init_huffs[0].v_first; - v_index = self->private_impl.c_init_huffs[0].v_index; - v_length = self->private_impl.c_init_huffs[0].v_length; + v_mask = self->private_impl.c_init_huffs[0].v_mask; + v_table_entry = self->private_impl.c_init_huffs[0].v_table_entry; + v_table_entry_n_bits = + self->private_impl.c_init_huffs[0].v_table_entry_n_bits; v_n_extra_bits = self->private_impl.c_init_huffs[0].v_n_extra_bits; v_rep_symbol = self->private_impl.c_init_huffs[0].v_rep_symbol; v_rep_count = self->private_impl.c_init_huffs[0].v_rep_count; @@ -835,84 +827,43 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, if (status) { goto suspend; } + v_mask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[0]) - 1); v_i = 0; label_0_continue:; while (v_i < (v_n_lit + v_n_dist)) { - v_symbol = 0; - v_code = 0; - v_count = 0; - v_first = 0; - v_index = 0; - v_length = 1; + v_table_entry = 0; while (true) { - if (v_length > 15) { - status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE; - goto exit; - } - while (v_n_bits <= 0) { - PUFFS_COROUTINE_SUSPENSION_POINT(4); - if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { - goto short_read_src; - } - uint8_t t_2 = *b_rptr_src++; - v_bits |= (((uint32_t)(t_2)) << v_n_bits); - v_n_bits += 8; - } - v_code |= (v_bits & 1); - if (v_code > 65535) { - status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; - goto exit; - } - v_bits >>= 1; - v_n_bits -= 1; - v_count = ((uint32_t)(self->private_impl.f_counts[v_length])); - if (v_count > 65535) { - status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; - goto exit; - } - if (v_code < (v_first + v_count)) { - v_index += v_code; - if (v_index < v_first) { - status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; - goto exit; - } - v_index -= v_first; - if (v_index >= 290) { - status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; - goto exit; - } - v_symbol = ((uint32_t)(self->private_impl.f_symbols[v_index])); + v_table_entry = self->private_impl.f_huffs[0][v_bits & v_mask]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; goto label_1_break; } - v_index += v_count; - if (v_index > 65535) { - status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; - goto exit; + PUFFS_COROUTINE_SUSPENSION_POINT(4); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; } - v_first = ((v_first + v_count) << 1); - if (v_first > 65535) { - status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; - goto exit; - } - v_code <<= 1; - v_length += 1; + uint8_t t_2 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_2)) << v_n_bits); + v_n_bits += 8; } label_1_break:; - if (v_symbol < 16) { - self->private_impl.f_code_lengths[v_i] = ((uint8_t)(v_symbol)); + if ((v_table_entry >> 30) != 1) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_table_entry = ((v_table_entry >> 8) & 255); + if (v_table_entry < 16) { + self->private_impl.f_code_lengths[v_i] = ((uint8_t)(v_table_entry)); v_i += 1; goto label_0_continue; } v_n_extra_bits = 0; v_rep_symbol = 0; v_rep_count = 0; - if (v_symbol == 16) { + if (v_table_entry == 16) { v_n_extra_bits = 2; if (v_i <= 0) { status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_REPETITION; @@ -920,14 +871,18 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, } v_rep_symbol = self->private_impl.f_code_lengths[v_i - 1]; v_rep_count = 3; - } else if (v_symbol == 17) { + } else if (v_table_entry == 17) { v_n_extra_bits = 3; v_rep_symbol = 0; v_rep_count = 3; - } else { + } else if (v_table_entry == 18) { v_n_extra_bits = 7; v_rep_symbol = 0; v_rep_count = 11; + } else { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; } while (v_n_bits < v_n_extra_bits) { PUFFS_COROUTINE_SUSPENSION_POINT(5); @@ -974,12 +929,10 @@ suspend: self->private_impl.c_init_huffs[0].v_n_dist = v_n_dist; self->private_impl.c_init_huffs[0].v_n_clen = v_n_clen; self->private_impl.c_init_huffs[0].v_i = v_i; - self->private_impl.c_init_huffs[0].v_symbol = v_symbol; - self->private_impl.c_init_huffs[0].v_code = v_code; - self->private_impl.c_init_huffs[0].v_count = v_count; - self->private_impl.c_init_huffs[0].v_first = v_first; - self->private_impl.c_init_huffs[0].v_index = v_index; - self->private_impl.c_init_huffs[0].v_length = v_length; + self->private_impl.c_init_huffs[0].v_mask = v_mask; + self->private_impl.c_init_huffs[0].v_table_entry = v_table_entry; + self->private_impl.c_init_huffs[0].v_table_entry_n_bits = + v_table_entry_n_bits; self->private_impl.c_init_huffs[0].v_n_extra_bits = v_n_extra_bits; self->private_impl.c_init_huffs[0].v_rep_symbol = v_rep_symbol; self->private_impl.c_init_huffs[0].v_rep_count = v_rep_count; @@ -1161,6 +1114,7 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_prev_cl = v_cl; v_i += 1; } + self->private_impl.f_n_huffs_bits[a_which] = v_max_cl; self->private_impl.c_init_huff[0].coro_susp_point = 0; goto exit; } diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index 8bf4626c..aa8452d5 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -172,12 +172,9 @@ typedef struct { uint32_t v_n_dist; uint32_t v_n_clen; uint32_t v_i; - uint32_t v_symbol; - uint32_t v_code; - uint32_t v_count; - uint32_t v_first; - uint32_t v_index; - uint32_t v_length; + uint32_t v_mask; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; uint32_t v_n_extra_bits; uint8_t v_rep_symbol; uint32_t v_rep_count; diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index fe72a652..5fdf14e9 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -165,6 +165,7 @@ pri func decoder.init_huffs?(src reader1)() { this.init_huff?(which:0, n_codes:19) // Decode the code lengths for the next two Huffman tables. + var mask u32[..511] = ((1 as u32) << this.n_huffs_bits[0]) - 1 i = 0 while i < (n_lit + n_dist), inv n_dist <= 30, @@ -173,108 +174,32 @@ pri func decoder.init_huffs?(src reader1)() { assert i < (286 + 30) via "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"(b0:n_lit, c0:n_dist) // Decode a clcode symbol from H-CL. - var symbol u32[..289] - var code u32 - var count u32 - var first u32 - var index u32 - var length u32 = 1 + var table_entry u32 while true, - pre first <= 0xFFFF, - pre index <= 0xFFFF, inv i < 316, inv n_dist <= 30, inv n_lit <= 286, { - if length > 15 { - return error "bad Huffman code" - } - - // Read the next bit. - while n_bits <= 0, - inv first <= 0xFFFF, - inv i < 316, - inv index <= 0xFFFF, - inv length <= 15, - inv n_dist <= 30, - inv n_lit <= 286, - post n_bits > 0, - { - bits |= (in.src.read_u8?() as u32) << n_bits - n_bits += 8 - } - code |= bits & 1 - // TODO: improve the proof engine's bounds checking so that any - // check returning "internal error: inconsistent Huffman decoder - // state" can be removed. There is one here and more below. - if code > 0xFFFF { - return error "internal error: inconsistent Huffman decoder state" - } - bits >>= 1 - n_bits -= 1 - - count = this.counts[length] as u32 - if count > 0xFFFF { - return error "internal error: inconsistent Huffman decoder state" - } - - // Compare the code so far to the counts. - // - // For the clcode example in this package's README.md: - // - counts[1] = 0 - // - counts[2] = 0 - // - counts[3] = 7 - // - counts[4] = 1 - // - counts[5] = 2 - // - // Recall that the map from bits to clcode symbols is: - // bits clcode - // 0b000 0 - // 0b001 4 - // 0b010 5 - // 0b011 6 - // 0b100 7 - // 0b101 8 - // 0b110 9 - // 0b1110 17 - // 0b11110 3 - // 0b11111 18 - // - // The loop iterations at this point have: - // - length=1, count=0, first= 0, index=0, code in [0.. 1]. - // - length=2, count=0, first= 0, index=0, code in [0.. 3]. - // - length=3, count=7, first= 0, index=0, code in [0.. 7]. - // - length=4, count=1, first=14, index=7, code in [0..15]. - // - length=5, count=2, first=30, index=8, code in [0..31]. - if code < (first + count) { - index += code - if index < first { - return error "internal error: inconsistent Huffman decoder state" - } - index -= first - if index >= 290 { - return error "internal error: inconsistent Huffman decoder state" - } - symbol = this.symbols[index] as u32 + table_entry = this.huffs[0][bits & mask] + var table_entry_n_bits u32[..15] = table_entry & 15 + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits break } - - index += count - if index > 0xFFFF { - return error "internal error: inconsistent Huffman decoder state" - } - first = (first + count) << 1 - if first > 0xFFFF { - return error "internal error: inconsistent Huffman decoder state" - } - code <<= 1 - - length += 1 + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 } + // For H-CL, there are no redirections and all symbols are literals. + if (table_entry >> 30) != 1 { + return error "internal error: inconsistent Huffman decoder state" + } + table_entry = (table_entry >> 8) & 0xFF // Write a literal code length. - if symbol < 16 { - this.code_lengths[i] = symbol as u8 + if table_entry < 16 { + this.code_lengths[i] = table_entry as u8 i += 1 continue } @@ -283,7 +208,7 @@ pri func decoder.init_huffs?(src reader1)() { var n_extra_bits u32[..7] var rep_symbol u8[..15] var rep_count u32 - if symbol == 16 { + if table_entry == 16 { n_extra_bits = 2 if i <= 0 { return error "bad Huffman code length repetition" @@ -291,16 +216,18 @@ pri func decoder.init_huffs?(src reader1)() { rep_symbol = this.code_lengths[i - 1] rep_count = 3 assert rep_count <= 11 - } else if symbol == 17 { + } else if table_entry == 17 { n_extra_bits = 3 rep_symbol = 0 rep_count = 3 assert rep_count <= 11 - } else { + } else if table_entry == 18 { n_extra_bits = 7 rep_symbol = 0 rep_count = 11 assert rep_count <= 11 + } else { + return error "internal error: inconsistent Huffman decoder state" } while n_bits < n_extra_bits, inv i < 316, @@ -524,6 +451,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() { prev_cl = cl i += 1 } + this.n_huffs_bits[in.which] = max_cl // For the clcode example in this package's README.md: // - huffs[0][0b..000] maps to value 0x40000003 (literal, symbols[0]=0x00, code_length=3)