From 7cf7829c9ef62645ece55cbea709fbd1df9376da Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 29 Jul 2017 16:20:10 +1000 Subject: [PATCH] Implement std/flate Huffman table redirects. --- gen/c/std/flate.c | 322 +++++++++++++++++++++++++++-------- gen/h/std/flate.h | 41 +++-- lang/check/data.go | 150 ++++++++++++++-- lang/check/gen.go | 13 +- std/flate/decode_flate.puffs | 179 +++++++++++++++---- test/c/std/flate.c | 137 ++++++++++++++- 6 files changed, 702 insertions(+), 140 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index ad0d9c83..0fac4664 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -109,22 +109,22 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_COUNT \ -1157040126 // 0xbb08f802 #define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_REPETITION \ - -1157040125 // 0xbb08f803 -#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040124 // 0xbb08f804 -#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040122 // 0xbb08f806 + -1157040125 // 0xbb08f803 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040124 // 0xbb08f804 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_MINIMUM_CODE_LENGTH \ + -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040121 // 0xbb08f807 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040121 // 0xbb08f807 -#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040120 // 0xbb08f808 -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ +#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040119 // 0xbb08f809 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040118 // 0xbb08f80a #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040118 // 0xbb08f80a -#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040117 // 0xbb08f80b -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040116 // 0xbb08f80c -#define PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES \ - -1157040115 // 0xbb08f80d + -1157040117 // 0xbb08f80b +#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040116 // 0xbb08f80c +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040115 // 0xbb08f80d bool puffs_flate_status_is_error(puffs_flate_status s); @@ -149,7 +149,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; - uint32_t f_huffs[2][512]; + uint32_t f_huffs[2][1234]; uint32_t f_n_huffs_bits[2]; uint8_t f_code_lengths[320]; @@ -170,6 +170,8 @@ typedef struct { uint32_t v_table_entry_n_bits; uint32_t v_lmask; uint32_t v_dmask; + uint32_t v_redir_top; + uint32_t v_redir_mask; uint32_t v_length; uint32_t v_distance; } c_decode_huffman[1]; @@ -201,12 +203,21 @@ typedef struct { uint32_t v_n_symbols; uint32_t v_count; uint16_t v_symbols[320]; + uint32_t v_min_cl; uint32_t v_max_cl; + uint32_t v_initial_high_bits; uint32_t v_prev_cl; + uint32_t v_prev_redirect_key; + uint32_t v_top; + uint32_t v_next_top; uint32_t v_code; - uint32_t v_cl; - uint32_t v_reversed_code; + uint32_t v_key; uint32_t v_value; + uint32_t v_cl; + uint32_t v_tmp; + uint32_t v_redirect_key; + uint32_t v_j; + uint32_t v_reversed_key; uint32_t v_symbol; uint32_t v_high_bits; uint32_t v_delta; @@ -302,6 +313,7 @@ const char* puffs_flate_status_strings1[14] = { "flate: bad Huffman code length count", "flate: bad Huffman code length repetition", "flate: bad Huffman code", + "flate: bad Huffman minimum code length", "flate: bad distance code count", "flate: bad flate block", "flate: bad literal/length code count", @@ -310,7 +322,6 @@ const char* puffs_flate_status_strings1[14] = { "flate: internal error: inconsistent n_bits", "flate: missing end-of-block code", "flate: no Huffman codes", - "flate: TODO: indirect Huffman tables", }; const char* puffs_flate_status_string(puffs_flate_status s) { @@ -777,6 +788,8 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( uint32_t v_table_entry_n_bits; uint32_t v_lmask; uint32_t v_dmask; + uint32_t v_redir_top; + uint32_t v_redir_mask; uint32_t v_length; uint32_t v_distance; @@ -817,6 +830,8 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( self->private_impl.c_decode_huffman[0].v_table_entry_n_bits; v_lmask = self->private_impl.c_decode_huffman[0].v_lmask; v_dmask = self->private_impl.c_decode_huffman[0].v_dmask; + v_redir_top = self->private_impl.c_decode_huffman[0].v_redir_top; + v_redir_mask = self->private_impl.c_decode_huffman[0].v_redir_mask; v_length = self->private_impl.c_decode_huffman[0].v_length; v_distance = self->private_impl.c_decode_huffman[0].v_distance; } @@ -849,11 +864,40 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( } label_1_break:; if ((v_table_entry >> 31) != 0) { - status = PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES; - goto exit; + v_redir_top = ((v_table_entry >> 8) & 65535); + v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1); + while (true) { + if ((v_redir_top + (v_bits & v_redir_mask)) >= 1234) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_table_entry = + self->private_impl + .f_huffs[0][v_redir_top + (v_bits & v_redir_mask)]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_2_break; + } + PUFFS_COROUTINE_SUSPENSION_POINT(2); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_1 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_1)) << v_n_bits); + v_n_bits += 8; + } + label_2_break:; + if ((v_table_entry >> 31) != 0) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } } if ((v_table_entry >> 30) != 0) { - PUFFS_COROUTINE_SUSPENSION_POINT(2); + PUFFS_COROUTINE_SUSPENSION_POINT(3); if (b_wptr_dst == b_wend_dst) { status = PUFFS_FLATE_SUSPENSION_SHORT_WRITE; goto suspend; @@ -879,12 +923,12 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( } v_table_entry_n_bits = ((v_table_entry >> 4) & 15); while (v_n_bits < v_table_entry_n_bits) { - PUFFS_COROUTINE_SUSPENSION_POINT(3); + PUFFS_COROUTINE_SUSPENSION_POINT(4); if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { goto short_read_src; } - uint8_t t_1 = *b_rptr_src++; - v_bits |= (((uint32_t)(t_1)) << v_n_bits); + uint8_t t_2 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_2)) << v_n_bits); v_n_bits += 8; } v_length += ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1)); @@ -896,20 +940,49 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( if (v_n_bits >= v_table_entry_n_bits) { v_bits >>= v_table_entry_n_bits; v_n_bits -= v_table_entry_n_bits; - goto label_2_break; + goto label_3_break; } - PUFFS_COROUTINE_SUSPENSION_POINT(4); + PUFFS_COROUTINE_SUSPENSION_POINT(5); if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { goto short_read_src; } - uint8_t t_2 = *b_rptr_src++; - v_bits |= (((uint32_t)(t_2)) << v_n_bits); + uint8_t t_3 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_3)) << v_n_bits); v_n_bits += 8; } - label_2_break:; + label_3_break:; if ((v_table_entry >> 31) != 0) { - status = PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES; - goto exit; + v_redir_top = ((v_table_entry >> 8) & 65535); + v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1); + while (true) { + if ((v_redir_top + (v_bits & v_redir_mask)) >= 1234) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_table_entry = + self->private_impl + .f_huffs[1][v_redir_top + (v_bits & v_redir_mask)]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_4_break; + } + PUFFS_COROUTINE_SUSPENSION_POINT(6); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_4 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_4)) << v_n_bits); + v_n_bits += 8; + } + label_4_break:; + if ((v_table_entry >> 31) != 0) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } } if ((v_table_entry >> 24) != 32) { if ((v_table_entry >> 24) == 8) { @@ -928,39 +1001,39 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( } v_table_entry_n_bits = ((v_table_entry >> 4) & 15); while (v_n_bits < v_table_entry_n_bits) { - PUFFS_COROUTINE_SUSPENSION_POINT(5); + PUFFS_COROUTINE_SUSPENSION_POINT(7); if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { goto short_read_src; } - uint8_t t_3 = *b_rptr_src++; - v_bits |= (((uint32_t)(t_3)) << v_n_bits); + uint8_t t_5 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_5)) << v_n_bits); v_n_bits += 8; } v_distance += ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1)); v_bits >>= v_table_entry_n_bits; v_n_bits -= v_table_entry_n_bits; - PUFFS_COROUTINE_SUSPENSION_POINT(6); + PUFFS_COROUTINE_SUSPENSION_POINT(8); { self->private_impl.scratch = ((uint64_t)(v_distance) << 32) | (uint64_t)(v_length); - PUFFS_COROUTINE_SUSPENSION_POINT(7); - size_t t_4 = (size_t)(self->private_impl.scratch >> 32); - if (PUFFS_UNLIKELY((t_4 == 0) || - (t_4 > (b_wptr_dst - a_dst.buf->ptr)))) { + PUFFS_COROUTINE_SUSPENSION_POINT(9); + size_t t_6 = (size_t)(self->private_impl.scratch >> 32); + if (PUFFS_UNLIKELY((t_6 == 0) || + (t_6 > (b_wptr_dst - a_dst.buf->ptr)))) { status = PUFFS_FLATE_ERROR_BAD_ARGUMENT; goto exit; } - uint8_t* t_5 = b_wptr_dst - t_4; - uint32_t t_6 = (uint32_t)(self->private_impl.scratch); - if (PUFFS_LIKELY((size_t)(t_6) <= (b_wend_dst - b_wptr_dst))) { - for (; t_6; t_6--) { - *b_wptr_dst++ = *t_5++; + uint8_t* t_7 = b_wptr_dst - t_6; + uint32_t t_8 = (uint32_t)(self->private_impl.scratch); + if (PUFFS_LIKELY((size_t)(t_8) <= (b_wend_dst - b_wptr_dst))) { + for (; t_8; t_8--) { + *b_wptr_dst++ = *t_7++; } } else { - t_6 = (uint32_t)(b_wend_dst - b_wptr_dst); - self->private_impl.scratch -= (uint64_t)(t_6); - for (; t_6; t_6--) { - *b_wptr_dst++ = *t_5++; + t_8 = (uint32_t)(b_wend_dst - b_wptr_dst); + self->private_impl.scratch -= (uint64_t)(t_8); + for (; t_8; t_8--) { + *b_wptr_dst++ = *t_7++; } status = PUFFS_FLATE_SUSPENSION_SHORT_WRITE; goto suspend; @@ -984,6 +1057,8 @@ suspend: v_table_entry_n_bits; self->private_impl.c_decode_huffman[0].v_lmask = v_lmask; self->private_impl.c_decode_huffman[0].v_dmask = v_dmask; + self->private_impl.c_decode_huffman[0].v_redir_top = v_redir_top; + self->private_impl.c_decode_huffman[0].v_redir_mask = v_redir_mask; self->private_impl.c_decode_huffman[0].v_length = v_length; self->private_impl.c_decode_huffman[0].v_distance = v_distance; @@ -1348,12 +1423,21 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, uint32_t v_n_symbols; uint32_t v_count; uint16_t v_symbols[320]; + uint32_t v_min_cl; uint32_t v_max_cl; + uint32_t v_initial_high_bits; uint32_t v_prev_cl; + uint32_t v_prev_redirect_key; + uint32_t v_top; + uint32_t v_next_top; uint32_t v_code; - uint32_t v_cl; - uint32_t v_reversed_code; + uint32_t v_key; uint32_t v_value; + uint32_t v_cl; + uint32_t v_tmp; + uint32_t v_redirect_key; + uint32_t v_j; + uint32_t v_reversed_key; uint32_t v_symbol; uint32_t v_high_bits; uint32_t v_delta; @@ -1370,12 +1454,21 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_count = self->private_impl.c_init_huff[0].v_count; memcpy(v_symbols, self->private_impl.c_init_huff[0].v_symbols, sizeof(v_symbols)); + v_min_cl = self->private_impl.c_init_huff[0].v_min_cl; v_max_cl = self->private_impl.c_init_huff[0].v_max_cl; + v_initial_high_bits = self->private_impl.c_init_huff[0].v_initial_high_bits; v_prev_cl = self->private_impl.c_init_huff[0].v_prev_cl; + v_prev_redirect_key = self->private_impl.c_init_huff[0].v_prev_redirect_key; + v_top = self->private_impl.c_init_huff[0].v_top; + v_next_top = self->private_impl.c_init_huff[0].v_next_top; v_code = self->private_impl.c_init_huff[0].v_code; - v_cl = self->private_impl.c_init_huff[0].v_cl; - v_reversed_code = self->private_impl.c_init_huff[0].v_reversed_code; + v_key = self->private_impl.c_init_huff[0].v_key; v_value = self->private_impl.c_init_huff[0].v_value; + v_cl = self->private_impl.c_init_huff[0].v_cl; + v_tmp = self->private_impl.c_init_huff[0].v_tmp; + v_redirect_key = self->private_impl.c_init_huff[0].v_redirect_key; + v_j = self->private_impl.c_init_huff[0].v_j; + v_reversed_key = self->private_impl.c_init_huff[0].v_reversed_key; v_symbol = self->private_impl.c_init_huff[0].v_symbol; v_high_bits = self->private_impl.c_init_huff[0].v_high_bits; v_delta = self->private_impl.c_init_huff[0].v_delta; @@ -1457,10 +1550,22 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, } v_i += 1; } + v_min_cl = 1; + while (true) { + if (v_counts[v_min_cl] != 0) { + goto label_0_break; + } + if (v_min_cl >= 9) { + status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_MINIMUM_CODE_LENGTH; + goto exit; + } + v_min_cl += 1; + } + label_0_break:; v_max_cl = 15; while (true) { if (v_counts[v_max_cl] != 0) { - goto label_0_break; + goto label_1_break; } if (v_max_cl <= 1) { status = PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES; @@ -1468,10 +1573,11 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, } v_max_cl -= 1; } - label_0_break:; - if (v_max_cl > 9) { - status = PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES; - goto exit; + label_1_break:; + if (v_max_cl <= 9) { + self->private_impl.f_n_huffs_bits[a_which] = v_max_cl; + } else { + self->private_impl.f_n_huffs_bits[a_which] = 9; } v_i = 0; if ((v_n_symbols != ((uint32_t)(v_offsets[v_max_cl]))) || @@ -1485,10 +1591,19 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } + v_initial_high_bits = 512; + if (v_max_cl < 9) { + v_initial_high_bits = (((uint32_t)(1)) << v_max_cl); + } v_prev_cl = ((uint32_t)( self->private_impl .f_code_lengths[a_n_codes0 + ((uint32_t)(v_symbols[0]))])); + v_prev_redirect_key = 4294967295; + v_top = 0; + v_next_top = 512; v_code = 0; + v_key = 0; + v_value = 0; while (true) { if ((a_n_codes0 + ((uint32_t)(v_symbols[v_i]))) >= 320) { status = @@ -1498,23 +1613,69 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_cl = ((uint32_t)( self->private_impl .f_code_lengths[a_n_codes0 + ((uint32_t)(v_symbols[v_i]))])); - if (v_cl > v_max_cl) { - status = - PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; - goto exit; - } if (v_cl > v_prev_cl) { v_code <<= (v_cl - v_prev_cl); - if (v_code >= 512) { + if (v_code >= 32768) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } } - v_reversed_code = (((uint32_t)(puffs_flate_reverse8[v_code >> 1])) | - ((v_code & 1) << 8)); - v_reversed_code >>= (9 - v_cl); - v_value = 0; + v_prev_cl = v_cl; + v_key = v_code; + if (v_cl > 9) { + v_tmp = (v_cl - 9); + v_cl = v_tmp; + v_redirect_key = ((v_key >> v_tmp) & 511); + v_key = ((v_key) & ((1 << (v_tmp)) - 1)); + if (v_prev_redirect_key != v_redirect_key) { + v_prev_redirect_key = v_redirect_key; + v_remaining = (((uint32_t)(1)) << v_cl); + v_j = v_prev_cl; + while (v_j <= 15) { + if (v_remaining <= ((uint32_t)(v_counts[v_j]))) { + goto label_2_break; + } + v_remaining -= ((uint32_t)(v_counts[v_j])); + if (v_remaining > 1073741824) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_remaining <<= 1; + v_j += 1; + } + label_2_break:; + if ((v_j <= 9) || (15 < v_j)) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_tmp = (v_j - 9); + v_initial_high_bits = (((uint32_t)(1)) << v_tmp); + v_top = v_next_top; + if ((v_top + (((uint32_t)(1)) << v_tmp)) > 1234) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_next_top = (v_top + (((uint32_t)(1)) << v_tmp)); + v_redirect_key = + (((uint32_t)(puffs_flate_reverse8[v_redirect_key >> 1])) | + ((v_redirect_key & 1) << 8)); + self->private_impl.f_huffs[a_which][v_redirect_key] = + (2147483657 | (v_top << 8) | (v_tmp << 4)); + } + } + if ((v_key >= 512) || (v_counts[v_prev_cl] <= 0)) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_counts[v_prev_cl] -= 1; + v_reversed_key = + (((uint32_t)(puffs_flate_reverse8[v_key >> 1])) | ((v_key & 1) << 8)); + v_reversed_key >>= (9 - v_cl); v_symbol = ((uint32_t)(v_symbols[v_i])); if (v_symbol == 256) { v_value = (268435456 | v_cl); @@ -1532,27 +1693,31 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } - v_high_bits = (((uint32_t)(1)) << v_max_cl); + v_high_bits = v_initial_high_bits; v_delta = (((uint32_t)(1)) << v_cl); while (v_high_bits >= v_delta) { v_high_bits -= v_delta; + if ((v_top + ((v_high_bits | v_reversed_key) & 511)) >= 1234) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } self->private_impl - .f_huffs[a_which][(v_high_bits | v_reversed_code) & 511] = v_value; + .f_huffs[a_which][v_top + ((v_high_bits | v_reversed_key) & 511)] = + v_value; } v_i += 1; if (v_i >= v_n_symbols) { - goto label_1_break; + goto label_3_break; } v_code += 1; - if (v_code >= 512) { + if (v_code >= 32768) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } - v_prev_cl = v_cl; } - label_1_break:; - self->private_impl.f_n_huffs_bits[a_which] = v_max_cl; + label_3_break:; self->private_impl.c_init_huff[0].coro_susp_point = 0; goto exit; } @@ -1570,12 +1735,21 @@ suspend: self->private_impl.c_init_huff[0].v_count = v_count; memcpy(self->private_impl.c_init_huff[0].v_symbols, v_symbols, sizeof(v_symbols)); + self->private_impl.c_init_huff[0].v_min_cl = v_min_cl; self->private_impl.c_init_huff[0].v_max_cl = v_max_cl; + self->private_impl.c_init_huff[0].v_initial_high_bits = v_initial_high_bits; self->private_impl.c_init_huff[0].v_prev_cl = v_prev_cl; + self->private_impl.c_init_huff[0].v_prev_redirect_key = v_prev_redirect_key; + self->private_impl.c_init_huff[0].v_top = v_top; + self->private_impl.c_init_huff[0].v_next_top = v_next_top; self->private_impl.c_init_huff[0].v_code = v_code; - self->private_impl.c_init_huff[0].v_cl = v_cl; - self->private_impl.c_init_huff[0].v_reversed_code = v_reversed_code; + self->private_impl.c_init_huff[0].v_key = v_key; self->private_impl.c_init_huff[0].v_value = v_value; + self->private_impl.c_init_huff[0].v_cl = v_cl; + self->private_impl.c_init_huff[0].v_tmp = v_tmp; + self->private_impl.c_init_huff[0].v_redirect_key = v_redirect_key; + self->private_impl.c_init_huff[0].v_j = v_j; + self->private_impl.c_init_huff[0].v_reversed_key = v_reversed_key; self->private_impl.c_init_huff[0].v_symbol = v_symbol; self->private_impl.c_init_huff[0].v_high_bits = v_high_bits; self->private_impl.c_init_huff[0].v_delta = v_delta; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index fef1f851..8c7bb4df 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -109,22 +109,22 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_COUNT \ -1157040126 // 0xbb08f802 #define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE_LENGTH_REPETITION \ - -1157040125 // 0xbb08f803 -#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040124 // 0xbb08f804 -#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040122 // 0xbb08f806 + -1157040125 // 0xbb08f803 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040124 // 0xbb08f804 +#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_MINIMUM_CODE_LENGTH \ + -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040121 // 0xbb08f807 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040121 // 0xbb08f807 -#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040120 // 0xbb08f808 -#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ +#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ -1157040119 // 0xbb08f809 +#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \ + -1157040118 // 0xbb08f80a #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040118 // 0xbb08f80a -#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040117 // 0xbb08f80b -#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040116 // 0xbb08f80c -#define PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES \ - -1157040115 // 0xbb08f80d + -1157040117 // 0xbb08f80b +#define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040116 // 0xbb08f80c +#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040115 // 0xbb08f80d bool puffs_flate_status_is_error(puffs_flate_status s); @@ -149,7 +149,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; - uint32_t f_huffs[2][512]; + uint32_t f_huffs[2][1234]; uint32_t f_n_huffs_bits[2]; uint8_t f_code_lengths[320]; @@ -170,6 +170,8 @@ typedef struct { uint32_t v_table_entry_n_bits; uint32_t v_lmask; uint32_t v_dmask; + uint32_t v_redir_top; + uint32_t v_redir_mask; uint32_t v_length; uint32_t v_distance; } c_decode_huffman[1]; @@ -201,12 +203,21 @@ typedef struct { uint32_t v_n_symbols; uint32_t v_count; uint16_t v_symbols[320]; + uint32_t v_min_cl; uint32_t v_max_cl; + uint32_t v_initial_high_bits; uint32_t v_prev_cl; + uint32_t v_prev_redirect_key; + uint32_t v_top; + uint32_t v_next_top; uint32_t v_code; - uint32_t v_cl; - uint32_t v_reversed_code; + uint32_t v_key; uint32_t v_value; + uint32_t v_cl; + uint32_t v_tmp; + uint32_t v_redirect_key; + uint32_t v_j; + uint32_t v_reversed_key; uint32_t v_symbol; uint32_t v_high_bits; uint32_t v_delta; diff --git a/lang/check/data.go b/lang/check/data.go index 392758b1..d4bf7af9 100644 --- a/lang/check/data.go +++ b/lang/check/data.go @@ -15,38 +15,61 @@ var reasons = [...]struct { r reason }{ - {`"a < (b + c): a < c; 0 <= b"`, func(q *checker, n *a.Assert) error { - op, xa, t0 := parseBinaryOp(n.Condition()) + {`"a < b: a < c; c < b"`, func(q *checker, n *a.Assert) error { + op, xa, xb := parseBinaryOp(n.Condition()) if op.Key() != t.KeyXBinaryLessThan { return errFailed } - op, xb, xc := parseBinaryOp(t0) - if op.Key() != t.KeyXBinaryPlus { + // a < c + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { return errFailed } - // a < c if err := proveReasonRequirement(q, t.IDXBinaryLessThan, xa, xc); err != nil { return err } - // 0 <= b - if err := proveReasonRequirement(q, t.IDXBinaryLessEq, zeroExpr, xb); err != nil { + // c < b + if err := proveReasonRequirement(q, t.IDXBinaryLessThan, xc, xb); err != nil { return err } return nil }}, - {`"(a + b) <= c: a <= (c - b)"`, func(q *checker, n *a.Assert) error { - op, t0, xc := parseBinaryOp(n.Condition()) - if op.Key() != t.KeyXBinaryLessEq { + {`"a < b: a < c; c == b"`, func(q *checker, n *a.Assert) error { + op, xa, xb := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessThan { return errFailed } - op, xa, xb := parseBinaryOp(t0) - if op.Key() != t.KeyXBinaryPlus { + // a < c + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { return errFailed } - // a <= (c - b) - t1 := a.NewExpr(a.FlagsTypeChecked, t.IDXBinaryMinus, 0, xc.Node(), nil, xb.Node(), nil) - if err := proveReasonRequirement(q, t.IDXBinaryLessEq, xa, t1); err != nil { + if err := proveReasonRequirement(q, t.IDXBinaryLessThan, xa, xc); err != nil { + return err + } + // c == b + if err := proveReasonRequirement(q, t.IDXBinaryEqEq, xc, xb); err != nil { + return err + } + return nil + }}, + + {`"a < b: a == c; c < b"`, func(q *checker, n *a.Assert) error { + op, xa, xb := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessThan { + return errFailed + } + // a == c + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { + return errFailed + } + if err := proveReasonRequirement(q, t.IDXBinaryEqEq, xa, xc); err != nil { + return err + } + // c < b + if err := proveReasonRequirement(q, t.IDXBinaryLessThan, xc, xb); err != nil { return err } return nil @@ -72,6 +95,26 @@ var reasons = [...]struct { return nil }}, + {`"a < b: a <= c; c < b"`, func(q *checker, n *a.Assert) error { + op, xa, xb := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessThan { + return errFailed + } + // a <= c + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { + return errFailed + } + if err := proveReasonRequirement(q, t.IDXBinaryLessEq, xa, xc); err != nil { + return err + } + // c < b + if err := proveReasonRequirement(q, t.IDXBinaryLessThan, xc, xb); err != nil { + return err + } + return nil + }}, + {`"a <= b: a <= c; c <= b"`, func(q *checker, n *a.Assert) error { op, xa, xb := parseBinaryOp(n.Condition()) if op.Key() != t.KeyXBinaryLessEq { @@ -92,6 +135,66 @@ var reasons = [...]struct { return nil }}, + {`"a <= b: a <= c; c == b"`, func(q *checker, n *a.Assert) error { + op, xa, xb := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessEq { + return errFailed + } + // a <= c + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { + return errFailed + } + if err := proveReasonRequirement(q, t.IDXBinaryLessEq, xa, xc); err != nil { + return err + } + // c == b + if err := proveReasonRequirement(q, t.IDXBinaryEqEq, xc, xb); err != nil { + return err + } + return nil + }}, + + {`"a <= b: a == c; c <= b"`, func(q *checker, n *a.Assert) error { + op, xa, xb := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessEq { + return errFailed + } + // a == c + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { + return errFailed + } + if err := proveReasonRequirement(q, t.IDXBinaryEqEq, xa, xc); err != nil { + return err + } + // c <= b + if err := proveReasonRequirement(q, t.IDXBinaryLessEq, xc, xb); err != nil { + return err + } + return nil + }}, + + {`"a < (b + c): a < c; 0 <= b"`, func(q *checker, n *a.Assert) error { + op, xa, t0 := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessThan { + return errFailed + } + op, xb, xc := parseBinaryOp(t0) + if op.Key() != t.KeyXBinaryPlus { + return errFailed + } + // a < c + if err := proveReasonRequirement(q, t.IDXBinaryLessThan, xa, xc); err != nil { + return err + } + // 0 <= b + if err := proveReasonRequirement(q, t.IDXBinaryLessEq, zeroExpr, xb); err != nil { + return err + } + return nil + }}, + {`"a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"`, func(q *checker, n *a.Assert) error { op, xa, t0 := parseBinaryOp(n.Condition()) if op.Key() != t.KeyXBinaryLessThan { @@ -124,4 +227,21 @@ var reasons = [...]struct { } return nil }}, + + {`"(a + b) <= c: a <= (c - b)"`, func(q *checker, n *a.Assert) error { + op, t0, xc := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessEq { + return errFailed + } + op, xa, xb := parseBinaryOp(t0) + if op.Key() != t.KeyXBinaryPlus { + return errFailed + } + // a <= (c - b) + t1 := a.NewExpr(a.FlagsTypeChecked, t.IDXBinaryMinus, 0, xc.Node(), nil, xb.Node(), nil) + if err := proveReasonRequirement(q, t.IDXBinaryLessEq, xa, t1); err != nil { + return err + } + return nil + }}, } diff --git a/lang/check/gen.go b/lang/check/gen.go index b4a9dd77..4f30f8e3 100644 --- a/lang/check/gen.go +++ b/lang/check/gen.go @@ -19,11 +19,20 @@ import ( ) var reasons = [...]string{ - "a < (b + c): a < c; 0 <= b", - "(a + b) <= c: a <= (c - b)", + "a < b: a < c; c < b", + "a < b: a < c; c == b", + "a < b: a == c; c < b", "a < b: a < c; c <= b", + "a < b: a <= c; c < b", + "a <= b: a <= c; c <= b", + "a <= b: a <= c; c == b", + "a <= b: a == c; c <= b", + + "a < (b + c): a < c; 0 <= b", "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c", + + "(a + b) <= c: a <= (c - b)", } func main() { diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 2d4be0a4..c567a951 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -10,6 +10,7 @@ pub error "bad Huffman code (under-subscribed)" pub error "bad Huffman code length count" pub error "bad Huffman code length repetition" pub error "bad Huffman code" +pub error "bad Huffman minimum code length" pub error "bad distance code count" pub error "bad flate block" pub error "bad literal/length code count" @@ -19,8 +20,6 @@ pub error "internal error: inconsistent n_bits" pub error "missing end-of-block code" pub error "no Huffman codes" -pub error "TODO: indirect Huffman tables" - // The next two tables were created by script/print-flate-magic-numbers.go. // // The u32 values' meanings are the same as the decoder.huffs u32 values. In @@ -73,12 +72,15 @@ pub struct decoder?( // - bit 28 indicates the symbol is the end-of-block. // - bit 27 indicates an invalid value. // - bits 26-24 are zero. - // - bits 23- 8 is the literal (in bits 15-8) or base number. - // - bits 7- 4 is the redirect offset or the number of extra bits. + // - bits 23- 8 is the redirect offset, literal (in bits 15-8) or base number. + // - bits 7- 4 is the redirected table's bits or number of extra bits. // - bits 3- 0 are the number of decoder.bits to consume. // // Exactly one of the eight bits 31-24 should be set. - huffs[2][512] u32, + // + // TODO: replace the magic "big enough" 1234 with something more + // principled, perhaps discovered via an exhaustive search. + huffs[2][1234] u32, n_huffs_bits[2] u32[..9], // code_lengths is used to pass out-of-band data to init_huff. @@ -152,11 +154,32 @@ pri func decoder.decode_huffman?(dst writer1, src reader1)() { bits |= (in.src.read_u8?() as u32) << n_bits n_bits += 8 } + // Check for a redirect. if (table_entry >> 31) != 0 { - return error "TODO: indirect Huffman tables" + var redir_top u32[..0xFFFF] = (table_entry >> 8) & 0xFFFF + var redir_mask u32[..0x7FFF] = ((1 as u32) << ((table_entry >> 4) & 0x0F)) - 1 + while true { + if (redir_top + (bits & redir_mask)) >= 1234 { + return error "internal error: inconsistent Huffman decoder state" + } + table_entry = this.huffs[0][redir_top + (bits & redir_mask)] + table_entry_n_bits = table_entry & 0x0F + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + if (table_entry >> 31) != 0 { + return error "internal error: inconsistent Huffman decoder state" + } } if (table_entry >> 30) != 0 { + // Literal. in.dst.write_u8?(x:(table_entry >> 8) & 0xFF) continue } else if (table_entry >> 29) != 0 { @@ -202,8 +225,28 @@ pri func decoder.decode_huffman?(dst writer1, src reader1)() { bits |= (in.src.read_u8?() as u32) << n_bits n_bits += 8 } + // Check for a redirect. if (table_entry >> 31) != 0 { - return error "TODO: indirect Huffman tables" + redir_top = (table_entry >> 8) & 0xFFFF + redir_mask = ((1 as u32) << ((table_entry >> 4) & 0x0F)) - 1 + while true { + if (redir_top + (bits & redir_mask)) >= 1234 { + return error "internal error: inconsistent Huffman decoder state" + } + table_entry = this.huffs[1][redir_top + (bits & redir_mask)] + table_entry_n_bits = table_entry & 0x0F + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + if (table_entry >> 31) != 0 { + return error "internal error: inconsistent Huffman decoder state" + } } // For H-D, all symbols should be base number + extra bits. @@ -560,10 +603,23 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..320], n_codes1 u32[.. i += 1 } - // Calculate max_cl. + // Calculate min_cl and max_cl. // // For the clcode example in this package's README.md: + // - min_cl = 3 // - max_cl = 5 + var min_cl u32[1..9] = 1 + while true, + inv n_symbols <= 288, + { + if counts[min_cl] != 0 { + break + } + if min_cl >= 9 { + return error "bad Huffman minimum code length" + } + min_cl += 1 + } var max_cl u32[1..15] = 15 while true, inv n_symbols <= 288, @@ -577,8 +633,10 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..320], n_codes1 u32[.. } max_cl -= 1 } - if max_cl > 9 { - return error "TODO: indirect Huffman tables" + if max_cl <= 9 { + this.n_huffs_bits[in.which] = max_cl + } else { + this.n_huffs_bits[in.which] = 9 } // Calculate this.huffs[in.which]. @@ -601,33 +659,94 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..320], n_codes1 u32[.. if (in.n_codes0 + (symbols[0] as u32)) >= 320 { return error "internal error: inconsistent Huffman decoder state" } + + var initial_high_bits u32 = 1 << 9 + if max_cl < 9 { + initial_high_bits = (1 as u32) << max_cl + } var prev_cl u32[..15] = this.code_lengths[in.n_codes0 + (symbols[0] as u32)] as u32 + var prev_redirect_key u32 = 0xFFFFFFFF + var top u32[..1234] + var next_top u32[..1234] = 512 var code u32 + var key u32 + var value u32 while true, - pre code < (1 << 9), + pre code < (1 << 15), pre i < 288, - inv max_cl <= 9, inv n_symbols <= 288, { if (in.n_codes0 + (symbols[i] as u32)) >= 320 { return error "internal error: inconsistent Huffman decoder state" } var cl u32[..15] = this.code_lengths[in.n_codes0 + (symbols[i] as u32)] as u32 - if cl > max_cl { - return error "internal error: inconsistent Huffman decoder state" - } - assert cl <= 9 via "a <= b: a <= c; c <= b"(c:max_cl) if cl > prev_cl { code <<= cl - prev_cl - if code >= (1 << 9) { + if code >= (1 << 15) { return error "internal error: inconsistent Huffman decoder state" } } + // For the remainder of this loop body, prev_cl is the original code + // length, cl is possibly clipped by 9, if in the 2nd-level table. + prev_cl = cl - var reversed_code u32[..511] = (reverse8[code >> 1] as u32) | ((code & 1) << 8) - reversed_code >>= 9 - cl + key = code + if cl > 9 { + // TODO: we shouldn't need a temporary variable. + var tmp u32[..6] = cl - 9 + cl = tmp + assert cl <= 9 via "a <= b: a == c; c <= b"(c:tmp) + + var redirect_key u32[..511] = (key >> tmp) & 511 + key = key.low_bits(n:tmp) + if prev_redirect_key != redirect_key { + prev_redirect_key = redirect_key + + // Calculate the number of bits needed for the 2nd level table. + // This computation is similar to "check that the Huffman code + // completely covers all possible input bits" above. + remaining = (1 as u32) << cl + var j u32 = prev_cl + while j <= 15, + inv cl <= 9, + inv code < (1 << 15), + inv i < 288, + inv n_symbols <= 288, + { + if remaining <= (counts[j] as u32) { + break + } + remaining -= counts[j] as u32 + if remaining > (1 << 30) { + return error "internal error: inconsistent Huffman decoder state" + } + remaining <<= 1 + j += 1 + } + if (j <= 9) or (15 < j) { + return error "internal error: inconsistent Huffman decoder state" + } + tmp = j - 9 + initial_high_bits = (1 as u32) << tmp + + top = next_top + if (top + ((1 as u32) << tmp)) > 1234 { + return error "internal error: inconsistent Huffman decoder state" + } + next_top = top + ((1 as u32) << tmp) + + redirect_key = (reverse8[redirect_key >> 1] as u32) | ((redirect_key & 1) << 8) + this.huffs[in.which][redirect_key] = 0x80000009 | (top << 8) | (tmp << 4) + } + } + if (key >= (1 << 9)) or (counts[prev_cl] <= 0) { + return error "internal error: inconsistent Huffman decoder state" + } + counts[prev_cl] -= 1 + + var reversed_key u32[..511] = (reverse8[key >> 1] as u32) | ((key & 1) << 8) + reversed_key >>= 9 - cl - var value u32 var symbol u32[..319] = symbols[i] as u32 if symbol == 256 { // End-of-block. @@ -647,19 +766,21 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..320], n_codes1 u32[.. return error "internal error: inconsistent Huffman decoder state" } - // The table uses max_cl bits, but reversed_code only has cl bits. We - // duplicate the key-value pair across all possible values of the high - // (max_cl - cl) bits. - var high_bits u32 = (1 as u32) << max_cl + // The table uses log2(initial_high_bits) bits, but reversed_key only + // has cl bits. We duplicate the key-value pair across all possible + // values of the high (log2(initial_high_bits) - cl) bits. + var high_bits u32 = initial_high_bits var delta u32 = (1 as u32) << cl while high_bits >= delta, - inv code < (1 << 9), + inv code < (1 << 15), inv i < 288, - inv max_cl <= 9, inv n_symbols <= 288, { high_bits -= delta - this.huffs[in.which][(high_bits | reversed_code) & 511] = value + if (top + ((high_bits | reversed_key) & 511)) >= 1234 { + return error "internal error: inconsistent Huffman decoder state" + } + this.huffs[in.which][top + ((high_bits | reversed_key) & 511)] = value } i += 1 @@ -668,10 +789,8 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..320], n_codes1 u32[.. } assert i < 288 via "a < b: a < c; c <= b"(c:n_symbols) code += 1 - if code >= (1 << 9) { + if code >= (1 << 15) { return error "internal error: inconsistent Huffman decoder state" } - prev_cl = cl } - this.n_huffs_bits[in.which] = max_cl } diff --git a/test/c/std/flate.c b/test/c/std/flate.c index a77ec601..263f8dd0 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -207,6 +207,138 @@ void test_puffs_flate_decode_split_src() { } } +void test_puffs_flate_table_redirect() { + proc_funcname = __func__; + + // Call init_huff with a Huffman code that looks like: + // + // code_bits cl c r s 1st 2nd + // 0b_______________0 1 1 1 0 0b........0 + // 0b______________10 2 1 1 1 0b.......01 + // 0b_____________110 3 1 1 2 0b......011 + // 0b____________1110 4 1 1 3 0b.....0111 + // 0b__________1_1110 5 1 1 4 0b....01111 + // 0b_________11_1110 6 1 1 5 0b...011111 + // 0b________111_1110 7 1 1 6 0b..0111111 + // 8 0 2 + // 0b_____1_1111_1100 9 1 3 7 0b001111111 + // 0b____11_1111_1010 10 1 5 8 0b101111111 0b..0 (3 bits) + // 11 0 10 + // 0b__1111_1110_1100 12 19 19 9 0b101111111 0b001 + // 0b__1111_1110_1101 12 18 10 0b101111111 0b101 + // 0b__1111_1110_1110 12 17 11 0b101111111 0b011 + // 0b__1111_1110_1111 12 16 12 0b101111111 0b111 + // 0b__1111_1111_0000 12 15 13 0b011111111 0b000 (3 bits) + // 0b__1111_1111_0001 12 14 14 0b011111111 0b100 + // 0b__1111_1111_0010 12 13 15 0b011111111 0b010 + // 0b__1111_1111_0011 12 12 16 0b011111111 0b110 + // 0b__1111_1111_0100 12 11 17 0b011111111 0b001 + // 0b__1111_1111_0101 12 10 18 0b011111111 0b101 + // 0b__1111_1111_0110 12 9 19 0b011111111 0b011 + // 0b__1111_1111_0111 12 8 20 0b011111111 0b111 + // 0b__1111_1111_1000 12 7 21 0b111111111 0b.000 (4 bits) + // 0b__1111_1111_1001 12 6 22 0b111111111 0b.100 + // 0b__1111_1111_1010 12 5 23 0b111111111 0b.010 + // 0b__1111_1111_1011 12 4 24 0b111111111 0b.110 + // 0b__1111_1111_1100 12 3 25 0b111111111 0b.001 + // 0b__1111_1111_1101 12 2 26 0b111111111 0b.101 + // 0b__1111_1111_1110 12 1 27 0b111111111 0b.011 + // 0b1_1111_1111_1110 13 2 1 28 0b111111111 0b0111 + // 0b1_1111_1111_1111 13 0 29 0b111111111 0b1111 + // + // cl is the code_length. + // c is counts[code_length] + // r is the number of codes (of that code_length) remaining. + // s is the symbol + // 1st is the key in the first level table (9 bits). + // 2nd is the key in the second level table (variable bits). + + puffs_flate_decoder dec; + puffs_flate_decoder_constructor(&dec, PUFFS_VERSION, 0); + + // The constructor should zero out dec's fields, but to be paranoid, we zero + // it out explicitly. + memset(&dec.private_impl.f_huffs[0], 0, sizeof(dec.private_impl.f_huffs[0])); + + int i; + int n = 0; + dec.private_impl.f_code_lengths[n++] = 1; + dec.private_impl.f_code_lengths[n++] = 2; + dec.private_impl.f_code_lengths[n++] = 3; + dec.private_impl.f_code_lengths[n++] = 4; + dec.private_impl.f_code_lengths[n++] = 5; + dec.private_impl.f_code_lengths[n++] = 6; + dec.private_impl.f_code_lengths[n++] = 7; + dec.private_impl.f_code_lengths[n++] = 9; + dec.private_impl.f_code_lengths[n++] = 10; + for (i = 0; i < 19; i++) { + dec.private_impl.f_code_lengths[n++] = 12; + } + dec.private_impl.f_code_lengths[n++] = 13; + dec.private_impl.f_code_lengths[n++] = 13; + + puffs_flate_status s = puffs_flate_decoder_init_huff(&dec, 0, 0, n, 257); + puffs_flate_decoder_destructor(&dec); + if (s) { + FAIL("%s", puffs_flate_status_string(s)); + return; + } + + // There is one 1st-level table (9 bits), and three 2nd-level tables (3, 3 + // and 4 bits). f_huffs[0]'s elements should be non-zero for those tables and + // should be zero outside of those tables. + const int n_f_huffs = sizeof(dec.private_impl.f_huffs[0]) / + sizeof(dec.private_impl.f_huffs[0][0]); + for (i = 0; i < n_f_huffs; i++) { + bool got = dec.private_impl.f_huffs[0][i] == 0; + bool want = i >= (1 << 9) + (1 << 3) + (1 << 3) + (1 << 4); + if (got != want) { + FAIL("huffs[0][%d] == 0: got %d, want %d", i, got, want); + return; + } + } + + // The redirects in the 1st-level table should be at: + // - 0b101111111 (0x017F) to the table offset 512 (0x0200), a 3-bit table. + // - 0b011111111 (0x00FF) to the table offset 520 (0x0208), a 3-bit table. + // - 0b111111111 (0x01FF) to the table offset 528 (0x0210), a 4-bit table. + uint32_t got; + uint32_t want; + got = dec.private_impl.f_huffs[0][0x017F]; + want = 0x80020039; + if (got != want) { + FAIL("huffs[0][0x017F]: got 0x%08" PRIX32 ", want 0x%08" PRIX32, got, want); + return; + } + got = dec.private_impl.f_huffs[0][0x00FF]; + want = 0x80020839; + if (got != want) { + FAIL("huffs[0][0x00FF]: got 0x%08" PRIX32 ", want 0x%08" PRIX32, got, want); + return; + } + got = dec.private_impl.f_huffs[0][0x01FF]; + want = 0x80021049; + if (got != want) { + FAIL("huffs[0][0x01FF]: got 0x%08" PRIX32 ", want 0x%08" PRIX32, got, want); + return; + } + + // The first 2nd-level table should look like wants. + const uint32_t wants[8] = { + 0x40000801, 0x40000903, 0x40000801, 0x40000B03, + 0x40000801, 0x40000A03, 0x40000801, 0x40000C03, + }; + for (i = 0; i < 8; i++) { + got = dec.private_impl.f_huffs[0][0x0200 + i]; + want = wants[i]; + if (got != want) { + FAIL("huffs[0][0x%04" PRIX32 "]: got 0x%08" PRIX32 ", want 0x%08" PRIX32, + (uint32_t)(0x0200 + i), got, want); + return; + } + } +} + // ---------------- Mimic Tests #ifdef PUFFS_MIMIC @@ -344,13 +476,12 @@ void bench_mimic_zlib_decode_100k() { proc tests[] = { // Flate Tests test_puffs_flate_decode_256_bytes, // - /* TODO: implement puffs_flate, then uncomment these. test_puffs_flate_decode_midsummer, // test_puffs_flate_decode_pi, // - */ test_puffs_flate_decode_romeo, // test_puffs_flate_decode_romeo_fixed, // test_puffs_flate_decode_split_src, // + test_puffs_flate_table_redirect, // #ifdef PUFFS_MIMIC @@ -374,10 +505,8 @@ proc tests[] = { proc benches[] = { // Flate Benches bench_puffs_flate_decode_1k, // - /* TODO: implement puffs_flate, then uncomment these. bench_puffs_flate_decode_10k, // bench_puffs_flate_decode_100k, // - */ #ifdef PUFFS_MIMIC