From d7cb39b76657d30a24d57b0c7e57cf730b5a4060 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 28 Jul 2017 09:57:39 +1000 Subject: [PATCH] Decode base+extra length and distance pairs. --- gen/c/std/flate.c | 118 +++++++++++++++++++++++++++++++---- gen/h/std/flate.h | 5 +- std/flate/decode_flate.puffs | 89 +++++++++++++++++++++++--- 3 files changed, 190 insertions(+), 22 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 9e090b34..5bc756bb 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -165,9 +165,12 @@ typedef struct { uint32_t coro_susp_point; uint32_t v_bits; uint32_t v_n_bits; - uint32_t v_lmask; uint32_t v_table_entry; uint32_t v_table_entry_n_bits; + uint32_t v_lmask; + uint32_t v_dmask; + uint32_t v_length; + uint32_t v_distance; } c_decode_huffman[1]; struct { uint32_t coro_susp_point; @@ -753,9 +756,12 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( uint32_t v_bits; uint32_t v_n_bits; - uint32_t v_lmask; uint32_t v_table_entry; uint32_t v_table_entry_n_bits; + uint32_t v_lmask; + uint32_t v_dmask; + uint32_t v_length; + uint32_t v_distance; uint8_t* b_wptr_dst = NULL; uint8_t* b_wend_dst = NULL; @@ -789,20 +795,25 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( if (coro_susp_point) { v_bits = self->private_impl.c_decode_huffman[0].v_bits; v_n_bits = self->private_impl.c_decode_huffman[0].v_n_bits; - v_lmask = self->private_impl.c_decode_huffman[0].v_lmask; v_table_entry = self->private_impl.c_decode_huffman[0].v_table_entry; v_table_entry_n_bits = self->private_impl.c_decode_huffman[0].v_table_entry_n_bits; + v_lmask = self->private_impl.c_decode_huffman[0].v_lmask; + v_dmask = self->private_impl.c_decode_huffman[0].v_dmask; + v_length = self->private_impl.c_decode_huffman[0].v_length; + v_distance = self->private_impl.c_decode_huffman[0].v_distance; } switch (coro_susp_point) { PUFFS_COROUTINE_SUSPENSION_POINT(0); v_bits = self->private_impl.f_bits; v_n_bits = self->private_impl.f_n_bits; + v_table_entry = 0; + v_table_entry_n_bits = 0; v_lmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[0]) - 1); + v_dmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[1]) - 1); label_0_continue:; while (true) { - v_table_entry = 0; while (true) { v_table_entry = self->private_impl.f_huffs[0][v_bits & v_lmask]; v_table_entry_n_bits = (v_table_entry & 15); @@ -835,7 +846,7 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( } else if ((v_table_entry >> 29) != 0) { } else if ((v_table_entry >> 28) != 0) { goto label_0_break; - } else if ((v_table_entry >> 27) != 0) { + } else if ((v_table_entry >> 24) == 8) { status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE; goto exit; } else { @@ -843,7 +854,74 @@ puffs_flate_status puffs_flate_decoder_decode_huffman( PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } - goto label_0_break; + v_length = ((v_table_entry >> 8) & 65535); + if (v_length > 65535) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_table_entry_n_bits = ((v_table_entry >> 4) & 15); + while (v_n_bits < v_table_entry_n_bits) { + PUFFS_COROUTINE_SUSPENSION_POINT(3); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_1 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_1)) << v_n_bits); + v_n_bits += 8; + } + v_length += ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1)); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + while (true) { + v_table_entry = self->private_impl.f_huffs[1][v_bits & v_dmask]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_2_break; + } + PUFFS_COROUTINE_SUSPENSION_POINT(4); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_2 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_2)) << v_n_bits); + v_n_bits += 8; + } + label_2_break:; + if ((v_table_entry >> 31) != 0) { + status = PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES; + goto exit; + } + if ((v_table_entry >> 24) != 32) { + if ((v_table_entry >> 24) == 8) { + status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE; + goto exit; + } + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_distance = ((v_table_entry >> 8) & 65535); + if (v_distance > 65535) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_table_entry_n_bits = ((v_table_entry >> 4) & 15); + while (v_n_bits < v_table_entry_n_bits) { + PUFFS_COROUTINE_SUSPENSION_POINT(5); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_3 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_3)) << v_n_bits); + v_n_bits += 8; + } + v_distance += ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1)); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; } label_0_break:; self->private_impl.f_bits = v_bits; @@ -857,10 +935,13 @@ suspend: self->private_impl.c_decode_huffman[0].coro_susp_point = coro_susp_point; self->private_impl.c_decode_huffman[0].v_bits = v_bits; self->private_impl.c_decode_huffman[0].v_n_bits = v_n_bits; - self->private_impl.c_decode_huffman[0].v_lmask = v_lmask; self->private_impl.c_decode_huffman[0].v_table_entry = v_table_entry; self->private_impl.c_decode_huffman[0].v_table_entry_n_bits = v_table_entry_n_bits; + self->private_impl.c_decode_huffman[0].v_lmask = v_lmask; + self->private_impl.c_decode_huffman[0].v_dmask = v_dmask; + self->private_impl.c_decode_huffman[0].v_length = v_length; + self->private_impl.c_decode_huffman[0].v_distance = v_distance; exit: if (a_dst.buf) { @@ -1025,7 +1106,7 @@ puffs_flate_status puffs_flate_decoder_init_dynamic_huffman( v_n_bits += 8; } label_1_break:; - if ((v_table_entry >> 30) != 1) { + if ((v_table_entry >> 24) != 64) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; @@ -1271,13 +1352,26 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, } v_i = 0; v_n_symbols = ((uint32_t)(v_offsets[v_max_cl])); + if ((a_n_codes0 + ((uint32_t)(self->private_impl.f_symbols[0]))) >= 316) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } v_prev_cl = ((uint32_t)( - self->private_impl.f_code_lengths[self->private_impl.f_symbols[0]])); + self->private_impl + .f_code_lengths[a_n_codes0 + + ((uint32_t)(self->private_impl.f_symbols[0]))])); v_code = 0; while (true) { - v_cl = - ((uint32_t)(self->private_impl - .f_code_lengths[self->private_impl.f_symbols[v_i]])); + if ((a_n_codes0 + ((uint32_t)(self->private_impl.f_symbols[v_i]))) >= + 316) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_cl = ((uint32_t)( + self->private_impl.f_code_lengths + [a_n_codes0 + ((uint32_t)(self->private_impl.f_symbols[v_i]))])); if (v_cl > v_max_cl) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index 9f4c23fc..bdcdfa97 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -165,9 +165,12 @@ typedef struct { uint32_t coro_susp_point; uint32_t v_bits; uint32_t v_n_bits; - uint32_t v_lmask; uint32_t v_table_entry; uint32_t v_table_entry_n_bits; + uint32_t v_lmask; + uint32_t v_dmask; + uint32_t v_length; + uint32_t v_distance; } c_decode_huffman[1]; struct { uint32_t coro_susp_point; diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 9e963b9e..a97d5324 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -132,13 +132,15 @@ pri func decoder.decode_huffman?(dst writer1, src reader1)() { var bits u32 = this.bits var n_bits u32 = this.n_bits + var table_entry u32 + var table_entry_n_bits u32[..15] var lmask u32[..511] = ((1 as u32) << this.n_huffs_bits[0]) - 1 + var dmask u32[..511] = ((1 as u32) << this.n_huffs_bits[1]) - 1 while true { // Decode an lcode symbol from H-L. - var table_entry u32 while true { table_entry = this.huffs[0][bits & lmask] - var table_entry_n_bits u32[..15] = table_entry & 15 + table_entry_n_bits = table_entry & 0x0F if n_bits >= table_entry_n_bits { bits >>= table_entry_n_bits n_bits -= table_entry_n_bits @@ -160,14 +162,76 @@ pri func decoder.decode_huffman?(dst writer1, src reader1)() { } else if (table_entry >> 28) != 0 { // End of block. break - } else if (table_entry >> 27) != 0 { + } else if (table_entry >> 24) == 0x08 { return error "bad Huffman code" } else { return error "internal error: inconsistent Huffman decoder state" } - // TODO: non-literal Huffman symbols. - break + // length = base number + extra bits. + var length u32 = (table_entry >> 8) & 0xFFFF + // TODO: this if shouldn't be necessary. + if length > 0xFFFF { + return error "internal error: inconsistent Huffman decoder state" + } + table_entry_n_bits = (table_entry >> 4) & 0x0F + while n_bits < table_entry_n_bits, + inv length <= 0xFFFF, + post n_bits >= table_entry_n_bits, + { + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + length += bits.low_bits(n:table_entry_n_bits) + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + + // Decode a dcode symbol from H-D. + while true { + table_entry = this.huffs[1][bits & dmask] + table_entry_n_bits = table_entry & 15 + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + if (table_entry >> 31) != 0 { + return error "TODO: indirect Huffman tables" + } + + // For H-D, all symbols should be base number + extra bits. + if (table_entry >> 24) != 0x20 { + if (table_entry >> 24) == 0x08 { + return error "bad Huffman code" + } + return error "internal error: inconsistent Huffman decoder state" + } + + // distance = base number + extra bits. + var distance u32 = (table_entry >> 8) & 0xFFFF + // TODO: this if shouldn't be necessary. + if distance > 0xFFFF { + return error "internal error: inconsistent Huffman decoder state" + } + table_entry_n_bits = (table_entry >> 4) & 0x0F + while n_bits < table_entry_n_bits, + inv distance <= 0xFFFF, + post n_bits >= table_entry_n_bits, + { + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + distance += bits.low_bits(n:table_entry_n_bits) + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + + // TODO: use length and distance. } this.bits = bits @@ -254,8 +318,9 @@ pri func decoder.init_dynamic_huffman?(src reader1)() { bits |= (in.src.read_u8?() as u32) << n_bits n_bits += 8 } - // For H-CL, there are no redirections and all symbols are literals. - if (table_entry >> 30) != 1 { + // For H-CL, there should be no redirections and all symbols should be + // literals. + if (table_entry >> 24) != 0x40 { return error "internal error: inconsistent Huffman decoder state" } table_entry = (table_entry >> 8) & 0xFF @@ -476,14 +541,20 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. i = 0 var n_symbols u32[..289] = offsets[max_cl] as u32 - var prev_cl u32[..15] = this.code_lengths[this.symbols[0]] as u32 + if (in.n_codes0 + (this.symbols[0] as u32)) >= 316 { + return error "internal error: inconsistent Huffman decoder state" + } + var prev_cl u32[..15] = this.code_lengths[in.n_codes0 + (this.symbols[0] as u32)] as u32 var code u32 while true, pre code < (1 << 9), pre i < 289, inv max_cl <= 9, { - var cl u32[..15] = this.code_lengths[this.symbols[i]] as u32 + if (in.n_codes0 + (this.symbols[i] as u32)) >= 316 { + return error "internal error: inconsistent Huffman decoder state" + } + var cl u32[..15] = this.code_lengths[in.n_codes0 + (this.symbols[i] as u32)] as u32 if cl > max_cl { return error "internal error: inconsistent Huffman decoder state" }