From 67b95c01c340179e8983fa3590bfb1fcfe7111be Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 27 Jul 2017 15:32:05 +1000 Subject: [PATCH] Decode literals from the lcode Huffman table. --- gen/c/std/flate.c | 166 ++++++++++++++++++++++++++++++++++- gen/h/std/flate.h | 6 ++ std/flate/decode_flate.puffs | 57 +++++++++++- 3 files changed, 225 insertions(+), 4 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 369c6bc3..496302f7 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -163,6 +163,11 @@ typedef struct { } c_decode_uncompressed[1]; struct { uint32_t coro_susp_point; + uint32_t v_bits; + uint32_t v_n_bits; + uint32_t v_lmask; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; } c_decode_dynamic[1]; struct { uint32_t coro_susp_point; @@ -192,6 +197,7 @@ typedef struct { uint32_t v_cl; uint32_t v_reversed_code; uint32_t v_value; + uint32_t v_symbol; uint32_t v_high_bits; uint32_t v_delta; } c_init_huff[1]; @@ -702,18 +708,126 @@ puffs_flate_status puffs_flate_decoder_decode_dynamic( puffs_base_reader1 a_src) { puffs_flate_status status = PUFFS_FLATE_STATUS_OK; + uint32_t v_bits; + uint32_t v_n_bits; + uint32_t v_lmask; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; + + uint8_t* b_wptr_dst = NULL; + uint8_t* b_wend_dst = NULL; + if (a_dst.buf) { + b_wptr_dst = a_dst.buf->ptr + a_dst.buf->wi; + size_t len = a_dst.buf->len - a_dst.buf->wi; + puffs_base_limit1* lim; + for (lim = &a_dst.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_wend_dst = b_wptr_dst + len; + } + uint8_t* b_rptr_src = NULL; + uint8_t* b_rend_src = NULL; + if (a_src.buf) { + b_rptr_src = a_src.buf->ptr + a_src.buf->ri; + size_t len = a_src.buf->wi - a_src.buf->ri; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_rend_src = b_rptr_src + len; + } + uint32_t coro_susp_point = self->private_impl.c_decode_dynamic[0].coro_susp_point; if (coro_susp_point) { + v_bits = self->private_impl.c_decode_dynamic[0].v_bits; + v_n_bits = self->private_impl.c_decode_dynamic[0].v_n_bits; + v_lmask = self->private_impl.c_decode_dynamic[0].v_lmask; + v_table_entry = self->private_impl.c_decode_dynamic[0].v_table_entry; + v_table_entry_n_bits = + self->private_impl.c_decode_dynamic[0].v_table_entry_n_bits; } switch (coro_susp_point) { PUFFS_COROUTINE_SUSPENSION_POINT(0); PUFFS_COROUTINE_SUSPENSION_POINT(1); + if (a_src.buf) { + size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); + a_src.buf->ri += n; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + } status = puffs_flate_decoder_init_huffs(self, a_src); + if (a_src.buf) { + b_rptr_src = a_src.buf->ptr + a_src.buf->ri; + size_t len = a_src.buf->wi - a_src.buf->ri; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_rend_src = b_rptr_src + len; + } if (status) { goto suspend; } + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + v_lmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[0]) - 1); + label_0_continue:; + while (true) { + v_table_entry = 0; + while (true) { + v_table_entry = self->private_impl.f_huffs[0][v_bits & v_lmask]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_1_break; + } + PUFFS_COROUTINE_SUSPENSION_POINT(2); + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_0 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_0)) << v_n_bits); + v_n_bits += 8; + } + label_1_break:; + if ((v_table_entry >> 31) != 0) { + status = PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES; + goto exit; + } + if ((v_table_entry >> 30) != 0) { + PUFFS_COROUTINE_SUSPENSION_POINT(3); + if (b_wptr_dst == b_wend_dst) { + status = PUFFS_FLATE_SUSPENSION_SHORT_WRITE; + goto suspend; + } + *b_wptr_dst++ = ((v_table_entry >> 8) & 255); + goto label_0_continue; + } else if ((v_table_entry >> 29) != 0) { + } else if ((v_table_entry >> 28) != 0) { + goto label_0_break; + } else { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + goto label_0_break; + } + label_0_break:; + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; self->private_impl.c_decode_dynamic[0].coro_susp_point = 0; goto exit; } @@ -721,9 +835,47 @@ puffs_flate_status puffs_flate_decoder_decode_dynamic( goto suspend; suspend: self->private_impl.c_decode_dynamic[0].coro_susp_point = coro_susp_point; + self->private_impl.c_decode_dynamic[0].v_bits = v_bits; + self->private_impl.c_decode_dynamic[0].v_n_bits = v_n_bits; + self->private_impl.c_decode_dynamic[0].v_lmask = v_lmask; + self->private_impl.c_decode_dynamic[0].v_table_entry = v_table_entry; + self->private_impl.c_decode_dynamic[0].v_table_entry_n_bits = + v_table_entry_n_bits; exit: + if (a_dst.buf) { + size_t n = b_wptr_dst - (a_dst.buf->ptr + a_dst.buf->wi); + a_dst.buf->wi += n; + puffs_base_limit1* lim; + for (lim = &a_dst.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + } + if (a_src.buf) { + size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); + a_src.buf->ri += n; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + } + return status; + +short_read_src: + if (a_src.limit.ptr_to_len) { + status = PUFFS_FLATE_SUSPENSION_LIMITED_READ; + } else if (a_src.buf->closed) { + status = PUFFS_FLATE_ERROR_UNEXPECTED_EOF; + goto exit; + } else { + status = PUFFS_FLATE_SUSPENSION_SHORT_READ; + } + goto suspend; } puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, @@ -992,6 +1144,7 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, uint32_t v_cl; uint32_t v_reversed_code; uint32_t v_value; + uint32_t v_symbol; uint32_t v_high_bits; uint32_t v_delta; @@ -1008,6 +1161,7 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_cl = self->private_impl.c_init_huff[0].v_cl; v_reversed_code = self->private_impl.c_init_huff[0].v_reversed_code; v_value = self->private_impl.c_init_huff[0].v_value; + v_symbol = self->private_impl.c_init_huff[0].v_symbol; v_high_bits = self->private_impl.c_init_huff[0].v_high_bits; v_delta = self->private_impl.c_init_huff[0].v_delta; } @@ -1110,8 +1264,15 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_reversed_code = (((uint32_t)(puffs_flate_reverse8[v_code >> 1])) | ((v_code & 1) << 8)); v_reversed_code >>= (9 - v_cl); - v_value = (1073741824 | - (((uint32_t)(self->private_impl.f_symbols[v_i])) << 8) | v_cl); + v_value = 0; + v_symbol = ((uint32_t)(self->private_impl.f_symbols[v_i])); + if (v_symbol == 256) { + v_value = (268435456 | v_cl); + } else if ((v_symbol < 256) && (a_which == 0)) { + v_value = (1073741824 | (v_symbol << 8) | v_cl); + } else { + v_value = (536870912 | v_cl); + } v_high_bits = (((uint32_t)(1)) << v_max_cl); v_delta = (((uint32_t)(1)) << v_cl); while (v_high_bits >= v_delta) { @@ -1151,6 +1312,7 @@ suspend: self->private_impl.c_init_huff[0].v_cl = v_cl; self->private_impl.c_init_huff[0].v_reversed_code = v_reversed_code; self->private_impl.c_init_huff[0].v_value = v_value; + self->private_impl.c_init_huff[0].v_symbol = v_symbol; self->private_impl.c_init_huff[0].v_high_bits = v_high_bits; self->private_impl.c_init_huff[0].v_delta = v_delta; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index aa8452d5..217d02a4 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -163,6 +163,11 @@ typedef struct { } c_decode_uncompressed[1]; struct { uint32_t coro_susp_point; + uint32_t v_bits; + uint32_t v_n_bits; + uint32_t v_lmask; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; } c_decode_dynamic[1]; struct { uint32_t coro_susp_point; @@ -192,6 +197,7 @@ typedef struct { uint32_t v_cl; uint32_t v_reversed_code; uint32_t v_value; + uint32_t v_symbol; uint32_t v_high_bits; uint32_t v_delta; } c_init_huff[1]; diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 3328aaa6..12cee235 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -109,6 +109,48 @@ pri func decoder.decode_uncompressed?(dst writer1, src reader1)() { // decode_dynamic decodes dynamic Huffman codes as per the RFC section 3.2.7. pri func decoder.decode_dynamic?(dst writer1, src reader1)() { this.init_huffs?(src:in.src) + + var bits u32 = this.bits + var n_bits u32 = this.n_bits + + var lmask u32[..511] = ((1 as u32) << this.n_huffs_bits[0]) - 1 + while true { + // Decode an lcode symbol from H-L. + var table_entry u32 + while true { + table_entry = this.huffs[0][bits & lmask] + var table_entry_n_bits u32[..15] = table_entry & 15 + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + if (table_entry >> 31) != 0 { + return error "TODO: indirect Huffman tables" + } + + if (table_entry >> 30) != 0 { + in.dst.write_u8?(x:(table_entry >> 8) & 0xFF) + continue + } else if (table_entry >> 29) != 0 { + // No-op; code continues past the if-else chain. + } else if (table_entry >> 28) != 0 { + // End of block. + break + } else { + return error "internal error: inconsistent Huffman decoder state" + } + + // TODO: non-literal Huffman symbols. + break + } + + this.bits = bits + this.n_bits = n_bits } pri func decoder.init_huffs?(src reader1)() { @@ -431,8 +473,19 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. var reversed_code u32[..511] = (reverse8[code >> 1] as u32) | ((code & 1) << 8) reversed_code >>= 9 - cl - // TODO: don't assume that the value is a literal. - var value u32 = 0x40000000 | ((this.symbols[i] as u32) << 8) | cl + var value u32 + var symbol u32[..289] = this.symbols[i] as u32 + if symbol == 256 { + // End-of-block. + value = 0x10000000 | cl + } else if (symbol < 256) and (in.which == 0) { + // Literal. + value = 0x40000000 | (symbol << 8) | cl + } else { + // Base number + extra bits. + // TODO: bits 27-8. + value = 0x20000000 | cl + } // The table uses max_cl bits, but reversed_code only has cl bits. We // duplicate the key-value pair across all possible values of the high