From aa1f0078db859ec7b63ca6533212762161665b65 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 4 Nov 2017 14:37:14 +1100 Subject: [PATCH] Add decode_huffman_fast. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's not fast yet, and the immediate impact is indeed a significant performance loss, but follow-up commits should make it faster. name old speed new speed delta puffs_adler32_10k/clang 2.41GB/s ± 0% 2.40GB/s ± 1% ~ (p=0.222 n=5+5) puffs_adler32_100k/clang 2.41GB/s ± 1% 2.39GB/s ± 1% -0.97% (p=0.016 n=5+5) puffs_flate_decode_1k/clang 116MB/s ± 5% 115MB/s ± 0% ~ (p=0.310 n=5+5) puffs_flate_decode_10k/clang 145MB/s ± 0% 139MB/s ± 0% -4.18% (p=0.016 n=4+5) puffs_flate_decode_100k/clang 164MB/s ± 5% 158MB/s ± 1% ~ (p=0.151 n=5+5) puffs_zlib_decode_10k/clang 135MB/s ± 2% 131MB/s ± 0% -2.84% (p=0.008 n=5+5) puffs_zlib_decode_100k/clang 154MB/s ± 1% 148MB/s ± 2% -4.09% (p=0.016 n=4+5) puffs_adler32_10k/gcc 3.13GB/s ± 4% 3.19GB/s ± 0% +1.94% (p=0.032 n=5+5) puffs_adler32_100k/gcc 3.14GB/s ± 3% 3.20GB/s ± 3% ~ (p=0.056 n=5+5) puffs_flate_decode_1k/gcc 122MB/s ± 1% 94MB/s ± 1% -22.84% (p=0.008 n=5+5) puffs_flate_decode_10k/gcc 143MB/s ± 0% 110MB/s ± 0% -23.24% (p=0.008 n=5+5) puffs_flate_decode_100k/gcc 158MB/s ± 0% 118MB/s ± 0% -24.98% (p=0.008 n=5+5) puffs_zlib_decode_10k/gcc 136MB/s ± 0% 119MB/s ± 1% -12.63% (p=0.008 n=5+5) puffs_zlib_decode_100k/gcc 150MB/s ± 1% 124MB/s ± 2% -17.17% (p=0.008 n=5+5) --- cmd/puffs-c/internal/cgen/statement.go | 8 + gen/c/std/flate.c | 467 +++++++++++++++++++++++++ gen/h/std/flate.h | 16 + lang/check/bounds.go | 2 +- lang/check/type.go | 2 +- std/flate/decode_flate.puffs | 2 + std/flate/decode_huffman_fast.puffs | 253 ++++++++++++++ std/flate/decode_huffman_slow.puffs | 5 + 8 files changed, 753 insertions(+), 2 deletions(-) create mode 100644 std/flate/decode_huffman_fast.puffs diff --git a/cmd/puffs-c/internal/cgen/statement.go b/cmd/puffs-c/internal/cgen/statement.go index fbcbcba5..a98fb81b 100644 --- a/cmd/puffs-c/internal/cgen/statement.go +++ b/cmd/puffs-c/internal/cgen/statement.go @@ -517,6 +517,14 @@ func (g *gen) writeCallSuspendibles(b *buffer, n *a.Expr, depth uint32) error { } b.writes("if (status) { goto suspend; }\n") + } else if isThisMethod(g.tm, n, "decode_huffman_fast", 2) { + b.printf("status = %s%s__decode_huffman_fast(self, %sdst, %ssrc);\n", + g.pkgPrefix, g.currFunk.astFunc.Receiver().String(g.tm), aPrefix, aPrefix) + if err := g.writeLoadExprDerivedVars(b, n); err != nil { + return err + } + b.writes("if (status) { goto suspend; }\n") + } else if isThisMethod(g.tm, n, "decode_huffman_slow", 2) { b.printf("status = %s%s__decode_huffman_slow(self, %sdst, %ssrc);\n", g.pkgPrefix, g.currFunk.astFunc.Receiver().String(g.tm), aPrefix, aPrefix) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index b51f73cd..ff9ea6dc 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -235,6 +235,22 @@ typedef struct { uint8_t v_rep_symbol; uint32_t v_rep_count; } c_init_dynamic_huffman[1]; + struct { + uint32_t coro_susp_point; + uint32_t v_bits; + uint32_t v_n_bits; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; + uint32_t v_lmask; + uint32_t v_dmask; + uint32_t v_redir_top; + uint32_t v_redir_mask; + uint32_t v_length; + uint32_t v_distance; + uint32_t v_n_copied; + uint32_t v_hlen; + uint32_t v_hdist; + } c_decode_huffman_fast[1]; struct { uint32_t coro_susp_point; uint32_t v_bits; @@ -784,6 +800,11 @@ static puffs_flate__status puffs_flate__flate_decoder__init_huff( uint32_t a_n_codes1, uint32_t a_base_symbol); +static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_fast( + puffs_flate__flate_decoder* self, + puffs_base__writer1 a_dst, + puffs_base__reader1 a_src); + static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_slow( puffs_flate__flate_decoder* self, puffs_base__writer1 a_dst, @@ -1106,6 +1127,25 @@ static puffs_flate__status puffs_flate__flate_decoder__decode_blocks( } } } + status = + puffs_flate__flate_decoder__decode_huffman_fast(self, a_dst, a_src); + if (a_src.buf) { + b_rptr_src = a_src.buf->ptr + a_src.buf->ri; + } + if (status) { + goto suspend; + } + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); + if (a_src.buf) { + size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); + a_src.buf->ri += n; + puffs_base__limit1* lim; + for (lim = &a_src.private_impl.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + } status = puffs_flate__flate_decoder__decode_huffman_slow(self, a_dst, a_src); if (a_src.buf) { @@ -1914,6 +1954,433 @@ exit: return status; } +static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_fast( + puffs_flate__flate_decoder* self, + puffs_base__writer1 a_dst, + puffs_base__reader1 a_src) { + puffs_flate__status status = PUFFS_FLATE__STATUS_OK; + + uint32_t v_bits; + uint32_t v_n_bits; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; + uint32_t v_lmask; + uint32_t v_dmask; + uint32_t v_redir_top; + uint32_t v_redir_mask; + uint32_t v_length; + uint32_t v_distance; + uint32_t v_n_copied; + uint32_t v_hlen; + uint32_t v_hdist; + + uint8_t* b_wptr_dst = NULL; + uint8_t* b_wend_dst = NULL; + if (a_dst.buf) { + b_wptr_dst = a_dst.buf->ptr + a_dst.buf->wi; + b_wend_dst = b_wptr_dst; + if (!a_dst.buf->closed) { + uint64_t len = a_dst.buf->len - a_dst.buf->wi; + puffs_base__limit1* lim; + for (lim = &a_dst.private_impl.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_wend_dst += len; + } + } + uint8_t* b_rptr_src = NULL; + uint8_t* b_rend_src = NULL; + if (a_src.buf) { + b_rptr_src = a_src.buf->ptr + a_src.buf->ri; + uint64_t len = a_src.buf->wi - a_src.buf->ri; + puffs_base__limit1* lim; + for (lim = &a_src.private_impl.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_rend_src = b_rptr_src + len; + } + + uint32_t coro_susp_point = + self->private_impl.c_decode_huffman_fast[0].coro_susp_point; + if (coro_susp_point) { + v_bits = self->private_impl.c_decode_huffman_fast[0].v_bits; + v_n_bits = self->private_impl.c_decode_huffman_fast[0].v_n_bits; + v_table_entry = self->private_impl.c_decode_huffman_fast[0].v_table_entry; + v_table_entry_n_bits = + self->private_impl.c_decode_huffman_fast[0].v_table_entry_n_bits; + v_lmask = self->private_impl.c_decode_huffman_fast[0].v_lmask; + v_dmask = self->private_impl.c_decode_huffman_fast[0].v_dmask; + v_redir_top = self->private_impl.c_decode_huffman_fast[0].v_redir_top; + v_redir_mask = self->private_impl.c_decode_huffman_fast[0].v_redir_mask; + v_length = self->private_impl.c_decode_huffman_fast[0].v_length; + v_distance = self->private_impl.c_decode_huffman_fast[0].v_distance; + v_n_copied = self->private_impl.c_decode_huffman_fast[0].v_n_copied; + v_hlen = self->private_impl.c_decode_huffman_fast[0].v_hlen; + v_hdist = self->private_impl.c_decode_huffman_fast[0].v_hdist; + } + switch (coro_susp_point) { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; + + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + v_table_entry = 0; + v_table_entry_n_bits = 0; + v_lmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[0]) - 1); + v_dmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[1]) - 1); + label_0_continue:; + while (((uint64_t)(b_wend_dst - b_wptr_dst)) > 999999999) { + while (true) { + v_table_entry = self->private_impl.f_huffs[0][v_bits & v_lmask]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_1_break; + } + { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_0 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_0)) << v_n_bits); + } + v_n_bits += 8; + } + label_1_break:; + if ((v_table_entry >> 31) != 0) { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + if (b_wptr_dst == b_wend_dst) { + status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE; + goto suspend; + } + *b_wptr_dst++ = ((v_table_entry >> 8) & 255); + goto label_0_continue; + } else if ((v_table_entry >> 30) != 0) { + } else if ((v_table_entry >> 29) != 0) { + self->private_impl.f_end_of_block = true; + goto label_0_break; + } else if ((v_table_entry >> 28) != 0) { + v_redir_top = ((v_table_entry >> 8) & 65535); + v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1); + while (true) { + if ((v_redir_top + (v_bits & v_redir_mask)) >= 1234) { + status = + PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_table_entry = + self->private_impl + .f_huffs[0][v_redir_top + (v_bits & v_redir_mask)]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_2_break; + } + { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); + if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_1 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_1)) << v_n_bits); + } + v_n_bits += 8; + } + label_2_break:; + if ((v_table_entry >> 31) != 0) { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); + if (b_wptr_dst == b_wend_dst) { + status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE; + goto suspend; + } + *b_wptr_dst++ = ((v_table_entry >> 8) & 255); + goto label_0_continue; + } else if ((v_table_entry >> 30) != 0) { + } else if ((v_table_entry >> 29) != 0) { + self->private_impl.f_end_of_block = true; + goto label_0_break; + } else if ((v_table_entry >> 28) != 0) { + status = + PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } else if ((v_table_entry >> 27) != 0) { + status = PUFFS_FLATE__ERROR_BAD_HUFFMAN_CODE; + goto exit; + } else { + status = + PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + } else if ((v_table_entry >> 27) != 0) { + status = PUFFS_FLATE__ERROR_BAD_HUFFMAN_CODE; + goto exit; + } else { + status = + PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_length = ((v_table_entry >> 8) & 32767); + v_table_entry_n_bits = ((v_table_entry >> 4) & 15); + while (v_n_bits < v_table_entry_n_bits) { + { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); + if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_2 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_2)) << v_n_bits); + } + v_n_bits += 8; + } + v_length = + ((v_length + ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1))) & + 32767); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + while (true) { + v_table_entry = self->private_impl.f_huffs[1][v_bits & v_dmask]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_3_break; + } + { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); + if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_3 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_3)) << v_n_bits); + } + v_n_bits += 8; + } + label_3_break:; + if ((v_table_entry >> 28) == 1) { + v_redir_top = ((v_table_entry >> 8) & 65535); + v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1); + while (true) { + if ((v_redir_top + (v_bits & v_redir_mask)) >= 1234) { + status = + PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_table_entry = + self->private_impl + .f_huffs[1][v_redir_top + (v_bits & v_redir_mask)]; + v_table_entry_n_bits = (v_table_entry & 15); + if (v_n_bits >= v_table_entry_n_bits) { + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + goto label_4_break; + } + { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(7); + if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_4 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_4)) << v_n_bits); + } + v_n_bits += 8; + } + label_4_break:; + if ((v_table_entry >> 31) != 0) { + status = + PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + } + if ((v_table_entry >> 24) != 64) { + if ((v_table_entry >> 24) == 8) { + status = PUFFS_FLATE__ERROR_BAD_HUFFMAN_CODE; + goto exit; + } + status = + PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_distance = ((v_table_entry >> 8) & 32767); + v_table_entry_n_bits = ((v_table_entry >> 4) & 15); + while (v_n_bits < v_table_entry_n_bits) { + { + PUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); + if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint8_t t_5 = *b_rptr_src++; + v_bits |= (((uint32_t)(t_5)) << v_n_bits); + } + v_n_bits += 8; + } + v_distance = + ((v_distance + ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1))) & + 32767); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + v_n_copied = 0; + while (true) { + if (((uint64_t)(v_distance)) > + ((uint64_t)( + ((puffs_base__slice_u8){ + .ptr = a_dst.private_impl.mark, + .len = a_dst.private_impl.mark + ? (size_t)(b_wptr_dst - a_dst.private_impl.mark) + : 0, + }) + .len))) { + v_hlen = 0; + v_hdist = ((uint32_t)( + (((uint64_t)(v_distance)) - + ((uint64_t)(((puffs_base__slice_u8){ + .ptr = a_dst.private_impl.mark, + .len = a_dst.private_impl.mark + ? (size_t)(b_wptr_dst - + a_dst.private_impl.mark) + : 0, + }) + .len))))); + if (v_length > v_hdist) { + v_length -= v_hdist; + v_hlen = v_hdist; + } else { + v_hlen = v_length; + v_length = 0; + } + if (self->private_impl.f_history_index < v_hdist) { + status = PUFFS_FLATE__ERROR_BAD_DISTANCE; + goto exit; + } + v_hdist = ((self->private_impl.f_history_index - v_hdist) & 32767); + while (true) { + v_n_copied = puffs_base__writer1__copy_from_slice32( + &b_wptr_dst, b_wend_dst, + puffs_base__slice_u8__subslice_i( + ((puffs_base__slice_u8){.ptr = self->private_impl.f_history, + .len = 32768}), + v_hdist), + v_hlen); + if (v_hlen <= v_n_copied) { + v_hlen = 0; + goto label_5_break; + } + if (v_n_copied > 0) { + v_hlen -= v_n_copied; + v_hdist = ((v_hdist + v_n_copied) & 32767); + if (v_hdist == 0) { + goto label_5_break; + } + } + status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE; + PUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9); + } + label_5_break:; + if (v_hlen > 0) { + while (true) { + v_n_copied = puffs_base__writer1__copy_from_slice32( + &b_wptr_dst, b_wend_dst, + puffs_base__slice_u8__subslice_i( + ((puffs_base__slice_u8){ + .ptr = self->private_impl.f_history, .len = 32768}), + v_hdist), + v_hlen); + if (v_hlen <= v_n_copied) { + v_hlen = 0; + goto label_6_break; + } + v_hlen -= v_n_copied; + v_hdist = ((v_hdist + (v_n_copied & 32767)) & 32767); + status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE; + PUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + } + label_6_break:; + } + if (v_length == 0) { + goto label_0_continue; + } + } + v_n_copied = puffs_base__writer1__copy_from_history32( + &b_wptr_dst, a_dst.private_impl.mark, b_wend_dst, v_distance, + v_length); + if (v_length <= v_n_copied) { + v_length = 0; + goto label_7_break; + } + v_length -= v_n_copied; + status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE; + PUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); + } + label_7_break:; + } + label_0_break:; + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; + + goto ok; + ok: + self->private_impl.c_decode_huffman_fast[0].coro_susp_point = 0; + goto exit; + } + + goto suspend; +suspend: + self->private_impl.c_decode_huffman_fast[0].coro_susp_point = coro_susp_point; + self->private_impl.c_decode_huffman_fast[0].v_bits = v_bits; + self->private_impl.c_decode_huffman_fast[0].v_n_bits = v_n_bits; + self->private_impl.c_decode_huffman_fast[0].v_table_entry = v_table_entry; + self->private_impl.c_decode_huffman_fast[0].v_table_entry_n_bits = + v_table_entry_n_bits; + self->private_impl.c_decode_huffman_fast[0].v_lmask = v_lmask; + self->private_impl.c_decode_huffman_fast[0].v_dmask = v_dmask; + self->private_impl.c_decode_huffman_fast[0].v_redir_top = v_redir_top; + self->private_impl.c_decode_huffman_fast[0].v_redir_mask = v_redir_mask; + self->private_impl.c_decode_huffman_fast[0].v_length = v_length; + self->private_impl.c_decode_huffman_fast[0].v_distance = v_distance; + self->private_impl.c_decode_huffman_fast[0].v_n_copied = v_n_copied; + self->private_impl.c_decode_huffman_fast[0].v_hlen = v_hlen; + self->private_impl.c_decode_huffman_fast[0].v_hdist = v_hdist; + +exit: + if (a_dst.buf) { + size_t n = b_wptr_dst - (a_dst.buf->ptr + a_dst.buf->wi); + a_dst.buf->wi += n; + puffs_base__limit1* lim; + for (lim = &a_dst.private_impl.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + PUFFS_BASE__IGNORE_POTENTIALLY_UNUSED_VARIABLE(b_wend_dst); + } + if (a_src.buf) { + size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); + a_src.buf->ri += n; + puffs_base__limit1* lim; + for (lim = &a_src.private_impl.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + PUFFS_BASE__IGNORE_POTENTIALLY_UNUSED_VARIABLE(b_rend_src); + } + + return status; + +short_read_src: + if (a_src.buf && a_src.buf->closed && !a_src.private_impl.limit.ptr_to_len) { + status = PUFFS_FLATE__ERROR_UNEXPECTED_EOF; + goto exit; + } + status = PUFFS_FLATE__SUSPENSION_SHORT_READ; + goto suspend; +} + static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_slow( puffs_flate__flate_decoder* self, puffs_base__writer1 a_dst, diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index c4196836..d7aec4ff 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -235,6 +235,22 @@ typedef struct { uint8_t v_rep_symbol; uint32_t v_rep_count; } c_init_dynamic_huffman[1]; + struct { + uint32_t coro_susp_point; + uint32_t v_bits; + uint32_t v_n_bits; + uint32_t v_table_entry; + uint32_t v_table_entry_n_bits; + uint32_t v_lmask; + uint32_t v_dmask; + uint32_t v_redir_top; + uint32_t v_redir_mask; + uint32_t v_length; + uint32_t v_distance; + uint32_t v_n_copied; + uint32_t v_hlen; + uint32_t v_hdist; + } c_decode_huffman_fast[1]; struct { uint32_t coro_susp_point; uint32_t v_bits; diff --git a/lang/check/bounds.go b/lang/check/bounds.go index 9061820e..649c4d5a 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -725,7 +725,7 @@ func (q *checker) bcheckExprOther(n *a.Expr, depth uint32) (*big.Int, *big.Int, isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_blocks", 2) || - isThisMethod(q.tm, n, "decode_huffman_slow", 2) || + isThisMethod(q.tm, n, "decode_huffman_slow", 2) || isThisMethod(q.tm, n, "decode_huffman_fast", 2) || isThisMethod(q.tm, n, "init_fixed_huffman", 0) || isThisMethod(q.tm, n, "init_dynamic_huffman", 1) || isThisMethod(q.tm, n, "init_huff", 4) || isThatMethod(q.tm, n, t.KeyMark, 0) || isThatMethod(q.tm, n, t.KeyLimit, 1) || diff --git a/lang/check/type.go b/lang/check/type.go index daa0c1a9..d15e644e 100644 --- a/lang/check/type.go +++ b/lang/check/type.go @@ -419,7 +419,7 @@ func (q *checker) tcheckExprOther(n *a.Expr, depth uint32) error { isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_blocks", 2) || - isThisMethod(q.tm, n, "decode_huffman_slow", 2) || + isThisMethod(q.tm, n, "decode_huffman_slow", 2) || isThisMethod(q.tm, n, "decode_huffman_fast", 2) || isThisMethod(q.tm, n, "init_fixed_huffman", 0) || isThisMethod(q.tm, n, "init_dynamic_huffman", 1) || isThisMethod(q.tm, n, "init_huff", 4) || isThatMethod(q.tm, n, t.KeyMark, 0) || isThatMethod(q.tm, n, t.KeyLimit, 1) || diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 233e15d0..35f7dc91 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -200,6 +200,8 @@ pri func flate_decoder.decode_blocks?(dst writer1, src reader1)() { } this.end_of_block = false + this.decode_huffman_fast?(dst:in.dst, src:in.src) + // TODO: check this.end_of_block. this.decode_huffman_slow?(dst:in.dst, src:in.src) if not this.end_of_block { return error "internal error: inconsistent Huffman end_of_block" diff --git a/std/flate/decode_huffman_fast.puffs b/std/flate/decode_huffman_fast.puffs new file mode 100644 index 00000000..1b2eaa70 --- /dev/null +++ b/std/flate/decode_huffman_fast.puffs @@ -0,0 +1,253 @@ +// Copyright 2017 The Puffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pri func flate_decoder.decode_huffman_fast?(dst writer1, src reader1)() { + // When editing this function, consider making the equivalent change to the + // decode_huffman_slow function. Keep the diff between the two + // decode_huffman_*.puffs files as small as possible, while retaining both + // correctness and performance. + + var bits u32 = this.bits + var n_bits u32 = this.n_bits + + var table_entry u32 + var table_entry_n_bits u32[..15] + var lmask u32[..511] = ((1 as u32) << this.n_huffs_bits[0]) - 1 + var dmask u32[..511] = ((1 as u32) << this.n_huffs_bits[1]) - 1 + + while:loop in.dst.available() > 999999999 { // TODO: change 999999999 to the real condition. + // Decode an lcode symbol from H-L. + while true { + table_entry = this.huffs[0][bits & lmask] + table_entry_n_bits = table_entry & 0x0F + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + + if (table_entry >> 31) != 0 { + // Literal. + in.dst.write_u8?(x:(table_entry >> 8) & 0xFF) + continue:loop + } else if (table_entry >> 30) != 0 { + // No-op; code continues past the if-else chain. + } else if (table_entry >> 29) != 0 { + // End of block. + this.end_of_block = true + break:loop + } else if (table_entry >> 28) != 0 { + // Redirect. + var redir_top u32[..0xFFFF] = (table_entry >> 8) & 0xFFFF + var redir_mask u32[..0x7FFF] = ((1 as u32) << ((table_entry >> 4) & 0x0F)) - 1 + while true { + if (redir_top + (bits & redir_mask)) >= 1234 { + return error "internal error: inconsistent Huffman decoder state" + } + table_entry = this.huffs[0][redir_top + (bits & redir_mask)] + table_entry_n_bits = table_entry & 0x0F + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + + if (table_entry >> 31) != 0 { + // Literal. + in.dst.write_u8?(x:(table_entry >> 8) & 0xFF) + continue:loop + } else if (table_entry >> 30) != 0 { + // No-op; code continues past the if-else chain. + } else if (table_entry >> 29) != 0 { + // End of block. + this.end_of_block = true + break:loop + } else if (table_entry >> 28) != 0 { + return error "internal error: inconsistent Huffman decoder state" + } else if (table_entry >> 27) != 0 { + return error "bad Huffman code" + } else { + return error "internal error: inconsistent Huffman decoder state" + } + + } else if (table_entry >> 27) != 0 { + return error "bad Huffman code" + } else { + return error "internal error: inconsistent Huffman decoder state" + } + + // length = base number + extra bits. + var length u32[..0x7FFF] = (table_entry >> 8) & 0x7FFF + table_entry_n_bits = (table_entry >> 4) & 0x0F + while n_bits < table_entry_n_bits, + post n_bits >= table_entry_n_bits, + { + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + length = (length + bits.low_bits(n:table_entry_n_bits)) & 0x7FFF + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + + // Decode a dcode symbol from H-D. + while true { + table_entry = this.huffs[1][bits & dmask] + table_entry_n_bits = table_entry & 15 + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + // Check for a redirect. + if (table_entry >> 28) == 1 { + redir_top = (table_entry >> 8) & 0xFFFF + redir_mask = ((1 as u32) << ((table_entry >> 4) & 0x0F)) - 1 + while true { + if (redir_top + (bits & redir_mask)) >= 1234 { + return error "internal error: inconsistent Huffman decoder state" + } + table_entry = this.huffs[1][redir_top + (bits & redir_mask)] + table_entry_n_bits = table_entry & 0x0F + if n_bits >= table_entry_n_bits { + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + break + } + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + if (table_entry >> 31) != 0 { + return error "internal error: inconsistent Huffman decoder state" + } + } + + // For H-D, all symbols should be base number + extra bits. + if (table_entry >> 24) != 0x40 { + if (table_entry >> 24) == 0x08 { + return error "bad Huffman code" + } + return error "internal error: inconsistent Huffman decoder state" + } + + // distance = base number + extra bits. + var distance u32[..0x7FFF] = (table_entry >> 8) & 0x7FFF + table_entry_n_bits = (table_entry >> 4) & 0x0F + while n_bits < table_entry_n_bits, + post n_bits >= table_entry_n_bits, + { + assert n_bits < 15 via "a < b: a < c; c <= b"(c:table_entry_n_bits) + bits |= (in.src.read_u8?() as u32) << n_bits + n_bits += 8 + } + distance = (distance + bits.low_bits(n:table_entry_n_bits)) & 0x7FFF + bits >>= table_entry_n_bits + n_bits -= table_entry_n_bits + + var n_copied u32 + while true { + // Copy from this.history. + if (distance as u64) > in.dst.since_mark().length() { + // Set (hlen, hdist) to be the length-distance pair to copy + // from this.history, and (length, distance) to be the + // remaining length-distance pair to copy from in.dst. + var hlen u32[..0x7FFF] + var hdist u32[..0x7FFF] = ((distance as u64) - in.dst.since_mark().length()) as u32 + if length > hdist { + length -= hdist + hlen = hdist + } else { + hlen = length + length = 0 + } + if this.history_index < hdist { + return error "bad distance" + } + // Re-purpose the hdist variable as the this.history index to + // start copying from. + hdist = (this.history_index - hdist) & 0x7FFF + + // Copy from hdist to the end of this.history. + while true { + n_copied = in.dst.copy_from_slice32(s:this.history[hdist:], length:hlen) + if hlen <= n_copied { + hlen = 0 + break + } + if n_copied > 0 { + hlen -= n_copied + hdist = (hdist ~+ n_copied) & 0x7FFF + if hdist == 0 { + // Wrap around the this.history ringbuffer. + break + } + } + // TODO: "closed for writes" instead? + return suspension "short write" + // TODO: this "assert false" should be a compiler error. + // This might mean differentiating "return" and "raise". + // + // TODO: invalidate this-related facts because the + // suspension can change the state of this and its fields. + assert false + } + // Copy from the start of this.history, if we wrapped around. + if hlen > 0 { + while true { + n_copied = in.dst.copy_from_slice32(s:this.history[hdist:], length:hlen) + if hlen <= n_copied { + hlen = 0 + break + } + hlen -= n_copied + hdist = (hdist + (n_copied & 0x7FFF)) & 0x7FFF + // TODO: "closed for writes" instead? + return suspension "short write" + } + } + + if length == 0 { + // No need to copy from in.dst. + continue:loop + } + } + + // Copy from in.dst. + n_copied = in.dst.copy_from_history32(distance:distance, length:length) + if length <= n_copied { + length = 0 + break + } + length -= n_copied + // TODO: "closed for writes" instead? + return suspension "short write" + } + } + + this.bits = bits + this.n_bits = n_bits +} diff --git a/std/flate/decode_huffman_slow.puffs b/std/flate/decode_huffman_slow.puffs index 83d9a429..f203d386 100644 --- a/std/flate/decode_huffman_slow.puffs +++ b/std/flate/decode_huffman_slow.puffs @@ -13,6 +13,11 @@ // limitations under the License. pri func flate_decoder.decode_huffman_slow?(dst writer1, src reader1)() { + // When editing this function, consider making the equivalent change to the + // decode_huffman_fast function. Keep the diff between the two + // decode_huffman_*.puffs files as small as possible, while retaining both + // correctness and performance. + var bits u32 = this.bits var n_bits u32 = this.n_bits