Add decode_huffman_fast.

It's not fast yet, and the immediate impact is indeed a significant
performance loss, but follow-up commits should make it faster.

name                           old speed      new speed      delta

puffs_adler32_10k/clang        2.41GB/s ± 0%  2.40GB/s ± 1%     ~     (p=0.222 n=5+5)
puffs_adler32_100k/clang       2.41GB/s ± 1%  2.39GB/s ± 1%   -0.97%  (p=0.016 n=5+5)
puffs_flate_decode_1k/clang     116MB/s ± 5%   115MB/s ± 0%     ~     (p=0.310 n=5+5)
puffs_flate_decode_10k/clang    145MB/s ± 0%   139MB/s ± 0%   -4.18%  (p=0.016 n=4+5)
puffs_flate_decode_100k/clang   164MB/s ± 5%   158MB/s ± 1%     ~     (p=0.151 n=5+5)
puffs_zlib_decode_10k/clang     135MB/s ± 2%   131MB/s ± 0%   -2.84%  (p=0.008 n=5+5)
puffs_zlib_decode_100k/clang    154MB/s ± 1%   148MB/s ± 2%   -4.09%  (p=0.016 n=4+5)

puffs_adler32_10k/gcc          3.13GB/s ± 4%  3.19GB/s ± 0%   +1.94%  (p=0.032 n=5+5)
puffs_adler32_100k/gcc         3.14GB/s ± 3%  3.20GB/s ± 3%     ~     (p=0.056 n=5+5)
puffs_flate_decode_1k/gcc       122MB/s ± 1%    94MB/s ± 1%  -22.84%  (p=0.008 n=5+5)
puffs_flate_decode_10k/gcc      143MB/s ± 0%   110MB/s ± 0%  -23.24%  (p=0.008 n=5+5)
puffs_flate_decode_100k/gcc     158MB/s ± 0%   118MB/s ± 0%  -24.98%  (p=0.008 n=5+5)
puffs_zlib_decode_10k/gcc       136MB/s ± 0%   119MB/s ± 1%  -12.63%  (p=0.008 n=5+5)
puffs_zlib_decode_100k/gcc      150MB/s ± 1%   124MB/s ± 2%  -17.17%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2017-11-04 14:45:11 +11:00
1 parent c550ad9bda
commit aa1f0078db
8 files changed
+753 -2

No files matched your search

+467
View File
@@ -235,6 +235,22 @@ typedef struct {
uint8_t v_rep_symbol;
uint32_t v_rep_count;
} c_init_dynamic_huffman[1];
struct {
uint32_t coro_susp_point;
uint32_t v_bits;
uint32_t v_n_bits;
uint32_t v_table_entry;
uint32_t v_table_entry_n_bits;
uint32_t v_lmask;
uint32_t v_dmask;
uint32_t v_redir_top;
uint32_t v_redir_mask;
uint32_t v_length;
uint32_t v_distance;
uint32_t v_n_copied;
uint32_t v_hlen;
uint32_t v_hdist;
} c_decode_huffman_fast[1];
struct {
uint32_t coro_susp_point;
uint32_t v_bits;
@@ -784,6 +800,11 @@ static puffs_flate__status puffs_flate__flate_decoder__init_huff(
uint32_t a_n_codes1,
uint32_t a_base_symbol);
static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_fast(
puffs_flate__flate_decoder* self,
puffs_base__writer1 a_dst,
puffs_base__reader1 a_src);
static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_slow(
puffs_flate__flate_decoder* self,
puffs_base__writer1 a_dst,
@@ -1106,6 +1127,25 @@ static puffs_flate__status puffs_flate__flate_decoder__decode_blocks(
}
}
}
status =
puffs_flate__flate_decoder__decode_huffman_fast(self, a_dst, a_src);
if (a_src.buf) {
b_rptr_src = a_src.buf->ptr + a_src.buf->ri;
}
if (status) {
goto suspend;
}
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(6);
if (a_src.buf) {
size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri);
a_src.buf->ri += n;
puffs_base__limit1* lim;
for (lim = &a_src.private_impl.limit; lim; lim = lim->next) {
if (lim->ptr_to_len) {
*lim->ptr_to_len -= n;
}
}
}
status =
puffs_flate__flate_decoder__decode_huffman_slow(self, a_dst, a_src);
if (a_src.buf) {
@@ -1914,6 +1954,433 @@ exit:
return status;
}
static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_fast(
puffs_flate__flate_decoder* self,
puffs_base__writer1 a_dst,
puffs_base__reader1 a_src) {
puffs_flate__status status = PUFFS_FLATE__STATUS_OK;
uint32_t v_bits;
uint32_t v_n_bits;
uint32_t v_table_entry;
uint32_t v_table_entry_n_bits;
uint32_t v_lmask;
uint32_t v_dmask;
uint32_t v_redir_top;
uint32_t v_redir_mask;
uint32_t v_length;
uint32_t v_distance;
uint32_t v_n_copied;
uint32_t v_hlen;
uint32_t v_hdist;
uint8_t* b_wptr_dst = NULL;
uint8_t* b_wend_dst = NULL;
if (a_dst.buf) {
b_wptr_dst = a_dst.buf->ptr + a_dst.buf->wi;
b_wend_dst = b_wptr_dst;
if (!a_dst.buf->closed) {
uint64_t len = a_dst.buf->len - a_dst.buf->wi;
puffs_base__limit1* lim;
for (lim = &a_dst.private_impl.limit; lim; lim = lim->next) {
if (lim->ptr_to_len && (len > *lim->ptr_to_len)) {
len = *lim->ptr_to_len;
}
}
b_wend_dst += len;
}
}
uint8_t* b_rptr_src = NULL;
uint8_t* b_rend_src = NULL;
if (a_src.buf) {
b_rptr_src = a_src.buf->ptr + a_src.buf->ri;
uint64_t len = a_src.buf->wi - a_src.buf->ri;
puffs_base__limit1* lim;
for (lim = &a_src.private_impl.limit; lim; lim = lim->next) {
if (lim->ptr_to_len && (len > *lim->ptr_to_len)) {
len = *lim->ptr_to_len;
}
}
b_rend_src = b_rptr_src + len;
}
uint32_t coro_susp_point =
self->private_impl.c_decode_huffman_fast[0].coro_susp_point;
if (coro_susp_point) {
v_bits = self->private_impl.c_decode_huffman_fast[0].v_bits;
v_n_bits = self->private_impl.c_decode_huffman_fast[0].v_n_bits;
v_table_entry = self->private_impl.c_decode_huffman_fast[0].v_table_entry;
v_table_entry_n_bits =
self->private_impl.c_decode_huffman_fast[0].v_table_entry_n_bits;
v_lmask = self->private_impl.c_decode_huffman_fast[0].v_lmask;
v_dmask = self->private_impl.c_decode_huffman_fast[0].v_dmask;
v_redir_top = self->private_impl.c_decode_huffman_fast[0].v_redir_top;
v_redir_mask = self->private_impl.c_decode_huffman_fast[0].v_redir_mask;
v_length = self->private_impl.c_decode_huffman_fast[0].v_length;
v_distance = self->private_impl.c_decode_huffman_fast[0].v_distance;
v_n_copied = self->private_impl.c_decode_huffman_fast[0].v_n_copied;
v_hlen = self->private_impl.c_decode_huffman_fast[0].v_hlen;
v_hdist = self->private_impl.c_decode_huffman_fast[0].v_hdist;
}
switch (coro_susp_point) {
PUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
v_bits = self->private_impl.f_bits;
v_n_bits = self->private_impl.f_n_bits;
v_table_entry = 0;
v_table_entry_n_bits = 0;
v_lmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[0]) - 1);
v_dmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[1]) - 1);
label_0_continue:;
while (((uint64_t)(b_wend_dst - b_wptr_dst)) > 999999999) {
while (true) {
v_table_entry = self->private_impl.f_huffs[0][v_bits & v_lmask];
v_table_entry_n_bits = (v_table_entry & 15);
if (v_n_bits >= v_table_entry_n_bits) {
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
goto label_1_break;
}
{
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(1);
if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_0 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_0)) << v_n_bits);
}
v_n_bits += 8;
}
label_1_break:;
if ((v_table_entry >> 31) != 0) {
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(2);
if (b_wptr_dst == b_wend_dst) {
status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE;
goto suspend;
}
*b_wptr_dst++ = ((v_table_entry >> 8) & 255);
goto label_0_continue;
} else if ((v_table_entry >> 30) != 0) {
} else if ((v_table_entry >> 29) != 0) {
self->private_impl.f_end_of_block = true;
goto label_0_break;
} else if ((v_table_entry >> 28) != 0) {
v_redir_top = ((v_table_entry >> 8) & 65535);
v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1);
while (true) {
if ((v_redir_top + (v_bits & v_redir_mask)) >= 1234) {
status =
PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_table_entry =
self->private_impl
.f_huffs[0][v_redir_top + (v_bits & v_redir_mask)];
v_table_entry_n_bits = (v_table_entry & 15);
if (v_n_bits >= v_table_entry_n_bits) {
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
goto label_2_break;
}
{
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(3);
if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_1 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_1)) << v_n_bits);
}
v_n_bits += 8;
}
label_2_break:;
if ((v_table_entry >> 31) != 0) {
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(4);
if (b_wptr_dst == b_wend_dst) {
status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE;
goto suspend;
}
*b_wptr_dst++ = ((v_table_entry >> 8) & 255);
goto label_0_continue;
} else if ((v_table_entry >> 30) != 0) {
} else if ((v_table_entry >> 29) != 0) {
self->private_impl.f_end_of_block = true;
goto label_0_break;
} else if ((v_table_entry >> 28) != 0) {
status =
PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
} else if ((v_table_entry >> 27) != 0) {
status = PUFFS_FLATE__ERROR_BAD_HUFFMAN_CODE;
goto exit;
} else {
status =
PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
} else if ((v_table_entry >> 27) != 0) {
status = PUFFS_FLATE__ERROR_BAD_HUFFMAN_CODE;
goto exit;
} else {
status =
PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_length = ((v_table_entry >> 8) & 32767);
v_table_entry_n_bits = ((v_table_entry >> 4) & 15);
while (v_n_bits < v_table_entry_n_bits) {
{
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(5);
if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_2 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_2)) << v_n_bits);
}
v_n_bits += 8;
}
v_length =
((v_length + ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1))) &
32767);
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
while (true) {
v_table_entry = self->private_impl.f_huffs[1][v_bits & v_dmask];
v_table_entry_n_bits = (v_table_entry & 15);
if (v_n_bits >= v_table_entry_n_bits) {
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
goto label_3_break;
}
{
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(6);
if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_3 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_3)) << v_n_bits);
}
v_n_bits += 8;
}
label_3_break:;
if ((v_table_entry >> 28) == 1) {
v_redir_top = ((v_table_entry >> 8) & 65535);
v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1);
while (true) {
if ((v_redir_top + (v_bits & v_redir_mask)) >= 1234) {
status =
PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_table_entry =
self->private_impl
.f_huffs[1][v_redir_top + (v_bits & v_redir_mask)];
v_table_entry_n_bits = (v_table_entry & 15);
if (v_n_bits >= v_table_entry_n_bits) {
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
goto label_4_break;
}
{
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(7);
if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_4 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_4)) << v_n_bits);
}
v_n_bits += 8;
}
label_4_break:;
if ((v_table_entry >> 31) != 0) {
status =
PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
}
if ((v_table_entry >> 24) != 64) {
if ((v_table_entry >> 24) == 8) {
status = PUFFS_FLATE__ERROR_BAD_HUFFMAN_CODE;
goto exit;
}
status =
PUFFS_FLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_distance = ((v_table_entry >> 8) & 32767);
v_table_entry_n_bits = ((v_table_entry >> 4) & 15);
while (v_n_bits < v_table_entry_n_bits) {
{
PUFFS_BASE__COROUTINE_SUSPENSION_POINT(8);
if (PUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_5 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_5)) << v_n_bits);
}
v_n_bits += 8;
}
v_distance =
((v_distance + ((v_bits) & ((1 << (v_table_entry_n_bits)) - 1))) &
32767);
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
v_n_copied = 0;
while (true) {
if (((uint64_t)(v_distance)) >
((uint64_t)(
((puffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
.len = a_dst.private_impl.mark
? (size_t)(b_wptr_dst - a_dst.private_impl.mark)
: 0,
})
.len))) {
v_hlen = 0;
v_hdist = ((uint32_t)(
(((uint64_t)(v_distance)) -
((uint64_t)(((puffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
.len = a_dst.private_impl.mark
? (size_t)(b_wptr_dst -
a_dst.private_impl.mark)
: 0,
})
.len)))));
if (v_length > v_hdist) {
v_length -= v_hdist;
v_hlen = v_hdist;
} else {
v_hlen = v_length;
v_length = 0;
}
if (self->private_impl.f_history_index < v_hdist) {
status = PUFFS_FLATE__ERROR_BAD_DISTANCE;
goto exit;
}
v_hdist = ((self->private_impl.f_history_index - v_hdist) & 32767);
while (true) {
v_n_copied = puffs_base__writer1__copy_from_slice32(
&b_wptr_dst, b_wend_dst,
puffs_base__slice_u8__subslice_i(
((puffs_base__slice_u8){.ptr = self->private_impl.f_history,
.len = 32768}),
v_hdist),
v_hlen);
if (v_hlen <= v_n_copied) {
v_hlen = 0;
goto label_5_break;
}
if (v_n_copied > 0) {
v_hlen -= v_n_copied;
v_hdist = ((v_hdist + v_n_copied) & 32767);
if (v_hdist == 0) {
goto label_5_break;
}
}
status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE;
PUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9);
}
label_5_break:;
if (v_hlen > 0) {
while (true) {
v_n_copied = puffs_base__writer1__copy_from_slice32(
&b_wptr_dst, b_wend_dst,
puffs_base__slice_u8__subslice_i(
((puffs_base__slice_u8){
.ptr = self->private_impl.f_history, .len = 32768}),
v_hdist),
v_hlen);
if (v_hlen <= v_n_copied) {
v_hlen = 0;
goto label_6_break;
}
v_hlen -= v_n_copied;
v_hdist = ((v_hdist + (v_n_copied & 32767)) & 32767);
status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE;
PUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
}
label_6_break:;
}
if (v_length == 0) {
goto label_0_continue;
}
}
v_n_copied = puffs_base__writer1__copy_from_history32(
&b_wptr_dst, a_dst.private_impl.mark, b_wend_dst, v_distance,
v_length);
if (v_length <= v_n_copied) {
v_length = 0;
goto label_7_break;
}
v_length -= v_n_copied;
status = PUFFS_FLATE__SUSPENSION_SHORT_WRITE;
PUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
}
label_7_break:;
}
label_0_break:;
self->private_impl.f_bits = v_bits;
self->private_impl.f_n_bits = v_n_bits;
goto ok;
ok:
self->private_impl.c_decode_huffman_fast[0].coro_susp_point = 0;
goto exit;
}
goto suspend;
suspend:
self->private_impl.c_decode_huffman_fast[0].coro_susp_point = coro_susp_point;
self->private_impl.c_decode_huffman_fast[0].v_bits = v_bits;
self->private_impl.c_decode_huffman_fast[0].v_n_bits = v_n_bits;
self->private_impl.c_decode_huffman_fast[0].v_table_entry = v_table_entry;
self->private_impl.c_decode_huffman_fast[0].v_table_entry_n_bits =
v_table_entry_n_bits;
self->private_impl.c_decode_huffman_fast[0].v_lmask = v_lmask;
self->private_impl.c_decode_huffman_fast[0].v_dmask = v_dmask;
self->private_impl.c_decode_huffman_fast[0].v_redir_top = v_redir_top;
self->private_impl.c_decode_huffman_fast[0].v_redir_mask = v_redir_mask;
self->private_impl.c_decode_huffman_fast[0].v_length = v_length;
self->private_impl.c_decode_huffman_fast[0].v_distance = v_distance;
self->private_impl.c_decode_huffman_fast[0].v_n_copied = v_n_copied;
self->private_impl.c_decode_huffman_fast[0].v_hlen = v_hlen;
self->private_impl.c_decode_huffman_fast[0].v_hdist = v_hdist;
exit:
if (a_dst.buf) {
size_t n = b_wptr_dst - (a_dst.buf->ptr + a_dst.buf->wi);
a_dst.buf->wi += n;
puffs_base__limit1* lim;
for (lim = &a_dst.private_impl.limit; lim; lim = lim->next) {
if (lim->ptr_to_len) {
*lim->ptr_to_len -= n;
}
}
PUFFS_BASE__IGNORE_POTENTIALLY_UNUSED_VARIABLE(b_wend_dst);
}
if (a_src.buf) {
size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri);
a_src.buf->ri += n;
puffs_base__limit1* lim;
for (lim = &a_src.private_impl.limit; lim; lim = lim->next) {
if (lim->ptr_to_len) {
*lim->ptr_to_len -= n;
}
}
PUFFS_BASE__IGNORE_POTENTIALLY_UNUSED_VARIABLE(b_rend_src);
}
return status;
short_read_src:
if (a_src.buf && a_src.buf->closed && !a_src.private_impl.limit.ptr_to_len) {
status = PUFFS_FLATE__ERROR_UNEXPECTED_EOF;
goto exit;
}
status = PUFFS_FLATE__SUSPENSION_SHORT_READ;
goto suspend;
}
static puffs_flate__status puffs_flate__flate_decoder__decode_huffman_slow(
puffs_flate__flate_decoder* self,
puffs_base__writer1 a_dst,
+16
View File
@@ -235,6 +235,22 @@ typedef struct {
uint8_t v_rep_symbol;
uint32_t v_rep_count;
} c_init_dynamic_huffman[1];
struct {
uint32_t coro_susp_point;
uint32_t v_bits;
uint32_t v_n_bits;
uint32_t v_table_entry;
uint32_t v_table_entry_n_bits;
uint32_t v_lmask;
uint32_t v_dmask;
uint32_t v_redir_top;
uint32_t v_redir_mask;
uint32_t v_length;
uint32_t v_distance;
uint32_t v_n_copied;
uint32_t v_hlen;
uint32_t v_hdist;
} c_decode_huffman_fast[1];
struct {
uint32_t coro_susp_point;
uint32_t v_bits;