From eee67e083d7ec84ae5652680f497b6580c0e3153 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 27 Jul 2017 11:30:43 +1000 Subject: [PATCH] Add array-of-u32 Huffman decoder. --- gen/c/std/flate.c | 104 +++++++++++++++++++++++++++++++- gen/h/std/flate.h | 13 ++++ lang/check/data.go | 20 ++++++ lang/check/gen.go | 1 + std/flate/decode_flate.puffs | 114 +++++++++++++++++++++++++++++++++-- 5 files changed, 245 insertions(+), 7 deletions(-) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 4e052bd8..bdf25035 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -120,6 +120,8 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040119 // 0xbb08f809 #define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040118 // 0xbb08f80a #define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040117 // 0xbb08f80b +#define PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES \ + -1157040116 // 0xbb08f80c bool puffs_flate_status_is_error(puffs_flate_status s); @@ -144,6 +146,8 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; + uint32_t f_huffs[2][512]; + uint32_t f_n_huffs_bits[2]; uint8_t f_code_lengths[316]; uint16_t f_counts[16]; uint16_t f_symbols[290]; @@ -184,6 +188,15 @@ typedef struct { uint16_t v_offsets[16]; uint16_t v_total; uint16_t v_count; + uint32_t v_max_cl; + uint32_t v_n_symbols; + uint32_t v_prev_cl; + uint32_t v_code; + uint32_t v_cl; + uint32_t v_reversed_code; + uint32_t v_value; + uint32_t v_high_bits; + uint32_t v_delta; } c_init_huff[1]; } private_impl; } puffs_flate_decoder; @@ -267,7 +280,7 @@ const char* puffs_flate_status_strings0[11] = { "flate: limited write", }; -const char* puffs_flate_status_strings1[12] = { +const char* puffs_flate_status_strings1[13] = { "flate: bad Huffman code length count", "flate: bad Huffman code length repetition", "flate: bad Huffman code", @@ -280,6 +293,7 @@ const char* puffs_flate_status_strings1[12] = { "flate: missing end-of-block code", "flate: no Huffman codes", "flate: TODO: fixed Huffman blocks", + "flate: TODO: indirect Huffman tables", }; const char* puffs_flate_status_string(puffs_flate_status s) { @@ -292,7 +306,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) { break; case puffs_flate_packageid: a = puffs_flate_status_strings1; - n = 12; + n = 13; break; } uint32_t i = s & 0xff; @@ -1005,6 +1019,15 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, uint16_t v_offsets[16]; uint16_t v_total; uint16_t v_count; + uint32_t v_max_cl; + uint32_t v_n_symbols; + uint32_t v_prev_cl; + uint32_t v_code; + uint32_t v_cl; + uint32_t v_reversed_code; + uint32_t v_value; + uint32_t v_high_bits; + uint32_t v_delta; uint32_t coro_susp_point = self->private_impl.c_init_huff[0].coro_susp_point; if (coro_susp_point) { @@ -1012,6 +1035,15 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, memcpy(v_offsets, self->private_impl.c_init_huff[0].v_offsets, 2 * 16); v_total = self->private_impl.c_init_huff[0].v_total; v_count = self->private_impl.c_init_huff[0].v_count; + v_max_cl = self->private_impl.c_init_huff[0].v_max_cl; + v_n_symbols = self->private_impl.c_init_huff[0].v_n_symbols; + v_prev_cl = self->private_impl.c_init_huff[0].v_prev_cl; + v_code = self->private_impl.c_init_huff[0].v_code; + v_cl = self->private_impl.c_init_huff[0].v_cl; + v_reversed_code = self->private_impl.c_init_huff[0].v_reversed_code; + v_value = self->private_impl.c_init_huff[0].v_value; + v_high_bits = self->private_impl.c_init_huff[0].v_high_bits; + v_delta = self->private_impl.c_init_huff[0].v_delta; } switch (coro_susp_point) { PUFFS_COROUTINE_SUSPENSION_POINT(0); @@ -1070,6 +1102,65 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, } v_i += 1; } + v_max_cl = 15; + while (true) { + if (self->private_impl.f_counts[v_max_cl] != 0) { + goto label_0_break; + } + if (v_max_cl <= 1) { + status = PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES; + goto exit; + } + v_max_cl -= 1; + } + label_0_break:; + if (v_max_cl > 9) { + status = PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES; + goto exit; + } + v_i = 0; + v_n_symbols = ((uint32_t)(v_offsets[v_max_cl])); + v_prev_cl = ((uint32_t)( + self->private_impl.f_code_lengths[self->private_impl.f_symbols[0]])); + v_code = 0; + while (v_i < v_n_symbols) { + v_cl = + ((uint32_t)(self->private_impl + .f_code_lengths[self->private_impl.f_symbols[v_i]])); + if (v_cl > v_max_cl) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + if (v_cl > v_prev_cl) { + v_code <<= (v_cl - v_prev_cl); + if (v_code >= 512) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + } + v_reversed_code = (((uint32_t)(puffs_flate_reverse8[v_code >> 1])) | + ((v_code & 1) << 8)); + v_reversed_code >>= (9 - v_cl); + v_value = ((((uint32_t)(self->private_impl.f_symbols[v_i])) << 8) | v_cl); + v_value |= 1073741824; + v_high_bits = (((uint32_t)(1)) << v_max_cl); + v_delta = (((uint32_t)(1)) << v_cl); + while (v_high_bits >= v_delta) { + v_high_bits -= v_delta; + self->private_impl + .f_huffs[a_which][(v_high_bits | v_reversed_code) & 511] = v_value; + } + v_code += 1; + if (v_code >= 512) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + v_prev_cl = v_cl; + v_i += 1; + } self->private_impl.c_init_huff[0].coro_susp_point = 0; goto exit; } @@ -1081,6 +1172,15 @@ suspend: memcpy(self->private_impl.c_init_huff[0].v_offsets, v_offsets, 2 * 16); self->private_impl.c_init_huff[0].v_total = v_total; self->private_impl.c_init_huff[0].v_count = v_count; + self->private_impl.c_init_huff[0].v_max_cl = v_max_cl; + self->private_impl.c_init_huff[0].v_n_symbols = v_n_symbols; + self->private_impl.c_init_huff[0].v_prev_cl = v_prev_cl; + self->private_impl.c_init_huff[0].v_code = v_code; + self->private_impl.c_init_huff[0].v_cl = v_cl; + self->private_impl.c_init_huff[0].v_reversed_code = v_reversed_code; + self->private_impl.c_init_huff[0].v_value = v_value; + self->private_impl.c_init_huff[0].v_high_bits = v_high_bits; + self->private_impl.c_init_huff[0].v_delta = v_delta; exit: return status; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index fb4ed265..8bf4626c 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -120,6 +120,8 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040119 // 0xbb08f809 #define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040118 // 0xbb08f80a #define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040117 // 0xbb08f80b +#define PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES \ + -1157040116 // 0xbb08f80c bool puffs_flate_status_is_error(puffs_flate_status s); @@ -144,6 +146,8 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; + uint32_t f_huffs[2][512]; + uint32_t f_n_huffs_bits[2]; uint8_t f_code_lengths[316]; uint16_t f_counts[16]; uint16_t f_symbols[290]; @@ -184,6 +188,15 @@ typedef struct { uint16_t v_offsets[16]; uint16_t v_total; uint16_t v_count; + uint32_t v_max_cl; + uint32_t v_n_symbols; + uint32_t v_prev_cl; + uint32_t v_code; + uint32_t v_cl; + uint32_t v_reversed_code; + uint32_t v_value; + uint32_t v_high_bits; + uint32_t v_delta; } c_init_huff[1]; } private_impl; } puffs_flate_decoder; diff --git a/lang/check/data.go b/lang/check/data.go index ecc10d42..392758b1 100644 --- a/lang/check/data.go +++ b/lang/check/data.go @@ -72,6 +72,26 @@ var reasons = [...]struct { return nil }}, + {`"a <= b: a <= c; c <= b"`, func(q *checker, n *a.Assert) error { + op, xa, xb := parseBinaryOp(n.Condition()) + if op.Key() != t.KeyXBinaryLessEq { + return errFailed + } + // a <= c + xc := argValue(q.tm, n.Args(), "c") + if xc == nil { + return errFailed + } + if err := proveReasonRequirement(q, t.IDXBinaryLessEq, xa, xc); err != nil { + return err + } + // c <= b + if err := proveReasonRequirement(q, t.IDXBinaryLessEq, xc, xb); err != nil { + return err + } + return nil + }}, + {`"a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"`, func(q *checker, n *a.Assert) error { op, xa, t0 := parseBinaryOp(n.Condition()) if op.Key() != t.KeyXBinaryLessThan { diff --git a/lang/check/gen.go b/lang/check/gen.go index 1023799b..b4a9dd77 100644 --- a/lang/check/gen.go +++ b/lang/check/gen.go @@ -22,6 +22,7 @@ var reasons = [...]string{ "a < (b + c): a < c; 0 <= b", "(a + b) <= c: a <= (c - b)", "a < b: a < c; c <= b", + "a <= b: a <= c; c <= b", "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c", } diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 3f200ba4..33108e36 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -18,17 +18,44 @@ pub error "missing end-of-block code" pub error "no Huffman codes" pub error "TODO: fixed Huffman blocks" +pub error "TODO: indirect Huffman tables" pub struct decoder?( // These fields yield src's bits in Least Significant Bits order. bits u32, n_bits u32, - // There are up to 2 Huffman tables active at any one time. As per this + // huffs and n_huffs_bits are the lookup tables for Huffman decodings. + // + // There are up to 2 Huffman decoders active at any one time. As per this // package's README.md: // - huffs[0] is used for clcode and lcode. // - huffs[1] is used for dcode. - // TODO: define the huffs field. + // + // The initial table key is the low n_huffs_bits of the decoder.bits field. + // Keys longer than 9 bits require a two step lookup, the first step + // examines the low 9 bits, the second step examines the remaining bits. + // Two steps are required at most, as keys are at most 15 bits long. + // + // Using decoder.bits's low n_huffs_bits as a table key is valid even if + // decoder.n_bits is less than n_huffs_bits, because the immediate next + // step after indexing the table by the key is to compare decoder.n_bits to + // the table value's number of decoder.bits to consume. If it compares + // less, then more source bytes are read and the table lookup re-tried. + // + // The table value is: + // - bit 31 indicates a redirect to another part of the table. + // - bit 30 indicates the symbol is a literal. + // - bit 29 indicates the symbol is a base number + extra bits. + // - bit 28 indicates the symbol is the end-of-block. + // - bits 27-24 is the redirect offset or the number of extra bits. + // - bits 23- 8 is the literal (in bits 15-8) or base number. + // - bits 7- 4 are unused. + // - bits 3- 0 are the number of decoder.bits to consume. + // + // Exactly one of the four bits 31-28 should be set. + huffs[2][512] u32, + n_huffs_bits[2] u32[..9], // These fields are used as scratch space when constructing the huffs // tables. @@ -316,9 +343,6 @@ pri func decoder.init_huffs?(src reader1)() { this.n_bits = n_bits } -// TODO: make init_huff a huffman_decoder method instead of a decoder -// method? - // TODO: make named constants for 15, 19, 289, etc. // TODO: skip generating PUFFS_COROUTINE_SUSPENSION_POINTs for this function, @@ -433,4 +457,84 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() { // - offsets[3] = 7, formerly 0 // - offsets[4] = 8, formerly 7 // - offsets[5] = 10, formerly 8 + + var max_cl u32[1..15] = 15 + while true { + if this.counts[max_cl] != 0 { + break + } + if max_cl <= 1 { + // TODO: when is a degenerate Huffman table valid? + return error "no Huffman codes" + } + max_cl -= 1 + } + if max_cl > 9 { + return error "TODO: indirect Huffman tables" + } + + // For the clcode example in this package's README.md: + // - max_cl = 5 + + i = 0 + var n_symbols u32[..289] = offsets[max_cl] as u32 + var prev_cl u32[..15] = this.code_lengths[this.symbols[0]] as u32 + var code u32 + while i < n_symbols, + inv code < (1 << 9), + inv max_cl <= 9, + { + assert i < 289 via "a < b: a < c; c <= b"(c:n_symbols) + var cl u32[..15] = this.code_lengths[this.symbols[i]] as u32 + if cl > max_cl { + return error "internal error: inconsistent Huffman decoder state" + } + assert cl <= 9 via "a <= b: a <= c; c <= b"(c:max_cl) + if cl > prev_cl { + code <<= cl - prev_cl + if code >= (1 << 9) { + return error "internal error: inconsistent Huffman decoder state" + } + } + + var reversed_code u32[..511] = (reverse8[code >> 1] as u32) | ((code & 1) << 8) + reversed_code >>= 9 - cl + + // TODO: don't assume that the value is a literal. + var value u32 = ((this.symbols[i] as u32) << 8) | cl + value |= 0x40000000 + + // The table uses max_cl bits, but reversed_code only has cl bits. We + // duplicate the key-value pair across all possible values of the high + // (max_cl - cl) bits. + var high_bits u32 = (1 as u32) << max_cl + var delta u32 = (1 as u32) << cl + while high_bits >= delta, + inv code < (1 << 9), + inv i < 289, + inv max_cl <= 9, + { + high_bits -= delta + this.huffs[in.which][(high_bits | reversed_code) & 511] = value + } + + code += 1 + if code >= (1 << 9) { + return error "internal error: inconsistent Huffman decoder state" + } + prev_cl = cl + i += 1 + } + + // For the clcode example in this package's README.md: + // - huffs[0][0b..000] maps to value 0x40000003 (literal, symbols[0]=0x00, code_length=3) + // - huffs[0][0b..100] maps to value 0x40000403 (literal, symbols[1]=0x04, code_length=3) + // - huffs[0][0b..010] maps to value 0x40000503 (literal, symbols[2]=0x05, code_length=3) + // - huffs[0][0b..110] maps to value 0x40000603 (literal, symbols[3]=0x06, code_length=3) + // - huffs[0][0b..001] maps to value 0x40000703 (literal, symbols[4]=0x07, code_length=3) + // - huffs[0][0b..101] maps to value 0x40000803 (literal, symbols[5]=0x08, code_length=3) + // - huffs[0][0b..011] maps to value 0x40000903 (literal, symbols[6]=0x09, code_length=3) + // - huffs[0][0b.0111] maps to value 0x40001104 (literal, symbols[7]=0x11, code_length=4) + // - huffs[0][0b01111] maps to value 0x40000305 (literal, symbols[8]=0x03, code_length=5) + // - huffs[0][0b11111] maps to value 0x40001805 (literal, symbols[9]=0x18, code_length=5) }