From bfbcae43650a865d2d858891d42db8a2e7b64434 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 28 Jul 2017 13:28:31 +1000 Subject: [PATCH] Decode std/flate fixed Huffman tables. --- cmd/puffs-gen-c/func.go | 8 ++ gen/c/std/flate.c | 140 +++++++++++++++++------ gen/h/std/flate.h | 16 +-- lang/check/bounds.go | 3 +- lang/check/type.go | 3 +- std/flate/decode_flate.puffs | 136 +++++++++++++--------- test/c/std/flate.c | 43 ++++--- test/testdata/README.md | 5 +- test/testdata/romeo.txt.fixed-huff.flate | Bin 0 -> 692 bytes 9 files changed, 240 insertions(+), 114 deletions(-) create mode 100644 test/testdata/romeo.txt.fixed-huff.flate diff --git a/cmd/puffs-gen-c/func.go b/cmd/puffs-gen-c/func.go index 046fc597..5a8dee90 100644 --- a/cmd/puffs-gen-c/func.go +++ b/cmd/puffs-gen-c/func.go @@ -1025,6 +1025,14 @@ func (g *gen) writeCallSuspendibles(n *a.Expr, depth uint32) error { } g.writes("if (status) { goto suspend; }\n") + } else if isThisMethod(g.tm, n, "init_fixed_huffman", 0) { + g.printf("status = %s%s_init_fixed_huffman(self);\n", + g.pkgPrefix, g.perFunc.funk.Receiver().String(g.tm)) + if err := g.writeLoadExprDerivedVars(n); err != nil { + return err + } + g.writes("if (status) { goto suspend; }\n") + } else if isThisMethod(g.tm, n, "init_dynamic_huffman", 1) { g.printf("status = %s%s_init_dynamic_huffman(self, %ssrc);\n", g.pkgPrefix, g.perFunc.funk.Receiver().String(g.tm), aPrefix) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 28d9951c..f98dbba5 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -119,9 +119,8 @@ typedef int32_t puffs_flate_status; -1157040120 // 0xbb08f808 #define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040119 // 0xbb08f809 #define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040118 // 0xbb08f80a -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040117 // 0xbb08f80b #define PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES \ - -1157040116 // 0xbb08f80c + -1157040117 // 0xbb08f80b bool puffs_flate_status_is_error(puffs_flate_status s); @@ -148,9 +147,9 @@ typedef struct { uint32_t f_n_bits; uint32_t f_huffs[2][512]; uint32_t f_n_huffs_bits[2]; - uint8_t f_code_lengths[316]; + uint8_t f_code_lengths[320]; uint16_t f_counts[16]; - uint16_t f_symbols[290]; + uint16_t f_symbols[320]; struct { uint32_t coro_susp_point; @@ -172,6 +171,10 @@ typedef struct { uint32_t v_length; uint32_t v_distance; } c_decode_huffman[1]; + struct { + uint32_t coro_susp_point; + uint32_t v_i; + } c_init_fixed_huffman[1]; struct { uint32_t coro_susp_point; uint32_t v_bits; @@ -191,10 +194,9 @@ typedef struct { uint32_t coro_susp_point; uint32_t v_i; uint16_t v_offsets[16]; - uint16_t v_total; - uint16_t v_count; - uint32_t v_max_cl; uint32_t v_n_symbols; + uint32_t v_count; + uint32_t v_max_cl; uint32_t v_prev_cl; uint32_t v_code; uint32_t v_cl; @@ -286,7 +288,7 @@ const char* puffs_flate_status_strings0[11] = { "flate: limited write", }; -const char* puffs_flate_status_strings1[13] = { +const char* puffs_flate_status_strings1[12] = { "flate: bad Huffman code length count", "flate: bad Huffman code length repetition", "flate: bad Huffman code", @@ -298,7 +300,6 @@ const char* puffs_flate_status_strings1[13] = { "flate: internal error: inconsistent n_bits", "flate: missing end-of-block code", "flate: no Huffman codes", - "flate: TODO: fixed Huffman blocks", "flate: TODO: indirect Huffman tables", }; @@ -312,7 +313,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) { break; case puffs_flate_packageid: a = puffs_flate_status_strings1; - n = 13; + n = 12; break; } uint32_t i = s & 0xff; @@ -373,6 +374,9 @@ puffs_flate_status puffs_flate_decoder_decode_huffman(puffs_flate_decoder* self, puffs_base_writer1 a_dst, puffs_base_reader1 a_src); +puffs_flate_status puffs_flate_decoder_init_fixed_huffman( + puffs_flate_decoder* self); + puffs_flate_status puffs_flate_decoder_init_dynamic_huffman( puffs_flate_decoder* self, puffs_base_reader1 a_src); @@ -508,10 +512,13 @@ puffs_flate_status puffs_flate_decoder_decode(puffs_flate_decoder* self, } goto label_0_continue; } else if (v_type == 1) { - status = PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS; - goto exit; - } else if (v_type == 2) { PUFFS_COROUTINE_SUSPENSION_POINT(3); + status = puffs_flate_decoder_init_fixed_huffman(self); + if (status) { + goto suspend; + } + } else if (v_type == 2) { + PUFFS_COROUTINE_SUSPENSION_POINT(4); if (a_src.buf) { size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); a_src.buf->ri += n; @@ -541,7 +548,7 @@ puffs_flate_status puffs_flate_decoder_decode(puffs_flate_decoder* self, status = PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK; goto exit; } - PUFFS_COROUTINE_SUSPENSION_POINT(4); + PUFFS_COROUTINE_SUSPENSION_POINT(5); if (a_src.buf) { size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); a_src.buf->ri += n; @@ -1006,6 +1013,64 @@ short_read_src: goto suspend; } +puffs_flate_status puffs_flate_decoder_init_fixed_huffman( + puffs_flate_decoder* self) { + puffs_flate_status status = PUFFS_FLATE_STATUS_OK; + + uint32_t v_i; + + uint32_t coro_susp_point = + self->private_impl.c_init_fixed_huffman[0].coro_susp_point; + if (coro_susp_point) { + v_i = self->private_impl.c_init_fixed_huffman[0].v_i; + } + switch (coro_susp_point) { + PUFFS_COROUTINE_SUSPENSION_POINT(0); + + v_i = 0; + while (v_i < 144) { + self->private_impl.f_code_lengths[v_i] = 8; + v_i += 1; + } + while (v_i < 256) { + self->private_impl.f_code_lengths[v_i] = 9; + v_i += 1; + } + while (v_i < 280) { + self->private_impl.f_code_lengths[v_i] = 7; + v_i += 1; + } + while (v_i < 288) { + self->private_impl.f_code_lengths[v_i] = 8; + v_i += 1; + } + while (v_i < 320) { + self->private_impl.f_code_lengths[v_i] = 5; + v_i += 1; + } + PUFFS_COROUTINE_SUSPENSION_POINT(1); + status = puffs_flate_decoder_init_huff(self, 0, 0, 288, 257); + if (status) { + goto suspend; + } + PUFFS_COROUTINE_SUSPENSION_POINT(2); + status = puffs_flate_decoder_init_huff(self, 1, 288, 320, 0); + if (status) { + goto suspend; + } + self->private_impl.c_init_fixed_huffman[0].coro_susp_point = 0; + goto exit; + } + + goto suspend; +suspend: + self->private_impl.c_init_fixed_huffman[0].coro_susp_point = coro_susp_point; + self->private_impl.c_init_fixed_huffman[0].v_i = v_i; + +exit: + return status; +} + puffs_flate_status puffs_flate_decoder_init_dynamic_huffman( puffs_flate_decoder* self, puffs_base_reader1 a_src) { @@ -1268,10 +1333,9 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, uint32_t v_i; uint16_t v_offsets[16]; - uint16_t v_total; - uint16_t v_count; - uint32_t v_max_cl; uint32_t v_n_symbols; + uint32_t v_count; + uint32_t v_max_cl; uint32_t v_prev_cl; uint32_t v_code; uint32_t v_cl; @@ -1285,10 +1349,9 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, if (coro_susp_point) { v_i = self->private_impl.c_init_huff[0].v_i; memcpy(v_offsets, self->private_impl.c_init_huff[0].v_offsets, 2 * 16); - v_total = self->private_impl.c_init_huff[0].v_total; + v_n_symbols = self->private_impl.c_init_huff[0].v_n_symbols; v_count = self->private_impl.c_init_huff[0].v_count; v_max_cl = self->private_impl.c_init_huff[0].v_max_cl; - v_n_symbols = self->private_impl.c_init_huff[0].v_n_symbols; v_prev_cl = self->private_impl.c_init_huff[0].v_prev_cl; v_code = self->private_impl.c_init_huff[0].v_code; v_cl = self->private_impl.c_init_huff[0].v_cl; @@ -1309,7 +1372,7 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_i = a_n_codes0; while (v_i < a_n_codes1) { if (self->private_impl.f_counts[self->private_impl.f_code_lengths[v_i]] >= - 289) { + 320) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; @@ -1328,19 +1391,24 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_offsets[i] = 0; } }; - v_total = 0; + v_n_symbols = 0; v_i = 1; while (v_i <= 15) { - v_offsets[v_i] = v_total; - v_count = self->private_impl.f_counts[v_i]; - if (v_total > (289 - v_count)) { + v_offsets[v_i] = ((uint16_t)(v_n_symbols)); + v_count = ((uint32_t)(self->private_impl.f_counts[v_i])); + if (v_n_symbols > (320 - v_count)) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } - v_total = (v_total + v_count); + v_n_symbols = (v_n_symbols + v_count); v_i += 1; } + if (v_n_symbols > 288) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } v_i = a_n_codes0; while (v_i < a_n_codes1) { if (v_i < a_n_codes0) { @@ -1349,14 +1417,14 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, goto exit; } if (self->private_impl.f_code_lengths[v_i] != 0) { - self->private_impl - .f_symbols[v_offsets[self->private_impl.f_code_lengths[v_i]]] = - ((uint16_t)((v_i - a_n_codes0))); - if (v_offsets[self->private_impl.f_code_lengths[v_i]] >= 289) { + if (v_offsets[self->private_impl.f_code_lengths[v_i]] >= 320) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; } + self->private_impl + .f_symbols[v_offsets[self->private_impl.f_code_lengths[v_i]]] = + ((uint16_t)((v_i - a_n_codes0))); v_offsets[self->private_impl.f_code_lengths[v_i]] += 1; } v_i += 1; @@ -1378,8 +1446,13 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, goto exit; } v_i = 0; - v_n_symbols = ((uint32_t)(v_offsets[v_max_cl])); - if ((a_n_codes0 + ((uint32_t)(self->private_impl.f_symbols[0]))) >= 316) { + if ((v_n_symbols != ((uint32_t)(v_offsets[v_max_cl]))) || + (v_n_symbols != ((uint32_t)(v_offsets[15])))) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } + if ((a_n_codes0 + ((uint32_t)(self->private_impl.f_symbols[0]))) >= 320) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; @@ -1391,7 +1464,7 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_code = 0; while (true) { if ((a_n_codes0 + ((uint32_t)(self->private_impl.f_symbols[v_i]))) >= - 316) { + 320) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; goto exit; @@ -1463,10 +1536,9 @@ suspend: self->private_impl.c_init_huff[0].coro_susp_point = coro_susp_point; self->private_impl.c_init_huff[0].v_i = v_i; memcpy(self->private_impl.c_init_huff[0].v_offsets, v_offsets, 2 * 16); - self->private_impl.c_init_huff[0].v_total = v_total; + self->private_impl.c_init_huff[0].v_n_symbols = v_n_symbols; self->private_impl.c_init_huff[0].v_count = v_count; self->private_impl.c_init_huff[0].v_max_cl = v_max_cl; - self->private_impl.c_init_huff[0].v_n_symbols = v_n_symbols; self->private_impl.c_init_huff[0].v_prev_cl = v_prev_cl; self->private_impl.c_init_huff[0].v_code = v_code; self->private_impl.c_init_huff[0].v_cl = v_cl; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index bdcdfa97..c7fe1cdd 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -119,9 +119,8 @@ typedef int32_t puffs_flate_status; -1157040120 // 0xbb08f808 #define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040119 // 0xbb08f809 #define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040118 // 0xbb08f80a -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040117 // 0xbb08f80b #define PUFFS_FLATE_ERROR_TODO_INDIRECT_HUFFMAN_TABLES \ - -1157040116 // 0xbb08f80c + -1157040117 // 0xbb08f80b bool puffs_flate_status_is_error(puffs_flate_status s); @@ -148,9 +147,9 @@ typedef struct { uint32_t f_n_bits; uint32_t f_huffs[2][512]; uint32_t f_n_huffs_bits[2]; - uint8_t f_code_lengths[316]; + uint8_t f_code_lengths[320]; uint16_t f_counts[16]; - uint16_t f_symbols[290]; + uint16_t f_symbols[320]; struct { uint32_t coro_susp_point; @@ -172,6 +171,10 @@ typedef struct { uint32_t v_length; uint32_t v_distance; } c_decode_huffman[1]; + struct { + uint32_t coro_susp_point; + uint32_t v_i; + } c_init_fixed_huffman[1]; struct { uint32_t coro_susp_point; uint32_t v_bits; @@ -191,10 +194,9 @@ typedef struct { uint32_t coro_susp_point; uint32_t v_i; uint16_t v_offsets[16]; - uint16_t v_total; - uint16_t v_count; - uint32_t v_max_cl; uint32_t v_n_symbols; + uint32_t v_count; + uint32_t v_max_cl; uint32_t v_prev_cl; uint32_t v_code; uint32_t v_cl; diff --git a/lang/check/bounds.go b/lang/check/bounds.go index f461b7ec..589a4ed0 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -650,7 +650,8 @@ func (q *checker) bcheckExprOther(n *a.Expr, depth uint32) (*big.Int, *big.Int, isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_huffman", 2) || - isThisMethod(q.tm, n, "init_dynamic_huffman", 1) || isThisMethod(q.tm, n, "init_huff", 4) { + isThisMethod(q.tm, n, "init_fixed_huffman", 0) || isThisMethod(q.tm, n, "init_dynamic_huffman", 1) || + isThisMethod(q.tm, n, "init_huff", 4) { for _, o := range n.Args() { // TODO: check that the arg range at the caller and the diff --git a/lang/check/type.go b/lang/check/type.go index 08e85e0e..8838ed45 100644 --- a/lang/check/type.go +++ b/lang/check/type.go @@ -350,7 +350,8 @@ func (q *checker) tcheckExprOther(n *a.Expr, depth uint32) error { isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_huffman", 2) || - isThisMethod(q.tm, n, "init_dynamic_huffman", 1) || isThisMethod(q.tm, n, "init_huff", 4) { + isThisMethod(q.tm, n, "init_fixed_huffman", 0) || isThisMethod(q.tm, n, "init_dynamic_huffman", 1) || + isThisMethod(q.tm, n, "init_huff", 4) { if err := q.tcheckExpr(n.LHS().Expr(), depth); err != nil { return err diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index d89071af..139fa514 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -17,7 +17,6 @@ pub error "internal error: inconsistent n_bits" pub error "missing end-of-block code" pub error "no Huffman codes" -pub error "TODO: fixed Huffman blocks" pub error "TODO: indirect Huffman tables" // The next two tables were created by script/print-flate-magic-numbers.go. @@ -82,9 +81,9 @@ pub struct decoder?( // These fields are used as scratch space when constructing the huffs // tables. - code_lengths[316] u8[..15], - counts[16] u16[..289], - symbols[290] u16[..289], + code_lengths[320] u8[..15], + counts[16] u16[..320], + symbols[320] u16[..319], ) pub func decoder.decode?(dst writer1, src reader1)() { @@ -105,7 +104,7 @@ pub func decoder.decode?(dst writer1, src reader1)() { this.decode_uncompressed?(dst:in.dst, src:in.src) continue } else if type == 1 { - return error "TODO: fixed Huffman blocks" + this.init_fixed_huffman?() } else if type == 2 { this.init_dynamic_huffman?(src:in.src) } else { @@ -115,6 +114,8 @@ pub func decoder.decode?(dst writer1, src reader1)() { } } +// decode_uncompressed decodes an uncompresed block as per the RFC section +// 3.2.4. pri func decoder.decode_uncompressed?(dst writer1, src reader1)() { if this.n_bits >= 8 { return error "internal error: inconsistent n_bits" @@ -241,6 +242,34 @@ pri func decoder.decode_huffman?(dst writer1, src reader1)() { this.n_bits = n_bits } +// init_fixed_huffman initializes this.huffs as per the RFC section 3.2.6. +pri func decoder.init_fixed_huffman?()() { + var i u32 + while i < 144 { + this.code_lengths[i] = 8 + i += 1 + } + while i < 256 { + this.code_lengths[i] = 9 + i += 1 + } + while i < 280 { + this.code_lengths[i] = 7 + i += 1 + } + while i < 288 { + this.code_lengths[i] = 8 + i += 1 + } + while i < 320 { + this.code_lengths[i] = 5 + i += 1 + } + this.init_huff?(which:0, n_codes0:0, n_codes1:288, base_symbol:257) + this.init_huff?(which:1, n_codes0:288, n_codes1:320, base_symbol:0) +} + +// init_dynamic_huffman initializes this.huffs as per the RFC section 3.2.7. pri func decoder.init_dynamic_huffman?(src reader1)() { var bits u32 = this.bits var n_bits u32 = this.n_bits @@ -265,15 +294,10 @@ pri func decoder.init_dynamic_huffman?(src reader1)() { n_bits -= 14 // Read the clcode Huffman table: H-CL. - var i u32 = 0 - while i < n_clen, - inv n_dist <= 30, - inv n_lit <= 286, - { + var i u32 + while i < n_clen { while n_bits < 3, inv i < n_clen, - inv n_dist <= 30, - inv n_lit <= 286, post n_bits >= 3, { bits |= (in.src.read_u8?() as u32) << n_bits @@ -285,10 +309,7 @@ pri func decoder.init_dynamic_huffman?(src reader1)() { n_bits -= 3 i += 1 } - while i < 19, - inv n_dist <= 30, - inv n_lit <= 286, - { + while i < 19 { this.code_lengths[code_order[i]] = 0 i += 1 } @@ -297,18 +318,13 @@ pri func decoder.init_dynamic_huffman?(src reader1)() { // Decode the code lengths for the next two Huffman tables. var mask u32[..511] = ((1 as u32) << this.n_huffs_bits[0]) - 1 i = 0 - while i < (n_lit + n_dist), - inv n_dist <= 30, - inv n_lit <= 286, - { - assert i < (286 + 30) via "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"(b0:n_lit, c0:n_dist) + while i < (n_lit + n_dist) { + assert i < (288 + 32) via "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"(b0:n_lit, c0:n_dist) // Decode a clcode symbol from H-CL. var table_entry u32 while true, - inv i < 316, - inv n_dist <= 30, - inv n_lit <= 286, + inv i < 320, { table_entry = this.huffs[0][bits & mask] var table_entry_n_bits u32[..15] = table_entry & 15 @@ -361,10 +377,8 @@ pri func decoder.init_dynamic_huffman?(src reader1)() { return error "internal error: inconsistent Huffman decoder state" } while n_bits < n_extra_bits, - inv i < 316, + inv i < 320, inv rep_count <= 11, - inv n_dist <= 30, - inv n_lit <= 286, post n_bits >= n_extra_bits, { assert n_bits < 7 via "a < b: a < c; c <= b"(c:n_extra_bits) @@ -375,15 +389,12 @@ pri func decoder.init_dynamic_huffman?(src reader1)() { bits >>= n_extra_bits n_bits -= n_extra_bits - while rep_count > 0, - inv n_dist <= 30, - inv n_lit <= 286, - { + while rep_count > 0 { // TODO: hoist this check up one level? if i >= (n_lit + n_dist) { return error "bad Huffman code length count" } - assert i < (286 + 30) via "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"(b0:n_lit, c0:n_dist) + assert i < (288 + 32) via "a < (b + c): a < (b0 + c0); b0 <= b; c0 <= c"(b0:n_lit, c0:n_dist) this.code_lengths[i] = rep_symbol i += 1 rep_count -= 1 @@ -404,12 +415,12 @@ pri func decoder.init_dynamic_huffman?(src reader1)() { this.n_bits = n_bits } -// TODO: make named constants for 15, 19, 289, etc. +// TODO: make named constants for 15, 19, 319, etc. // TODO: skip generating PUFFS_COROUTINE_SUSPENSION_POINTs for this function, // and its callers, as it can only return errors, not suspensions. -pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[..289], base_symbol u32)() { +pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..320], n_codes1 u32[..320], base_symbol u32)() { // For the clcode example in this package's README.md: // - n_codes0 = 0 // - n_codes1 = 19 @@ -440,10 +451,10 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. } i = in.n_codes0 while i < in.n_codes1 { - assert i < 289 via "a < b: a < c; c <= b"(c:in.n_codes1) + assert i < 320 via "a < b: a < c; c <= b"(c:in.n_codes1) // TODO: this if should be unnecessary. Have some way to assert that, // for all j, counts[j] <= i, and thus counts[j]++ will not overflow. - if this.counts[this.code_lengths[i]] >= 289 { + if this.counts[this.code_lengths[i]] >= 320 { return error "internal error: inconsistent Huffman decoder state" } this.counts[this.code_lengths[i]] += 1 @@ -464,21 +475,24 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. // TODO: check for the total code_lengths being too short or too long. - var offsets[16] u16[..289] - var total u16[..289] + var offsets[16] u16[..320] + var n_symbols u32[..320] i = 1 while i <= 15 { - offsets[i] = total - var count u16[..289] = this.counts[i] - if total > (289 - count) { + offsets[i] = n_symbols as u16 + var count u32[..320] = this.counts[i] as u32 + if n_symbols > (320 - count) { return error "internal error: inconsistent Huffman decoder state" } - assert (total + count) <= 289 via "(a + b) <= c: a <= (c - b)"() - // TODO: change this to total += count, once the proof engine's bounds - // checking can handle it. - total = total + count + assert (n_symbols + count) <= 320 via "(a + b) <= c: a <= (c - b)"() + // TODO: change this to n_symbols += count, once the proof engine's + // bounds checking can handle it. + n_symbols = n_symbols + count i += 1 } + if n_symbols > 288 { + return error "internal error: inconsistent Huffman decoder state" + } // For the clcode example in this package's README.md: // - offsets[0] = 0 @@ -491,17 +505,19 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. // - all other offsets elements are 10. i = in.n_codes0 - while i < in.n_codes1 { - assert i < 289 via "a < b: a < c; c <= b"(c:in.n_codes1) + while i < in.n_codes1, + inv n_symbols <= 288, + { + assert i < 320 via "a < b: a < c; c <= b"(c:in.n_codes1) // TODO: this if check should be unnecessary. if i < in.n_codes0 { return error "internal error: inconsistent Huffman decoder state" } if this.code_lengths[i] != 0 { - this.symbols[offsets[this.code_lengths[i]]] = (i - in.n_codes0) as u16 - if offsets[this.code_lengths[i]] >= 289 { + if offsets[this.code_lengths[i]] >= 320 { return error "internal error: inconsistent Huffman decoder state" } + this.symbols[offsets[this.code_lengths[i]]] = (i - in.n_codes0) as u16 offsets[this.code_lengths[i]] += 1 } i += 1 @@ -525,7 +541,9 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. // - offsets[5] = 10, formerly 8 var max_cl u32[1..15] = 15 - while true { + while true, + inv n_symbols <= 288, + { if this.counts[max_cl] != 0 { break } @@ -543,18 +561,21 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. // - max_cl = 5 i = 0 - var n_symbols u32[..289] = offsets[max_cl] as u32 - if (in.n_codes0 + (this.symbols[0] as u32)) >= 316 { + if (n_symbols != (offsets[max_cl] as u32)) or (n_symbols != (offsets[15] as u32)) { + return error "internal error: inconsistent Huffman decoder state" + } + if (in.n_codes0 + (this.symbols[0] as u32)) >= 320 { return error "internal error: inconsistent Huffman decoder state" } var prev_cl u32[..15] = this.code_lengths[in.n_codes0 + (this.symbols[0] as u32)] as u32 var code u32 while true, pre code < (1 << 9), - pre i < 289, + pre i < 288, inv max_cl <= 9, + inv n_symbols <= 288, { - if (in.n_codes0 + (this.symbols[i] as u32)) >= 316 { + if (in.n_codes0 + (this.symbols[i] as u32)) >= 320 { return error "internal error: inconsistent Huffman decoder state" } var cl u32[..15] = this.code_lengths[in.n_codes0 + (this.symbols[i] as u32)] as u32 @@ -573,7 +594,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. reversed_code >>= 9 - cl var value u32 - var symbol u32[..289] = this.symbols[i] as u32 + var symbol u32[..319] = this.symbols[i] as u32 if symbol == 256 { // End-of-block. value = 0x10000000 | cl @@ -599,8 +620,9 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. var delta u32 = (1 as u32) << cl while high_bits >= delta, inv code < (1 << 9), - inv i < 289, + inv i < 288, inv max_cl <= 9, + inv n_symbols <= 288, { high_bits -= delta this.huffs[in.which][(high_bits | reversed_code) & 511] = value @@ -610,7 +632,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. if i >= n_symbols { break } - assert i < 289 via "a < b: a < c; c <= b"(c:n_symbols) + assert i < 288 via "a < b: a < c; c <= b"(c:n_symbols) code += 1 if code >= (1 << 9) { return error "internal error: inconsistent Huffman decoder state" diff --git a/test/c/std/flate.c b/test/c/std/flate.c index 65eb18dd..a77ec601 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -75,6 +75,11 @@ golden_test flate_romeo_gt = { .src_offset1 = 550, // }; +golden_test flate_romeo_fixed_gt = { + .want_filename = "../../testdata/romeo.txt", // + .src_filename = "../../testdata/romeo.txt.fixed-huff.flate", // +}; + golden_test gzip_midsummer_gt = { .want_filename = "../../testdata/midsummer.txt", // .src_filename = "../../testdata/midsummer.txt.gz", // @@ -131,6 +136,11 @@ void test_puffs_flate_decode_romeo() { test_buf1_buf1(puffs_flate_decode, &flate_romeo_gt); } +void test_puffs_flate_decode_romeo_fixed() { + proc_funcname = __func__; + test_buf1_buf1(puffs_flate_decode, &flate_romeo_fixed_gt); +} + void test_puffs_flate_decode_split_src() { proc_funcname = __func__; @@ -223,6 +233,11 @@ void test_mimic_flate_decode_romeo() { test_buf1_buf1(mimic_flate_decode, &flate_romeo_gt); } +void test_mimic_flate_decode_romeo_fixed() { + proc_funcname = __func__; + test_buf1_buf1(mimic_flate_decode, &flate_romeo_fixed_gt); +} + void test_mimic_gzip_decode_midsummer() { proc_funcname = __func__; test_buf1_buf1(mimic_gzip_decode, &gzip_midsummer_gt); @@ -328,25 +343,27 @@ void bench_mimic_zlib_decode_100k() { // The empty comments forces clang-format to place one element per line. proc tests[] = { // Flate Tests - test_puffs_flate_decode_256_bytes, // + test_puffs_flate_decode_256_bytes, // /* TODO: implement puffs_flate, then uncomment these. - test_puffs_flate_decode_midsummer, // - test_puffs_flate_decode_pi, // + test_puffs_flate_decode_midsummer, // + test_puffs_flate_decode_pi, // */ - test_puffs_flate_decode_romeo, // - test_puffs_flate_decode_split_src, // + test_puffs_flate_decode_romeo, // + test_puffs_flate_decode_romeo_fixed, // + test_puffs_flate_decode_split_src, // #ifdef PUFFS_MIMIC // Mimic Tests - test_mimic_flate_decode_256_bytes, // - test_mimic_flate_decode_midsummer, // - test_mimic_flate_decode_pi, // - test_mimic_flate_decode_romeo, // - test_mimic_gzip_decode_midsummer, // - test_mimic_gzip_decode_pi, // - test_mimic_zlib_decode_midsummer, // - test_mimic_zlib_decode_pi, // + test_mimic_flate_decode_256_bytes, // + test_mimic_flate_decode_midsummer, // + test_mimic_flate_decode_pi, // + test_mimic_flate_decode_romeo, // + test_mimic_flate_decode_romeo_fixed, // + test_mimic_gzip_decode_midsummer, // + test_mimic_gzip_decode_pi, // + test_mimic_zlib_decode_midsummer, // + test_mimic_zlib_decode_pi, // #endif // PUFFS_MIMIC diff --git a/test/testdata/README.md b/test/testdata/README.md index c44dd720..75c93b80 100644 --- a/test/testdata/README.md +++ b/test/testdata/README.md @@ -1,5 +1,5 @@ Unless otherwise noted, the *.gz files were generated by the gzip command line -tool and the .flate and .zlib versions were then generated by +tool and the *.flate and *.zlib versions were then generated by script/extract-flate-offsets.go. Similarly, the *.giflzw files were generated by script/extract-giflzw.go and the *.palette and *.indexes files were generated by script/extract-palette-indexes.go @@ -48,3 +48,6 @@ original photo of Peter J. Weinberger by Rob Pike . romeo.txt is an excerpt of Shakespeare's "Romeo and Juliet", copied from http://shakespeare.mit.edu/romeo_juliet/romeo_juliet.2.2.html + +romeo.txt.fixed-huff.flate was derived from romeo.txt by a custom program to +use fixed (not dynamic) Huffman tables for the flate encoding. diff --git a/test/testdata/romeo.txt.fixed-huff.flate b/test/testdata/romeo.txt.fixed-huff.flate new file mode 100644 index 0000000000000000000000000000000000000000..81de83029fb939fa01a2e4360be6ed8e93669891 GIT binary patch literal 692 zcmd-&b>8=ExW^gqzzmtA=%Ib;d}x7>UZBCHKn_2@v)-Wwj~;#E z`ueei_u;2-hV09Y0bN{AC0Dwx%>3!PlB?5Il}lCf=TZU5YQ5=K_IR~U>hWsba%57^ z6{QWXt}8Q_9$lGmanF%EA=!HWm1JAbm~`y>bY_lF>z+SCvaL%(T9;fBYJK$S&mx=F zJ6^4G{%i`7)jPgNgw_A+9GTWVHLY|0?D=vF3q_jWUIGMp*}vmeqF}# zM`xy7iD-S$b7YPTtAA_Hl_N^6cm6C2k!}5>)S9#+qtG!cX`7SMiL*10ek=IDvOPtt zS+{*>;6fGUjV_9tC7$$M*>mNNSN1xA3E7JG*6R43^}Niu+uoM>Nkyv*>-|4_CLQ^5 zDa3Z&hn^!sUz2t;>~&JRv1HF5rPiJ!dqiXnv~}(GyR@Iy-|wNgQA1p7%8WT&iK@>+ zwVpILuUFW(qUE%v=#!6U_LQ_vs%dp;y>rCImbJd1LiG9bWlo_@F>5(CTO0c;3hE^B zfApwyODk$~s$At{q#+?B+V!bBc;_6gi8@TxyKMHGsJkw-Fc)32B4XhV8`G^vW@%f< z3+)uhlx_WVWl>0LPsoP-ib4v(SA$D-GzEDJ8Glv?Th?%Bzrt1xUDKT^#`gTLT6#iS z|9rZVwC;mXz=Np{Q>LX%d$Q!!!ATKbbx~3_ata^kPm5;>rlog2V)9Y4o+@c?!@WAwuDXS>r}3%KV3D$V$LNTosh!rm;2xemtU^uO0LS4 zPq{ohSyg9-uGG91Be$~h@du7uWv_SXl6g*EsbCbpcA7Kl%q zBBrgfR7_ob>9S`lmp44p&^NYS%p$VysOr+4o|Y?*s&f5&uw?$6_*vm|W+k6z