diff --git a/cmd/puffs-gen-c/func.go b/cmd/puffs-gen-c/func.go index f0ce735b..e77e8bca 100644 --- a/cmd/puffs-gen-c/func.go +++ b/cmd/puffs-gen-c/func.go @@ -977,7 +977,7 @@ func (g *gen) writeCallSuspendibles(n *a.Expr, depth uint32) error { } g.writes("if (status) { goto suspend; }\n") - } else if isThisMethod(g.tm, n, "init_huff", 3) { + } else if isThisMethod(g.tm, n, "init_huff", 4) { g.printf("status = %s%s_init_huff(self,", g.pkgPrefix, g.perFunc.funk.Receiver().String(g.tm)) for i, o := range n.Args() { diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 496302f7..8e267c69 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -341,6 +341,22 @@ static const uint8_t puffs_flate_reverse8[256] = { 15, 143, 79, 207, 47, 175, 111, 239, 31, 159, 95, 223, 63, 191, 127, 255, }; +static const uint32_t puffs_flate_lcode_magic_numbers[32] = { + 536871680, 536871936, 536872192, 536872448, 536872704, 536872960, 536873216, + 536873472, 536873744, 536874256, 536874768, 536875280, 536875808, 536876832, + 536877856, 536878880, 536879920, 536881968, 536884016, 536886064, 536888128, + 536892224, 536896320, 536900416, 536904528, 536912720, 536920912, 536929104, + 536936960, 134217728, 134217728, 134217728, +}; + +static const uint32_t puffs_flate_dcode_magic_numbers[32] = { + 536871168, 536871424, 536871680, 536871936, 536872208, 536872720, 536873248, + 536874272, 536875312, 536877360, 536879424, 536883520, 536887632, 536895824, + 536904032, 536920416, 536936816, 536969584, 537002368, 537067904, 537133456, + 537264528, 537395616, 537657760, 537919920, 538444208, 538968512, 540017088, + 541065680, 543162832, 134217728, 134217728, +}; + // ---------------- Private Constructor and Destructor Prototypes // ---------------- Private Function Prototypes @@ -360,7 +376,8 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, uint32_t a_which, uint32_t a_n_codes0, - uint32_t a_n_codes1); + uint32_t a_n_codes1, + uint32_t a_base_symbol); // ---------------- Constructor and Destructor Implementations @@ -818,6 +835,9 @@ puffs_flate_status puffs_flate_decoder_decode_dynamic( } else if ((v_table_entry >> 29) != 0) { } else if ((v_table_entry >> 28) != 0) { goto label_0_break; + } else if ((v_table_entry >> 27) != 0) { + status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE; + goto exit; } else { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; @@ -976,7 +996,7 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, v_i += 1; } PUFFS_COROUTINE_SUSPENSION_POINT(3); - status = puffs_flate_decoder_init_huff(self, 0, 0, 19); + status = puffs_flate_decoder_init_huff(self, 0, 0, 19, 4095); if (status) { goto suspend; } @@ -1068,13 +1088,13 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self, goto exit; } PUFFS_COROUTINE_SUSPENSION_POINT(6); - status = puffs_flate_decoder_init_huff(self, 0, 0, v_n_lit); + status = puffs_flate_decoder_init_huff(self, 0, 0, v_n_lit, 257); if (status) { goto suspend; } PUFFS_COROUTINE_SUSPENSION_POINT(7); - status = - puffs_flate_decoder_init_huff(self, 1, v_n_lit, (v_n_lit + v_n_dist)); + status = puffs_flate_decoder_init_huff(self, 1, v_n_lit, + (v_n_lit + v_n_dist), 0); if (status) { goto suspend; } @@ -1130,7 +1150,8 @@ short_read_src: puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, uint32_t a_which, uint32_t a_n_codes0, - uint32_t a_n_codes1) { + uint32_t a_n_codes1, + uint32_t a_base_symbol) { puffs_flate_status status = PUFFS_FLATE_STATUS_OK; uint32_t v_i; @@ -1210,10 +1231,15 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, } v_i = a_n_codes0; while (v_i < a_n_codes1) { + if (v_i < a_n_codes0) { + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; + } if (self->private_impl.f_code_lengths[v_i] != 0) { self->private_impl .f_symbols[v_offsets[self->private_impl.f_code_lengths[v_i]]] = - ((uint16_t)(v_i)); + ((uint16_t)((v_i - a_n_codes0))); if (v_offsets[self->private_impl.f_code_lengths[v_i]] >= 289) { status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; @@ -1270,8 +1296,17 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self, v_value = (268435456 | v_cl); } else if ((v_symbol < 256) && (a_which == 0)) { v_value = (1073741824 | (v_symbol << 8) | v_cl); + } else if (v_symbol >= a_base_symbol) { + v_symbol -= a_base_symbol; + if (a_which == 0) { + v_value = (puffs_flate_lcode_magic_numbers[v_symbol & 31] | v_cl); + } else { + v_value = (puffs_flate_dcode_magic_numbers[v_symbol & 31] | v_cl); + } } else { - v_value = (536870912 | v_cl); + status = + PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE; + goto exit; } v_high_bits = (((uint32_t)(1)) << v_max_cl); v_delta = (((uint32_t)(1)) << v_cl); diff --git a/lang/check/bounds.go b/lang/check/bounds.go index c4f366c7..588f6c6b 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -650,7 +650,7 @@ func (q *checker) bcheckExprOther(n *a.Expr, depth uint32) (*big.Int, *big.Int, isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_dynamic", 2) || - isThisMethod(q.tm, n, "init_huffs", 1) || isThisMethod(q.tm, n, "init_huff", 3) { + isThisMethod(q.tm, n, "init_huffs", 1) || isThisMethod(q.tm, n, "init_huff", 4) { for _, o := range n.Args() { // TODO: check that the arg range at the caller and the diff --git a/lang/check/type.go b/lang/check/type.go index 7c86daa9..b0a5c252 100644 --- a/lang/check/type.go +++ b/lang/check/type.go @@ -350,7 +350,7 @@ func (q *checker) tcheckExprOther(n *a.Expr, depth uint32) error { isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_dynamic", 2) || - isThisMethod(q.tm, n, "init_huffs", 1) || isThisMethod(q.tm, n, "init_huff", 3) { + isThisMethod(q.tm, n, "init_huffs", 1) || isThisMethod(q.tm, n, "init_huff", 4) { if err := q.tcheckExpr(n.LHS().Expr(), depth); err != nil { return err diff --git a/script/print-flate-magic-numbers.go b/script/print-flate-magic-numbers.go new file mode 100644 index 00000000..17af14df --- /dev/null +++ b/script/print-flate-magic-numbers.go @@ -0,0 +1,70 @@ +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +// +build ignore + +package main + +// print-flate-magic-numbers.go prints the std/flate lcode_magic_numbers values +// based on the tables in RFC 1951 secion 3.2.5. +// +// Usage: go run print-flate-magic-numbers.go + +import ( + "fmt" + "os" +) + +func main() { + if err := main1(); err != nil { + os.Stderr.WriteString(err.Error() + "\n") + os.Exit(1) + } +} + +func main1() error { + for i := 0; i < 2; i++ { + if i != 0 { + fmt.Println() + } + for j := 0; j < 32; j++ { + x := uint32(0x08000000) + if baseNumbers[i][j] != bad { + x = 0x20000000 | (baseNumbers[i][j] << 8) | (extraBits[i][j] << 4) + } + fmt.Printf("0x%08X,", x) + if j&7 == 7 { + fmt.Println() + } + } + } + return nil +} + +const bad = 0xFFFFFFFF + +var ( + baseNumbers = [2][32]uint32{{ + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, + 131, 163, 195, 227, 258, bad, bad, bad, + }, { + 1, 2, 3, 4, 5, 7, 9, 13, + 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, + 4097, 6145, 8193, 12289, 16385, 24577, bad, bad, + }} + + extraBits = [2][32]uint32{{ + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 4, 4, 4, 4, + 5, 5, 5, 5, 0, bad, bad, bad, + }, { + 0, 0, 0, 0, 1, 1, 2, 2, + 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13, bad, bad, + }} +) diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index 12cee235..2234a805 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -20,6 +20,28 @@ pub error "no Huffman codes" pub error "TODO: fixed Huffman blocks" pub error "TODO: indirect Huffman tables" +// The next two tables were created by script/print-flate-magic-numbers.go. +// +// The u32 values' meanings are the same as the decoder.huffs u32 values. In +// particular, bit 29 indicates a base number + extra bits, bits 23-8 are the +// base number and bits 7-4 are the number of those extra bits. +// +// Some trailing elements are 0x08000000. Bit 27 indicates an invalid value. + +pri const lcode_magic_numbers[32] u32 = $( + 0x20000300, 0x20000400, 0x20000500, 0x20000600, 0x20000700, 0x20000800, 0x20000900, 0x20000A00, + 0x20000B10, 0x20000D10, 0x20000F10, 0x20001110, 0x20001320, 0x20001720, 0x20001B20, 0x20001F20, + 0x20002330, 0x20002B30, 0x20003330, 0x20003B30, 0x20004340, 0x20005340, 0x20006340, 0x20007340, + 0x20008350, 0x2000A350, 0x2000C350, 0x2000E350, 0x20010200, 0x08000000, 0x08000000, 0x08000000, +) + +pri const dcode_magic_numbers[32] u32 = $( + 0x20000100, 0x20000200, 0x20000300, 0x20000400, 0x20000510, 0x20000710, 0x20000920, 0x20000D20, + 0x20001130, 0x20001930, 0x20002140, 0x20003140, 0x20004150, 0x20006150, 0x20008160, 0x2000C160, + 0x20010170, 0x20018170, 0x20020180, 0x20030180, 0x20040190, 0x20060190, 0x200801A0, 0x200C01A0, + 0x201001B0, 0x201801B0, 0x202001C0, 0x203001C0, 0x204001D0, 0x206001D0, 0x08000000, 0x08000000, +) + pub struct decoder?( // These fields yield src's bits in Least Significant Bits order. bits u32, @@ -48,12 +70,13 @@ pub struct decoder?( // - bit 30 indicates the symbol is a literal. // - bit 29 indicates the symbol is a base number + extra bits. // - bit 28 indicates the symbol is the end-of-block. - // - bits 27-24 is the redirect offset or the number of extra bits. + // - bit 27 indicates an invalid value. + // - bits 26-24 are zero. // - bits 23- 8 is the literal (in bits 15-8) or base number. - // - bits 7- 4 are unused. + // - bits 7- 4 is the redirect offset or the number of extra bits. // - bits 3- 0 are the number of decoder.bits to consume. // - // Exactly one of the four bits 31-28 should be set. + // Exactly one of the eight bits 31-24 should be set. huffs[2][512] u32, n_huffs_bits[2] u32[..9], @@ -141,6 +164,8 @@ pri func decoder.decode_dynamic?(dst writer1, src reader1)() { } else if (table_entry >> 28) != 0 { // End of block. break + } else if (table_entry >> 27) != 0 { + return error "bad Huffman code" } else { return error "internal error: inconsistent Huffman decoder state" } @@ -204,7 +229,7 @@ pri func decoder.init_huffs?(src reader1)() { this.code_lengths[code_order[i]] = 0 i += 1 } - this.init_huff?(which:0, n_codes0:0, n_codes1:19) + this.init_huff?(which:0, n_codes0:0, n_codes1:19, base_symbol:0xFFF) // Decode the code lengths for the next two Huffman tables. var mask u32[..511] = ((1 as u32) << this.n_huffs_bits[0]) - 1 @@ -308,8 +333,8 @@ pri func decoder.init_huffs?(src reader1)() { return error "missing end-of-block code" } - this.init_huff?(which:0, n_codes0:0, n_codes1:n_lit) - this.init_huff?(which:1, n_codes0:n_lit, n_codes1:n_lit + n_dist) + this.init_huff?(which:0, n_codes0:0, n_codes1:n_lit, base_symbol:257) + this.init_huff?(which:1, n_codes0:n_lit, n_codes1:n_lit + n_dist, base_symbol:0) this.bits = bits this.n_bits = n_bits @@ -320,7 +345,7 @@ pri func decoder.init_huffs?(src reader1)() { // TODO: skip generating PUFFS_COROUTINE_SUSPENSION_POINTs for this function, // and its callers, as it can only return errors, not suspensions. -pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[..289])() { +pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[..289], base_symbol u32)() { // For the clcode example in this package's README.md: // - n_codes0 = 0 // - n_codes1 = 19 @@ -404,8 +429,12 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. i = in.n_codes0 while i < in.n_codes1 { assert i < 289 via "a < b: a < c; c <= b"(c:in.n_codes1) + // TODO: this if check should be unnecessary. + if i < in.n_codes0 { + return error "internal error: inconsistent Huffman decoder state" + } if this.code_lengths[i] != 0 { - this.symbols[offsets[this.code_lengths[i]]] = i as u16 + this.symbols[offsets[this.code_lengths[i]]] = (i - in.n_codes0) as u16 if offsets[this.code_lengths[i]] >= 289 { return error "internal error: inconsistent Huffman decoder state" } @@ -481,10 +510,16 @@ pri func decoder.init_huff?(which u32[..1], n_codes0 u32[..289], n_codes1 u32[.. } else if (symbol < 256) and (in.which == 0) { // Literal. value = 0x40000000 | (symbol << 8) | cl - } else { + } else if symbol >= in.base_symbol { // Base number + extra bits. - // TODO: bits 27-8. - value = 0x20000000 | cl + symbol -= in.base_symbol + if in.which == 0 { + value = lcode_magic_numbers[symbol & 31] | cl + } else { + value = dcode_magic_numbers[symbol & 31] | cl + } + } else { + return error "internal error: inconsistent Huffman decoder state" } // The table uses max_cl bits, but reversed_code only has cl bits. We