From 34198f3de566e0e89eac65882686e18d4a4d354b Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 21 Jul 2017 10:54:48 +1000 Subject: [PATCH] Start the init_huffman implementation. --- cmd/puffs-gen-c/func.go | 44 +++++++++++++++---- gen/c/std/flate.c | 84 +++++++++++++++++++++++++++++++----- gen/c/std/gif.c | 12 +++--- gen/h/std/flate.h | 18 +++++--- lang/check/bounds.go | 3 +- lang/check/type.go | 3 +- std/flate/decode_flate.puffs | 43 ++++++++++++++++-- 7 files changed, 171 insertions(+), 36 deletions(-) diff --git a/cmd/puffs-gen-c/func.go b/cmd/puffs-gen-c/func.go index 38199cfe..7f3293fe 100644 --- a/cmd/puffs-gen-c/func.go +++ b/cmd/puffs-gen-c/func.go @@ -493,13 +493,27 @@ func (g *gen) writeResumeSuspend1(n *a.Var, prefix string, suspend bool) error { g.printf("%s = %s;\n", lhs, rhs) return nil case t.KeyOpenBracket: - if inner := typ.Inner(); inner.Decorator() != 0 || inner.Name().Key() != t.KeyU8 { + inner := typ.Inner() + if inner.Decorator() != 0 { break } + multiplier := 0 + switch inner.Name().Key() { + case t.KeyU8: + multiplier = 1 + case t.KeyU16: + multiplier = 2 + case t.KeyU32: + multiplier = 4 + case t.KeyU64: + multiplier = 8 + } cv := typ.ArrayLength().ConstValue() - // TODO: check that cv is within size_t's range. - g.printf("memcpy(%s, %s, %v);\n", lhs, rhs, cv) - return nil + // TODO: check that multiplier * cv is within size_t's range. + if multiplier != 0 { + g.printf("memcpy(%s, %s, %d * %v);\n", lhs, rhs, multiplier, cv) + return nil + } } return fmt.Errorf("cannot resume or suspend a local variable %q of type %q", n.Name().String(g.tm), n.XType().String(g.tm)) @@ -758,14 +772,14 @@ func (g *gen) writeSuspendibles(n *a.Expr, depth uint32) error { } func (g *gen) writeCallSuspendibles(n *a.Expr, depth uint32) error { + if depth > a.MaxExprDepth { + return fmt.Errorf("expression recursion depth too large") + } + depth++ + // The evaluation order for suspendible calls (which can have side effects) // is important here: LHS, MHS, RHS, Args and finally the node itself. if !n.CallSuspendible() { - if depth > a.MaxExprDepth { - return fmt.Errorf("expression recursion depth too large") - } - depth++ - for _, o := range n.Node().Raw().SubNodes() { if o != nil && o.Kind() == a.KExpr { if err := g.writeCallSuspendibles(o.Expr(), depth); err != nil { @@ -955,6 +969,18 @@ func (g *gen) writeCallSuspendibles(n *a.Expr, depth uint32) error { } g.writes("if (status) { goto suspend; }\n") + } else if isThisMethod(g.tm, n, "init_huffman", 1) { + g.printf("status = %s%s_init_huffman(self,", + g.pkgPrefix, g.perFunc.funk.Receiver().String(g.tm)) + if err := g.writeExpr(n.Args()[0].Arg().Value(), replaceNothing, parenthesesMandatory, depth); err != nil { + return err + } + g.writes(");\n") + if err := g.writeLoadExprDerivedVars(n); err != nil { + return err + } + g.writes("if (status) { goto suspend; }\n") + } else if isDecode(g.tm, n) { g.printf("status = %slzw_decoder_decode(&self->private_impl.f_lzw, %sdst, %s%s);\n", g.pkgPrefix, aPrefix, vPrefix, n.Args()[1].Arg().Value().String(g.tm)) diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 8a37e3f7..2dff08fe 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -105,13 +105,14 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800 #define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040126 // 0xbb08f802 + -1157040126 // 0xbb08f802 +#define PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE -1157040125 // 0xbb08f803 #define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ - -1157040125 // 0xbb08f803 + -1157040124 // 0xbb08f804 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040124 // 0xbb08f804 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_TODO_DYNAMIC_HUFFMAN_BLOCKS -1157040122 // 0xbb08f806 + -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_TODO_DYNAMIC_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -149,7 +150,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; puffs_flate_huffman_decoder f_huff[2]; - uint8_t f_code_lengths[19]; + uint8_t f_code_lengths[289]; uint32_t f_wip0; uint32_t f_wip1; @@ -171,6 +172,11 @@ typedef struct { uint32_t v_hclen; uint32_t v_i; } c_decode_dynamic[1]; + struct { + uint32_t coro_susp_point; + uint32_t v_counts[16]; + uint32_t v_i; + } c_init_huffman[1]; } private_impl; } puffs_flate_decoder; @@ -253,10 +259,11 @@ const char* puffs_flate_status_strings0[11] = { "flate: limited write", }; -const char* puffs_flate_status_strings1[7] = { +const char* puffs_flate_status_strings1[8] = { "flate: bad distance code count", "flate: bad flate block", "flate: bad literal/length code count", + "flate: inconsistent Huffman table", "flate: inconsistent stored block length", "flate: internal error: inconsistent n_bits", "flate: TODO: fixed Huffman blocks", @@ -273,7 +280,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) { break; case puffs_flate_packageid: a = puffs_flate_status_strings1; - n = 7; + n = 8; break; } uint32_t i = s & 0xff; @@ -299,6 +306,9 @@ puffs_flate_status puffs_flate_decoder_decode_dynamic(puffs_flate_decoder* self, puffs_base_writer1 a_dst, puffs_base_reader1 a_src); +puffs_flate_status puffs_flate_decoder_init_huffman(puffs_flate_decoder* self, + uint32_t a_n_codes); + // ---------------- Constructor and Destructor Implementations // PUFFS_MAGIC is a magic number to check that constructors are called. It's @@ -726,10 +736,15 @@ puffs_flate_status puffs_flate_decoder_decode_dynamic( self->private_impl.f_code_lengths[puffs_flate_code_order[v_i]] = 0; v_i += 1; } - self->private_impl.f_wip0 = v_hlit; - self->private_impl.f_wip1 = v_hdist; self->private_impl.f_bits = v_bits; self->private_impl.f_n_bits = v_n_bits; + PUFFS_COROUTINE_SUSPENSION_POINT(3); + status = puffs_flate_decoder_init_huffman(self, 19); + if (status) { + goto suspend; + } + self->private_impl.f_wip0 = v_hlit; + self->private_impl.f_wip1 = v_hdist; self->private_impl.c_decode_dynamic[0].coro_susp_point = 0; goto exit; } @@ -769,3 +784,52 @@ short_read_src: } goto suspend; } + +puffs_flate_status puffs_flate_decoder_init_huffman(puffs_flate_decoder* self, + uint32_t a_n_codes) { + puffs_flate_status status = PUFFS_FLATE_STATUS_OK; + + uint32_t v_counts[16]; + uint32_t v_i; + + uint32_t coro_susp_point = + self->private_impl.c_init_huffman[0].coro_susp_point; + if (coro_susp_point) { + memcpy(v_counts, self->private_impl.c_init_huffman[0].v_counts, 4 * 16); + v_i = self->private_impl.c_init_huffman[0].v_i; + } + switch (coro_susp_point) { + PUFFS_COROUTINE_SUSPENSION_POINT(0); + + { + size_t i; + for (i = 0; i < 16; i++) { + v_counts[i] = 0; + } + }; + v_i = 0; + while (v_i < a_n_codes) { + if (v_counts[self->private_impl.f_code_lengths[v_i]] >= 289) { + status = PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE; + goto exit; + } + v_counts[self->private_impl.f_code_lengths[v_i]] += 1; + v_i += 1; + } + if (v_counts[0] == a_n_codes) { + status = PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE; + goto exit; + } + self->private_impl.c_init_huffman[0].coro_susp_point = 0; + goto exit; + } + + goto suspend; +suspend: + self->private_impl.c_init_huffman[0].coro_susp_point = coro_susp_point; + memcpy(self->private_impl.c_init_huffman[0].v_counts, v_counts, 4 * 16); + self->private_impl.c_init_huffman[0].v_i = v_i; + +exit: + return status; +} diff --git a/gen/c/std/gif.c b/gen/c/std/gif.c index a2c2af53..30c8d73c 100644 --- a/gen/c/std/gif.c +++ b/gen/c/std/gif.c @@ -632,7 +632,7 @@ puffs_gif_status puffs_gif_decoder_decode_header(puffs_gif_decoder* self, uint32_t coro_susp_point = self->private_impl.c_decode_header[0].coro_susp_point; if (coro_susp_point) { - memcpy(v_c, self->private_impl.c_decode_header[0].v_c, 6); + memcpy(v_c, self->private_impl.c_decode_header[0].v_c, 1 * 6); v_i = self->private_impl.c_decode_header[0].v_i; } switch (coro_susp_point) { @@ -666,7 +666,7 @@ puffs_gif_status puffs_gif_decoder_decode_header(puffs_gif_decoder* self, goto suspend; suspend: self->private_impl.c_decode_header[0].coro_susp_point = coro_susp_point; - memcpy(self->private_impl.c_decode_header[0].v_c, v_c, 6); + memcpy(self->private_impl.c_decode_header[0].v_c, v_c, 1 * 6); self->private_impl.c_decode_header[0].v_i = v_i; exit: @@ -719,7 +719,7 @@ puffs_gif_status puffs_gif_decoder_decode_lsd(puffs_gif_decoder* self, uint32_t coro_susp_point = self->private_impl.c_decode_lsd[0].coro_susp_point; if (coro_susp_point) { - memcpy(v_c, self->private_impl.c_decode_lsd[0].v_c, 7); + memcpy(v_c, self->private_impl.c_decode_lsd[0].v_c, 1 * 7); v_i = self->private_impl.c_decode_lsd[0].v_i; v_gct_size = self->private_impl.c_decode_lsd[0].v_gct_size; } @@ -779,7 +779,7 @@ puffs_gif_status puffs_gif_decoder_decode_lsd(puffs_gif_decoder* self, goto suspend; suspend: self->private_impl.c_decode_lsd[0].coro_susp_point = coro_susp_point; - memcpy(self->private_impl.c_decode_lsd[0].v_c, v_c, 7); + memcpy(self->private_impl.c_decode_lsd[0].v_c, v_c, 1 * 7); self->private_impl.c_decode_lsd[0].v_i = v_i; self->private_impl.c_decode_lsd[0].v_gct_size = v_gct_size; @@ -947,7 +947,7 @@ puffs_gif_status puffs_gif_decoder_decode_id(puffs_gif_decoder* self, uint32_t coro_susp_point = self->private_impl.c_decode_id[0].coro_susp_point; if (coro_susp_point) { - memcpy(v_c, self->private_impl.c_decode_id[0].v_c, 9); + memcpy(v_c, self->private_impl.c_decode_id[0].v_c, 1 * 9); v_i = self->private_impl.c_decode_id[0].v_i; v_interlace = self->private_impl.c_decode_id[0].v_interlace; v_lw = self->private_impl.c_decode_id[0].v_lw; @@ -1045,7 +1045,7 @@ puffs_gif_status puffs_gif_decoder_decode_id(puffs_gif_decoder* self, goto suspend; suspend: self->private_impl.c_decode_id[0].coro_susp_point = coro_susp_point; - memcpy(self->private_impl.c_decode_id[0].v_c, v_c, 9); + memcpy(self->private_impl.c_decode_id[0].v_c, v_c, 1 * 9); self->private_impl.c_decode_id[0].v_i = v_i; self->private_impl.c_decode_id[0].v_interlace = v_interlace; self->private_impl.c_decode_id[0].v_lw = v_lw; diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index 2707bced..216d801b 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -105,13 +105,14 @@ typedef int32_t puffs_flate_status; #define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800 #define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801 #define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \ - -1157040126 // 0xbb08f802 + -1157040126 // 0xbb08f802 +#define PUFFS_FLATE_ERROR_INCONSISTENT_HUFFMAN_TABLE -1157040125 // 0xbb08f803 #define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \ - -1157040125 // 0xbb08f803 + -1157040124 // 0xbb08f804 #define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \ - -1157040124 // 0xbb08f804 -#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040123 // 0xbb08f805 -#define PUFFS_FLATE_ERROR_TODO_DYNAMIC_HUFFMAN_BLOCKS -1157040122 // 0xbb08f806 + -1157040123 // 0xbb08f805 +#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040122 // 0xbb08f806 +#define PUFFS_FLATE_ERROR_TODO_DYNAMIC_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -149,7 +150,7 @@ typedef struct { uint32_t f_bits; uint32_t f_n_bits; puffs_flate_huffman_decoder f_huff[2]; - uint8_t f_code_lengths[19]; + uint8_t f_code_lengths[289]; uint32_t f_wip0; uint32_t f_wip1; @@ -171,6 +172,11 @@ typedef struct { uint32_t v_hclen; uint32_t v_i; } c_decode_dynamic[1]; + struct { + uint32_t coro_susp_point; + uint32_t v_counts[16]; + uint32_t v_i; + } c_init_huffman[1]; } private_impl; } puffs_flate_decoder; diff --git a/lang/check/bounds.go b/lang/check/bounds.go index 505bd2b5..0bbdd02c 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -624,7 +624,8 @@ func (q *checker) bcheckExprOther(n *a.Expr, depth uint32) (*big.Int, *big.Int, isInDst(q.tm, n, t.KeyCopyFrom32, 2) || isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || - isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_dynamic", 2) { + isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_dynamic", 2) || + isThisMethod(q.tm, n, "init_huffman", 1) { for _, o := range n.Args() { // TODO: check that the arg range at the caller and the diff --git a/lang/check/type.go b/lang/check/type.go index fc547157..918ed68d 100644 --- a/lang/check/type.go +++ b/lang/check/type.go @@ -349,7 +349,8 @@ func (q *checker) tcheckExprOther(n *a.Expr, depth uint32) error { isInDst(q.tm, n, t.KeyCopyFrom32, 2) || isThisMethod(q.tm, n, "decode_header", 1) || isThisMethod(q.tm, n, "decode_lsd", 1) || isThisMethod(q.tm, n, "decode_extension", 1) || isThisMethod(q.tm, n, "decode_id", 2) || - isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_dynamic", 2) { + isThisMethod(q.tm, n, "decode_uncompressed", 2) || isThisMethod(q.tm, n, "decode_dynamic", 2) || + isThisMethod(q.tm, n, "init_huffman", 1) { if err := q.tcheckExpr(n.LHS().Expr(), depth); err != nil { return err diff --git a/std/flate/decode_flate.puffs b/std/flate/decode_flate.puffs index f190c995..171c5843 100644 --- a/std/flate/decode_flate.puffs +++ b/std/flate/decode_flate.puffs @@ -8,6 +8,7 @@ packageid "flte" pub error "bad distance code count" pub error "bad flate block" pub error "bad literal/length code count" +pub error "inconsistent Huffman table" pub error "inconsistent stored block length" pub error "internal error: inconsistent n_bits" @@ -20,7 +21,7 @@ pub struct decoder?( n_bits u32, huff[2] huffman_decoder, - code_lengths[19] u8[..7], + code_lengths[289] u8[..15], // These temporary fields are used to debug work-in-progress code. wip0 u32, @@ -113,13 +114,49 @@ pri func decoder.decode_dynamic?(dst writer1, src reader1)() { this.code_lengths[code_order[i]] = 0 i += 1 } + this.bits = bits + this.n_bits = n_bits + + this.init_huffman?(n_codes:19) // TODO: implement. this.wip0 = hlit this.wip1 = hdist +} - this.bits = bits - this.n_bits = n_bits +// TODO: make init_huffman a huffman_decoder method instead of a decoder +// method? + +// TODO: make named constants for 15, 289, etc. + +// TODO: skip generating PUFFS_COROUTINE_SUSPENSION_POINTs for this function, +// and its callers, as it can only return errors, not suspensions. + +pri func decoder.init_huffman?(n_codes u32[..289])() { + var counts[16] u32[..289] + var i u32 + while i < in.n_codes { + assert i < 289 via "a < b: a < c; c <= b"(c:in.n_codes) + // TODO: this if should be unnecessary. Have some way to assert that, + // for all j, counts[j] <= i, and thus counts[j]++ will not overflow. + if counts[this.code_lengths[i]] >= 289 { + return error "inconsistent Huffman table" + } + counts[this.code_lengths[i]] += 1 + i += 1 + } + if counts[0] == in.n_codes { + // There are no codes. + return error "inconsistent Huffman table" + } + + // For the clcode_lengths example in this package's README.md: + // - counts[3] = 7 + // - counts[4] = 1 + // - counts[5] = 2 + // - all other counts elements are zero. + + // TODO: check for the total code_lengths being too short or too long. } pri struct huffman_decoder(