From b3710e583ffda9314c792218f8709952546c2ead Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 5 Aug 2017 16:34:03 +1000 Subject: [PATCH] Add a zlib decoder to std/flate. --- cmd/puffs-gen-c/base-impl.h | 5 + cmd/puffs-gen-c/data.go | 4 +- cmd/puffs-gen-c/func.go | 82 +++++++++--- cmd/puffs-gen-c/main.go | 24 ++-- gen/c/std/flate.c | 254 +++++++++++++++++++++++++++++++++++- gen/c/std/gif.c | 5 + gen/h/std/flate.h | 46 +++++++ lang/check/bounds.go | 13 +- lang/check/check.go | 1 + lang/check/type.go | 13 +- lang/token/list.go | 179 +++++++++++++------------ std/flate/decode_zlib.puffs | 31 +++++ test/c/std/flate.c | 26 ++++ 13 files changed, 566 insertions(+), 117 deletions(-) create mode 100644 std/flate/decode_zlib.puffs diff --git a/cmd/puffs-gen-c/base-impl.h b/cmd/puffs-gen-c/base-impl.h index c363eb17..793f5b54 100644 --- a/cmd/puffs-gen-c/base-impl.h +++ b/cmd/puffs-gen-c/base-impl.h @@ -8,6 +8,11 @@ // TODO: look for (ifdef) the x86 architecture and cast the pointer? Only do so // if a benchmark justifies the additional code path. +#define PUFFS_U16BE(p) (((uint16_t)(p[0]) << 8) | ((uint16_t)(p[1]) << 0)) +#define PUFFS_U16LE(p) (((uint16_t)(p[0]) << 0) | ((uint16_t)(p[1]) << 8)) +#define PUFFS_U32BE(p) \ + (((uint32_t)(p[0]) << 24) | ((uint32_t)(p[1]) << 16) | \ + ((uint32_t)(p[2]) << 8) | ((uint32_t)(p[3]) << 0)) #define PUFFS_U32LE(p) \ (((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | \ ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24)) diff --git a/cmd/puffs-gen-c/data.go b/cmd/puffs-gen-c/data.go index b00959c1..37b24b5d 100644 --- a/cmd/puffs-gen-c/data.go +++ b/cmd/puffs-gen-c/data.go @@ -12,6 +12,6 @@ const baseHeader = "" + "" const baseImpl = "" + - "#ifndef PUFFS_BASE_IMPL_H\n#define PUFFS_BASE_IMPL_H\n\n// Use of this source code is governed by a BSD-style license that can be found\n// in the LICENSE file.\n\n// TODO: look for (ifdef) the x86 architecture and cast the pointer? Only do so\n// if a benchmark justifies the additional code path.\n#define PUFFS_U32LE(p) \\\n (((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | \\\n ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24))\n\n// Use switch cases for coroutine suspension points, similar to the technique\n// in https://www.chiark.greenend.org.uk/~sgtatham/coroutines.html\n//\n// We use a trivial macro instead of an explicit assignment and case statement\n// so that clang-format doesn't get confused by the unusual \"case\"s.\n#define PUFFS_COROUTINE_SUSPENSION_POINT(n) \\\n coro_susp_point = n; \\\n case n:\n\n// Clang also defines \"__GNUC__\".\n#if defined(__GNUC__)\n#define PUFFS_LIKELY(expr) (__builtin_expect(!!(expr), 1))\n#define PUFFS_UNLIKELY(expr) (__builtin_expect(" + - "!!(expr), 0))\n// Declare the printf prototype. The generated code shouldn't need this at all,\n// but it's useful for manual printf debugging.\nextern int printf(const char* __restrict __format, ...);\n#else\n#define PUFFS_LIKELY(expr) (expr)\n#define PUFFS_UNLIKELY(expr) (expr)\n#endif\n\n#endif // PUFFS_BASE_IMPL_H\n" + + "#ifndef PUFFS_BASE_IMPL_H\n#define PUFFS_BASE_IMPL_H\n\n// Use of this source code is governed by a BSD-style license that can be found\n// in the LICENSE file.\n\n// TODO: look for (ifdef) the x86 architecture and cast the pointer? Only do so\n// if a benchmark justifies the additional code path.\n#define PUFFS_U16BE(p) (((uint16_t)(p[0]) << 8) | ((uint16_t)(p[1]) << 0))\n#define PUFFS_U16LE(p) (((uint16_t)(p[0]) << 0) | ((uint16_t)(p[1]) << 8))\n#define PUFFS_U32BE(p) \\\n (((uint32_t)(p[0]) << 24) | ((uint32_t)(p[1]) << 16) | \\\n ((uint32_t)(p[2]) << 8) | ((uint32_t)(p[3]) << 0))\n#define PUFFS_U32LE(p) \\\n (((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | \\\n ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24))\n\n// Use switch cases for coroutine suspension points, similar to the technique\n// in https://www.chiark.greenend.org.uk/~sgtatham/coroutines.html\n//\n// We use a trivial macro instead of an explicit assignment and case statement\n// so that c" + + "lang-format doesn't get confused by the unusual \"case\"s.\n#define PUFFS_COROUTINE_SUSPENSION_POINT(n) \\\n coro_susp_point = n; \\\n case n:\n\n// Clang also defines \"__GNUC__\".\n#if defined(__GNUC__)\n#define PUFFS_LIKELY(expr) (__builtin_expect(!!(expr), 1))\n#define PUFFS_UNLIKELY(expr) (__builtin_expect(!!(expr), 0))\n// Declare the printf prototype. The generated code shouldn't need this at all,\n// but it's useful for manual printf debugging.\nextern int printf(const char* __restrict __format, ...);\n#else\n#define PUFFS_LIKELY(expr) (expr)\n#define PUFFS_UNLIKELY(expr) (expr)\n#endif\n\n#endif // PUFFS_BASE_IMPL_H\n" + "" diff --git a/cmd/puffs-gen-c/func.go b/cmd/puffs-gen-c/func.go index 25d37f9a..c522aba9 100644 --- a/cmd/puffs-gen-c/func.go +++ b/cmd/puffs-gen-c/func.go @@ -812,8 +812,17 @@ func (g *gen) writeCallSuspendibles(n *a.Expr, depth uint32) error { } g.printf(" = *%srptr_src++;\n", bPrefix) + } else if isInSrc(g.tm, n, t.KeyReadU16BE, 0) { + return g.writeReadUXX(n, "src", 16, "BE") + + } else if isInSrc(g.tm, n, t.KeyReadU16LE, 0) { + return g.writeReadUXX(n, "src", 16, "LE") + + } else if isInSrc(g.tm, n, t.KeyReadU32BE, 0) { + return g.writeReadUXX(n, "src", 32, "BE") + } else if isInSrc(g.tm, n, t.KeyReadU32LE, 0) { - return g.writeReadUXX(n, "src", 32, "le") + return g.writeReadUXX(n, "src", 32, "LE") } else if isInSrc(g.tm, n, t.KeySkip32, 1) { // TODO: is the scratch variable safe to use if callers can trap @@ -1056,13 +1065,25 @@ func (g *gen) writeCallSuspendibles(n *a.Expr, depth uint32) error { g.writes("if (status) { goto suspend; }\n") } else if isDecode(g.tm, n) { - g.printf("status = %slzw_decoder_decode(&self->private_impl.f_lzw, %sdst, %s%s);\n", - g.pkgPrefix, aPrefix, vPrefix, n.Args()[1].Arg().Value().String(g.tm)) - if err := g.writeLoadExprDerivedVars(n); err != nil { - return err + switch g.pkgName { + case "flate": + g.printf("status = %sdecoder_decode(&self->private_impl.f_dec, %sdst, %ssrc);\n", + g.pkgPrefix, aPrefix, aPrefix) + if err := g.writeLoadExprDerivedVars(n); err != nil { + return err + } + g.writes("if (status) { goto suspend; }\n") + case "gif": + g.printf("status = %slzw_decoder_decode(&self->private_impl.f_lzw, %sdst, %s%s);\n", + g.pkgPrefix, aPrefix, vPrefix, n.Args()[1].Arg().Value().String(g.tm)) + if err := g.writeLoadExprDerivedVars(n); err != nil { + return err + } + // TODO: be principled with "if (l_lzw_src && etc)". + g.writes("if (l_lzw_src && status) { goto suspend; }\n") + default: + return fmt.Errorf("cannot convert Puffs call %q to C", n.String(g.tm)) } - // TODO: be principled with "if (l_lzw_src && etc)". - g.writes("if (l_lzw_src && status) { goto suspend; }\n") } else { // TODO: fix this. @@ -1087,6 +1108,13 @@ func (g *gen) writeShortRead(name string) error { } func (g *gen) writeReadUXX(n *a.Expr, name string, size uint32, endianness string) error { + if size != 16 && size != 32 { + return fmt.Errorf("internal error: bad writeReadUXX size %d", size) + } + if endianness != "BE" && endianness != "LE" { + return fmt.Errorf("internal error: bad writeReadUXX endianness %q", endianness) + } + if g.perFunc.tempW > maxTemp-1 { return fmt.Errorf("too many temporary variables required") } @@ -1101,9 +1129,9 @@ func (g *gen) writeReadUXX(n *a.Expr, name string, size uint32, endianness strin } g.writes(";") - g.printf("if (PUFFS_LIKELY(%srend_src - %srptr_src >= 4)) {", bPrefix, bPrefix) - g.printf("%s%d = PUFFS_U32LE(%srptr_src);\n", tPrefix, temp1, bPrefix) - g.printf("%srptr_src += 4;\n", bPrefix) + g.printf("if (PUFFS_LIKELY(%srend_src - %srptr_src >= %d)) {", bPrefix, bPrefix, size/8) + g.printf("%s%d = PUFFS_U%d%s(%srptr_src);\n", tPrefix, temp1, size, endianness, bPrefix) + g.printf("%srptr_src += %d;\n", bPrefix, size/8) g.printf("} else {") g.printf("self->private_impl.scratch = 0;\n") if err := g.writeCoroSuspPoint(); err != nil { @@ -1115,19 +1143,39 @@ func (g *gen) writeReadUXX(n *a.Expr, name string, size uint32, endianness strin bPrefix, name, bPrefix, name, name) g.perFunc.shortReads = append(g.perFunc.shortReads, name) - // TODO: look at endianness. - g.printf("uint32_t %s%d = self->private_impl.scratch >> 56;", tPrefix, temp0) - g.printf("self->private_impl.scratch <<= 8;") - g.printf("self->private_impl.scratch >>= 8;") - g.printf("self->private_impl.scratch |= ((uint64_t)(*%srptr_%s++)) << %s%d;", bPrefix, name, tPrefix, temp0) + g.printf("uint32_t %s%d = self->private_impl.scratch", tPrefix, temp0) + switch endianness { + case "BE": + g.printf("& 0xFF;") + g.printf("self->private_impl.scratch >>= 8;") + g.printf("self->private_impl.scratch <<= 8;") + g.printf("self->private_impl.scratch |= ((uint64_t)(*%srptr_%s++)) << (64 - %s%d);", + bPrefix, name, tPrefix, temp0) + case "LE": + g.printf(">> 56;") + g.printf("self->private_impl.scratch <<= 8;") + g.printf("self->private_impl.scratch >>= 8;") + g.printf("self->private_impl.scratch |= ((uint64_t)(*%srptr_%s++)) << %s%d;", + bPrefix, name, tPrefix, temp0) + } g.printf("if (%s%d == %d) {", tPrefix, temp0, size-8) - g.printf("%s%d = self->private_impl.scratch;", tPrefix, temp1) + switch endianness { + case "BE": + g.printf("%s%d = self->private_impl.scratch >> (64 - %d);", tPrefix, temp1, size) + case "LE": + g.printf("%s%d = self->private_impl.scratch;", tPrefix, temp1) + } g.printf("break;") g.printf("}") g.printf("%s%d += 8;", tPrefix, temp0) - g.printf("self->private_impl.scratch |= ((uint64_t)(%s%d)) << 56;", tPrefix, temp0) + switch endianness { + case "BE": + g.printf("self->private_impl.scratch |= ((uint64_t)(%s%d));", tPrefix, temp0) + case "LE": + g.printf("self->private_impl.scratch |= ((uint64_t)(%s%d)) << 56;", tPrefix, temp0) + } g.writes("}}\n") return nil diff --git a/cmd/puffs-gen-c/main.go b/cmd/puffs-gen-c/main.go index cd1c3cbf..e535e44b 100644 --- a/cmd/puffs-gen-c/main.go +++ b/cmd/puffs-gen-c/main.go @@ -807,17 +807,18 @@ var cTypeNames = [...]string{ } var cOpNames = [256]string{ - t.KeyEq: " = ", - t.KeyPlusEq: " += ", - t.KeyMinusEq: " -= ", - t.KeyStarEq: " *= ", - t.KeySlashEq: " /= ", - t.KeyShiftLEq: " <<= ", - t.KeyShiftREq: " >>= ", - t.KeyAmpEq: " &= ", - t.KeyAmpHatEq: " no_such_amp_hat_C_operator ", - t.KeyPipeEq: " |= ", - t.KeyHatEq: " ^= ", + t.KeyEq: " = ", + t.KeyPlusEq: " += ", + t.KeyMinusEq: " -= ", + t.KeyStarEq: " *= ", + t.KeySlashEq: " /= ", + t.KeyShiftLEq: " <<= ", + t.KeyShiftREq: " >>= ", + t.KeyAmpEq: " &= ", + t.KeyAmpHatEq: " no_such_amp_hat_C_operator ", + t.KeyPipeEq: " |= ", + t.KeyHatEq: " ^= ", + t.KeyPercentEq: " %= ", t.KeyXUnaryPlus: "+", t.KeyXUnaryMinus: "-", @@ -833,6 +834,7 @@ var cOpNames = [256]string{ t.KeyXBinaryAmpHat: " no_such_amp_hat_C_operator ", t.KeyXBinaryPipe: " | ", t.KeyXBinaryHat: " ^ ", + t.KeyXBinaryPercent: " % ", t.KeyXBinaryNotEq: " != ", t.KeyXBinaryLessThan: " < ", t.KeyXBinaryLessEq: " <= ", diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index b4b13040..4c2bccc5 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -125,6 +125,13 @@ typedef int32_t puffs_flate_status; -1157040117 // 0xbb08f80b #define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040116 // 0xbb08f80c #define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040115 // 0xbb08f80d +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_METHOD \ + -1157040114 // 0xbb08f80e +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_WINDOW_SIZE \ + -1157040113 // 0xbb08f80f +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK -1157040112 // 0xbb08f810 +#define PUFFS_FLATE_ERROR_TODO_UNSUPPORTED_ZLIB_PRESET_DICTIONARY \ + -1157040111 // 0xbb08f811 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -225,6 +232,29 @@ typedef struct { } private_impl; } puffs_flate_decoder; +typedef struct { + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. Instead, use the + // puffs_flate_zlib_decoder_etc functions. + // + // In C++, these fields would be "private", but C does not support that. + // + // It is a struct, not a struct*, so that it can be stack allocated. + struct { + puffs_flate_status status; + uint32_t magic; + uint64_t scratch; + + puffs_flate_decoder f_dec; + + struct { + uint32_t coro_susp_point; + uint16_t v_x; + uint32_t v_checksum; + } c_decode[1]; + } private_impl; +} puffs_flate_zlib_decoder; + // ---------------- Public Constructor and Destructor Prototypes // puffs_flate_decoder_constructor is a constructor function. @@ -238,12 +268,28 @@ void puffs_flate_decoder_constructor(puffs_flate_decoder* self, void puffs_flate_decoder_destructor(puffs_flate_decoder* self); +// puffs_flate_zlib_decoder_constructor is a constructor function. +// +// It should be called before any other puffs_flate_zlib_decoder_* function. +// +// Pass PUFFS_VERSION and 0 for puffs_version and for_internal_use_only. +void puffs_flate_zlib_decoder_constructor(puffs_flate_zlib_decoder* self, + uint32_t puffs_version, + uint32_t for_internal_use_only); + +void puffs_flate_zlib_decoder_destructor(puffs_flate_zlib_decoder* self); + // ---------------- Public Function Prototypes puffs_flate_status puffs_flate_decoder_decode(puffs_flate_decoder* self, puffs_base_writer1 a_dst, puffs_base_reader1 a_src); +puffs_flate_status puffs_flate_zlib_decoder_decode( + puffs_flate_zlib_decoder* self, + puffs_base_writer1 a_dst, + puffs_base_reader1 a_src); + #ifdef __cplusplus } // extern "C" #endif @@ -260,6 +306,11 @@ puffs_flate_status puffs_flate_decoder_decode(puffs_flate_decoder* self, // TODO: look for (ifdef) the x86 architecture and cast the pointer? Only do so // if a benchmark justifies the additional code path. +#define PUFFS_U16BE(p) (((uint16_t)(p[0]) << 8) | ((uint16_t)(p[1]) << 0)) +#define PUFFS_U16LE(p) (((uint16_t)(p[0]) << 0) | ((uint16_t)(p[1]) << 8)) +#define PUFFS_U32BE(p) \ + (((uint32_t)(p[0]) << 24) | ((uint32_t)(p[1]) << 16) | \ + ((uint32_t)(p[2]) << 8) | ((uint32_t)(p[3]) << 0)) #define PUFFS_U32LE(p) \ (((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | \ ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24)) @@ -307,7 +358,7 @@ const char* puffs_flate_status_strings0[11] = { "flate: limited write", }; -const char* puffs_flate_status_strings1[14] = { +const char* puffs_flate_status_strings1[18] = { "flate: bad Huffman code (over-subscribed)", "flate: bad Huffman code (under-subscribed)", "flate: bad Huffman code length count", @@ -322,6 +373,10 @@ const char* puffs_flate_status_strings1[14] = { "flate: internal error: inconsistent n_bits", "flate: missing end-of-block code", "flate: no Huffman codes", + "flate: invalid zlib compression method", + "flate: invalid zlib compression window size", + "flate: invalid zlib parity check", + "flate: TODO: unsupported zlib preset dictionary", }; const char* puffs_flate_status_string(puffs_flate_status s) { @@ -334,7 +389,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) { break; case puffs_flate_packageid: a = puffs_flate_status_strings1; - n = 14; + n = 18; break; } uint32_t i = s & 0xff; @@ -446,6 +501,31 @@ void puffs_flate_decoder_destructor(puffs_flate_decoder* self) { } } +void puffs_flate_zlib_decoder_constructor(puffs_flate_zlib_decoder* self, + uint32_t puffs_version, + uint32_t for_internal_use_only) { + if (!self) { + return; + } + if (puffs_version != PUFFS_VERSION) { + self->private_impl.status = PUFFS_FLATE_ERROR_BAD_PUFFS_VERSION; + return; + } + if (for_internal_use_only != PUFFS_ALREADY_ZEROED) { + memset(self, 0, sizeof(*self)); + } + self->private_impl.magic = PUFFS_MAGIC; + puffs_flate_decoder_constructor(&self->private_impl.f_dec, PUFFS_VERSION, + PUFFS_ALREADY_ZEROED); +} + +void puffs_flate_zlib_decoder_destructor(puffs_flate_zlib_decoder* self) { + if (!self) { + return; + } + puffs_flate_decoder_destructor(&self->private_impl.f_dec); +} + // ---------------- Function Implementations puffs_flate_status puffs_flate_decoder_decode(puffs_flate_decoder* self, @@ -1767,3 +1847,173 @@ suspend: exit: return status; } + +puffs_flate_status puffs_flate_zlib_decoder_decode( + puffs_flate_zlib_decoder* self, + puffs_base_writer1 a_dst, + puffs_base_reader1 a_src) { + if (!self) { + return PUFFS_FLATE_ERROR_BAD_RECEIVER; + } + if (self->private_impl.magic != PUFFS_MAGIC) { + self->private_impl.status = PUFFS_FLATE_ERROR_CONSTRUCTOR_NOT_CALLED; + } + if (self->private_impl.status < 0) { + return self->private_impl.status; + } + puffs_flate_status status = PUFFS_FLATE_STATUS_OK; + + uint16_t v_x; + uint32_t v_checksum; + + uint8_t* b_rptr_src = NULL; + uint8_t* b_rend_src = NULL; + if (a_src.buf) { + b_rptr_src = a_src.buf->ptr + a_src.buf->ri; + size_t len = a_src.buf->wi - a_src.buf->ri; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_rend_src = b_rptr_src + len; + } + + uint32_t coro_susp_point = self->private_impl.c_decode[0].coro_susp_point; + if (coro_susp_point) { + v_x = self->private_impl.c_decode[0].v_x; + v_checksum = self->private_impl.c_decode[0].v_checksum; + } + switch (coro_susp_point) { + PUFFS_COROUTINE_SUSPENSION_POINT(0); + + PUFFS_COROUTINE_SUSPENSION_POINT(1); + uint16_t t_1; + if (PUFFS_LIKELY(b_rend_src - b_rptr_src >= 2)) { + t_1 = PUFFS_U16BE(b_rptr_src); + b_rptr_src += 2; + } else { + self->private_impl.scratch = 0; + PUFFS_COROUTINE_SUSPENSION_POINT(2); + while (true) { + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint32_t t_0 = self->private_impl.scratch & 0xFF; + self->private_impl.scratch >>= 8; + self->private_impl.scratch <<= 8; + self->private_impl.scratch |= ((uint64_t)(*b_rptr_src++)) << (64 - t_0); + if (t_0 == 8) { + t_1 = self->private_impl.scratch >> (64 - 16); + break; + } + t_0 += 8; + self->private_impl.scratch |= ((uint64_t)(t_0)); + } + } + v_x = t_1; + if (((v_x >> 8) & 15) != 8) { + status = PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_METHOD; + goto exit; + } + if ((v_x >> 12) > 7) { + status = PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_WINDOW_SIZE; + goto exit; + } + if ((v_x & 32) != 0) { + status = PUFFS_FLATE_ERROR_TODO_UNSUPPORTED_ZLIB_PRESET_DICTIONARY; + goto exit; + } + if ((v_x % 31) != 0) { + status = PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK; + goto exit; + } + PUFFS_COROUTINE_SUSPENSION_POINT(3); + if (a_src.buf) { + size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); + a_src.buf->ri += n; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + } + status = + puffs_flate_decoder_decode(&self->private_impl.f_dec, a_dst, a_src); + if (a_src.buf) { + b_rptr_src = a_src.buf->ptr + a_src.buf->ri; + size_t len = a_src.buf->wi - a_src.buf->ri; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len && (len > *lim->ptr_to_len)) { + len = *lim->ptr_to_len; + } + } + b_rend_src = b_rptr_src + len; + } + if (status) { + goto suspend; + } + PUFFS_COROUTINE_SUSPENSION_POINT(4); + uint32_t t_3; + if (PUFFS_LIKELY(b_rend_src - b_rptr_src >= 4)) { + t_3 = PUFFS_U32BE(b_rptr_src); + b_rptr_src += 4; + } else { + self->private_impl.scratch = 0; + PUFFS_COROUTINE_SUSPENSION_POINT(5); + while (true) { + if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) { + goto short_read_src; + } + uint32_t t_2 = self->private_impl.scratch & 0xFF; + self->private_impl.scratch >>= 8; + self->private_impl.scratch <<= 8; + self->private_impl.scratch |= ((uint64_t)(*b_rptr_src++)) << (64 - t_2); + if (t_2 == 24) { + t_3 = self->private_impl.scratch >> (64 - 32); + break; + } + t_2 += 8; + self->private_impl.scratch |= ((uint64_t)(t_2)); + } + } + v_checksum = t_3; + self->private_impl.c_decode[0].coro_susp_point = 0; + goto exit; + } + + goto suspend; +suspend: + self->private_impl.c_decode[0].coro_susp_point = coro_susp_point; + self->private_impl.c_decode[0].v_x = v_x; + self->private_impl.c_decode[0].v_checksum = v_checksum; + +exit: + if (a_src.buf) { + size_t n = b_rptr_src - (a_src.buf->ptr + a_src.buf->ri); + a_src.buf->ri += n; + puffs_base_limit1* lim; + for (lim = &a_src.limit; lim; lim = lim->next) { + if (lim->ptr_to_len) { + *lim->ptr_to_len -= n; + } + } + } + + self->private_impl.status = status; + return status; + +short_read_src: + if (a_src.limit.ptr_to_len) { + status = PUFFS_FLATE_SUSPENSION_LIMITED_READ; + } else if (a_src.buf->closed) { + status = PUFFS_FLATE_ERROR_UNEXPECTED_EOF; + goto exit; + } else { + status = PUFFS_FLATE_SUSPENSION_SHORT_READ; + } + goto suspend; +} diff --git a/gen/c/std/gif.c b/gen/c/std/gif.c index e723edb9..d376b636 100644 --- a/gen/c/std/gif.c +++ b/gen/c/std/gif.c @@ -258,6 +258,11 @@ puffs_gif_status puffs_gif_lzw_decoder_decode(puffs_gif_lzw_decoder* self, // TODO: look for (ifdef) the x86 architecture and cast the pointer? Only do so // if a benchmark justifies the additional code path. +#define PUFFS_U16BE(p) (((uint16_t)(p[0]) << 8) | ((uint16_t)(p[1]) << 0)) +#define PUFFS_U16LE(p) (((uint16_t)(p[0]) << 0) | ((uint16_t)(p[1]) << 8)) +#define PUFFS_U32BE(p) \ + (((uint32_t)(p[0]) << 24) | ((uint32_t)(p[1]) << 16) | \ + ((uint32_t)(p[2]) << 8) | ((uint32_t)(p[3]) << 0)) #define PUFFS_U32LE(p) \ (((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | \ ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24)) diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index 8c7bb4df..99363db6 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -125,6 +125,13 @@ typedef int32_t puffs_flate_status; -1157040117 // 0xbb08f80b #define PUFFS_FLATE_ERROR_MISSING_END_OF_BLOCK_CODE -1157040116 // 0xbb08f80c #define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040115 // 0xbb08f80d +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_METHOD \ + -1157040114 // 0xbb08f80e +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_COMPRESSION_WINDOW_SIZE \ + -1157040113 // 0xbb08f80f +#define PUFFS_FLATE_ERROR_INVALID_ZLIB_PARITY_CHECK -1157040112 // 0xbb08f810 +#define PUFFS_FLATE_ERROR_TODO_UNSUPPORTED_ZLIB_PRESET_DICTIONARY \ + -1157040111 // 0xbb08f811 bool puffs_flate_status_is_error(puffs_flate_status s); @@ -225,6 +232,29 @@ typedef struct { } private_impl; } puffs_flate_decoder; +typedef struct { + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. Instead, use the + // puffs_flate_zlib_decoder_etc functions. + // + // In C++, these fields would be "private", but C does not support that. + // + // It is a struct, not a struct*, so that it can be stack allocated. + struct { + puffs_flate_status status; + uint32_t magic; + uint64_t scratch; + + puffs_flate_decoder f_dec; + + struct { + uint32_t coro_susp_point; + uint16_t v_x; + uint32_t v_checksum; + } c_decode[1]; + } private_impl; +} puffs_flate_zlib_decoder; + // ---------------- Public Constructor and Destructor Prototypes // puffs_flate_decoder_constructor is a constructor function. @@ -238,12 +268,28 @@ void puffs_flate_decoder_constructor(puffs_flate_decoder* self, void puffs_flate_decoder_destructor(puffs_flate_decoder* self); +// puffs_flate_zlib_decoder_constructor is a constructor function. +// +// It should be called before any other puffs_flate_zlib_decoder_* function. +// +// Pass PUFFS_VERSION and 0 for puffs_version and for_internal_use_only. +void puffs_flate_zlib_decoder_constructor(puffs_flate_zlib_decoder* self, + uint32_t puffs_version, + uint32_t for_internal_use_only); + +void puffs_flate_zlib_decoder_destructor(puffs_flate_zlib_decoder* self); + // ---------------- Public Function Prototypes puffs_flate_status puffs_flate_decoder_decode(puffs_flate_decoder* self, puffs_base_writer1 a_dst, puffs_base_reader1 a_src); +puffs_flate_status puffs_flate_zlib_decoder_decode( + puffs_flate_zlib_decoder* self, + puffs_base_writer1 a_dst, + puffs_base_reader1 a_src); + #ifdef __cplusplus } // extern "C" #endif diff --git a/lang/check/bounds.go b/lang/check/bounds.go index 589a4ed0..140f4fbe 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -643,7 +643,9 @@ func (q *checker) bcheckExprOther(n *a.Expr, depth uint32) (*big.Int, *big.Int, case t.KeyOpenParen: // TODO: delete this hack that only matches "in.src.read_u8?()" etc. - if isInSrc(q.tm, n, t.KeyReadU8, 0) || isInSrc(q.tm, n, t.KeyReadU32LE, 0) || + if isInSrc(q.tm, n, t.KeyReadU8, 0) || + isInSrc(q.tm, n, t.KeyReadU16BE, 0) || isInSrc(q.tm, n, t.KeyReadU32BE, 0) || + isInSrc(q.tm, n, t.KeyReadU32BE, 0) || isInSrc(q.tm, n, t.KeyReadU32LE, 0) || isInSrc(q.tm, n, t.KeySkip32, 1) || isInDst(q.tm, n, t.KeyWrite, 1) || isInDst(q.tm, n, t.KeyWriteU8, 1) || isInDst(q.tm, n, t.KeyCopyFrom32, 2) || isInDst(q.tm, n, t.KeyCopyHistory32, 2) || @@ -891,6 +893,15 @@ func (q *checker) bcheckExprBinaryOp1(op t.Key, lhs *a.Expr, lMin *big.Int, lMax case t.KeyXBinaryHat: // TODO. + case t.KeyXBinaryPercent: + if lMin.Cmp(zero) < 0 { + return nil, nil, fmt.Errorf("check: modulus op argument %q is possibly negative", lhs.String(q.tm)) + } + if rMin.Cmp(zero) <= 0 { + return nil, nil, fmt.Errorf("check: modulus op argument %q is possibly non-positive", rhs.String(q.tm)) + } + return zero, big.NewInt(0).Sub(rMax, one), nil + case t.KeyXBinaryNotEq, t.KeyXBinaryLessThan, t.KeyXBinaryLessEq, t.KeyXBinaryEqEq, t.KeyXBinaryGreaterEq, t.KeyXBinaryGreaterThan, t.KeyXBinaryAnd, t.KeyXBinaryOr: return zero, one, nil diff --git a/lang/check/check.go b/lang/check/check.go index 2f43a37b..8b8ceae8 100644 --- a/lang/check/check.go +++ b/lang/check/check.go @@ -63,6 +63,7 @@ var ( // TODO: delete this. typeExprPlaceholder = a.NewTypeExpr(0, t.IDU8, nil, nil, nil) + typeExprPlaceholder16 = a.NewTypeExpr(0, t.IDU16, nil, nil, nil) typeExprPlaceholder32 = a.NewTypeExpr(0, t.IDU32, nil, nil, nil) ) diff --git a/lang/check/type.go b/lang/check/type.go index 8838ed45..3c2017e3 100644 --- a/lang/check/type.go +++ b/lang/check/type.go @@ -343,7 +343,9 @@ func (q *checker) tcheckExprOther(n *a.Expr, depth uint32) error { case t.KeyOpenParen: // n is a function call. // TODO: delete this hack that only matches "in.src.read_u8?()" etc. - if isInSrc(q.tm, n, t.KeyReadU8, 0) || isInSrc(q.tm, n, t.KeyReadU32LE, 0) || + if isInSrc(q.tm, n, t.KeyReadU8, 0) || + isInSrc(q.tm, n, t.KeyReadU16BE, 0) || isInSrc(q.tm, n, t.KeyReadU32BE, 0) || + isInSrc(q.tm, n, t.KeyReadU32BE, 0) || isInSrc(q.tm, n, t.KeyReadU32LE, 0) || isInSrc(q.tm, n, t.KeySkip32, 1) || isInDst(q.tm, n, t.KeyWrite, 1) || isInDst(q.tm, n, t.KeyWriteU8, 1) || isInDst(q.tm, n, t.KeyCopyFrom32, 2) || isInDst(q.tm, n, t.KeyCopyHistory32, 2) || @@ -361,8 +363,10 @@ func (q *checker) tcheckExprOther(n *a.Expr, depth uint32) error { return err } } - if isInSrc(q.tm, n, t.KeyReadU32LE, 0) { + if isInSrc(q.tm, n, t.KeyReadU32BE, 0) || isInSrc(q.tm, n, t.KeyReadU32LE, 0) { n.SetMType(typeExprPlaceholder32) // HACK. + } else if isInSrc(q.tm, n, t.KeyReadU16BE, 0) || isInSrc(q.tm, n, t.KeyReadU16LE, 0) { + n.SetMType(typeExprPlaceholder16) // HACK. } else { n.SetMType(typeExprPlaceholder) // HACK. } @@ -774,6 +778,11 @@ func evalConstValueBinaryOp(tm *t.Map, n *a.Expr, l *big.Int, r *big.Int) (*big. return big.NewInt(0).Or(l, r), nil case t.KeyXBinaryHat: return big.NewInt(0).Xor(l, r), nil + case t.KeyXBinaryPercent: + if r.Cmp(zero) == 0 { + return nil, fmt.Errorf("check: division by zero in const expression %q", n.String(tm)) + } + return big.NewInt(0).Mod(l, r), nil case t.KeyXBinaryNotEq: return btoi(l.Cmp(r) != 0), nil case t.KeyXBinaryLessThan: diff --git a/lang/token/list.go b/lang/token/list.go index 03aea360..97dde2c8 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -152,28 +152,30 @@ const ( KeySemicolon = Key(IDSemicolon >> KeyShift) KeyDollar = Key(IDDollar >> KeyShift) - KeyEq = Key(IDEq >> KeyShift) - KeyPlusEq = Key(IDPlusEq >> KeyShift) - KeyMinusEq = Key(IDMinusEq >> KeyShift) - KeyStarEq = Key(IDStarEq >> KeyShift) - KeySlashEq = Key(IDSlashEq >> KeyShift) - KeyShiftLEq = Key(IDShiftLEq >> KeyShift) - KeyShiftREq = Key(IDShiftREq >> KeyShift) - KeyAmpEq = Key(IDAmpEq >> KeyShift) - KeyAmpHatEq = Key(IDAmpHatEq >> KeyShift) - KeyPipeEq = Key(IDPipeEq >> KeyShift) - KeyHatEq = Key(IDHatEq >> KeyShift) + KeyEq = Key(IDEq >> KeyShift) + KeyPlusEq = Key(IDPlusEq >> KeyShift) + KeyMinusEq = Key(IDMinusEq >> KeyShift) + KeyStarEq = Key(IDStarEq >> KeyShift) + KeySlashEq = Key(IDSlashEq >> KeyShift) + KeyShiftLEq = Key(IDShiftLEq >> KeyShift) + KeyShiftREq = Key(IDShiftREq >> KeyShift) + KeyAmpEq = Key(IDAmpEq >> KeyShift) + KeyAmpHatEq = Key(IDAmpHatEq >> KeyShift) + KeyPipeEq = Key(IDPipeEq >> KeyShift) + KeyHatEq = Key(IDHatEq >> KeyShift) + KeyPercentEq = Key(IDPercentEq >> KeyShift) - KeyPlus = Key(IDPlus >> KeyShift) - KeyMinus = Key(IDMinus >> KeyShift) - KeyStar = Key(IDStar >> KeyShift) - KeySlash = Key(IDSlash >> KeyShift) - KeyShiftL = Key(IDShiftL >> KeyShift) - KeyShiftR = Key(IDShiftR >> KeyShift) - KeyAmp = Key(IDAmp >> KeyShift) - KeyAmpHat = Key(IDAmpHat >> KeyShift) - KeyPipe = Key(IDPipe >> KeyShift) - KeyHat = Key(IDHat >> KeyShift) + KeyPlus = Key(IDPlus >> KeyShift) + KeyMinus = Key(IDMinus >> KeyShift) + KeyStar = Key(IDStar >> KeyShift) + KeySlash = Key(IDSlash >> KeyShift) + KeyShiftL = Key(IDShiftL >> KeyShift) + KeyShiftR = Key(IDShiftR >> KeyShift) + KeyAmp = Key(IDAmp >> KeyShift) + KeyAmpHat = Key(IDAmpHat >> KeyShift) + KeyPipe = Key(IDPipe >> KeyShift) + KeyHat = Key(IDHat >> KeyShift) + KeyPercent = Key(IDPercent >> KeyShift) KeyNotEq = Key(IDNotEq >> KeyShift) KeyLessThan = Key(IDLessThan >> KeyShift) @@ -277,6 +279,7 @@ const ( KeyXBinaryAmpHat = Key(IDXBinaryAmpHat >> KeyShift) KeyXBinaryPipe = Key(IDXBinaryPipe >> KeyShift) KeyXBinaryHat = Key(IDXBinaryHat >> KeyShift) + KeyXBinaryPercent = Key(IDXBinaryPercent >> KeyShift) KeyXBinaryNotEq = Key(IDXBinaryNotEq >> KeyShift) KeyXBinaryLessThan = Key(IDXBinaryLessThan >> KeyShift) KeyXBinaryLessEq = Key(IDXBinaryLessEq >> KeyShift) @@ -317,28 +320,30 @@ const ( IDSemicolon = ID(0x26<>=", IDShiftREq}, - KeyAmpEq: {"&=", IDAmpEq}, - KeyAmpHatEq: {"&^=", IDAmpHatEq}, - KeyPipeEq: {"|=", IDPipeEq}, - KeyHatEq: {"^=", IDHatEq}, + KeyEq: {"=", IDEq}, + KeyPlusEq: {"+=", IDPlusEq}, + KeyMinusEq: {"-=", IDMinusEq}, + KeyStarEq: {"*=", IDStarEq}, + KeySlashEq: {"/=", IDSlashEq}, + KeyShiftLEq: {"<<=", IDShiftLEq}, + KeyShiftREq: {">>=", IDShiftREq}, + KeyAmpEq: {"&=", IDAmpEq}, + KeyAmpHatEq: {"&^=", IDAmpHatEq}, + KeyPipeEq: {"|=", IDPipeEq}, + KeyHatEq: {"^=", IDHatEq}, + KeyPercentEq: {"%=", IDPercentEq}, - KeyPlus: {"+", IDPlus}, - KeyMinus: {"-", IDMinus}, - KeyStar: {"*", IDStar}, - KeySlash: {"/", IDSlash}, - KeyShiftL: {"<<", IDShiftL}, - KeyShiftR: {">>", IDShiftR}, - KeyAmp: {"&", IDAmp}, - KeyAmpHat: {"&^", IDAmpHat}, - KeyPipe: {"|", IDPipe}, - KeyHat: {"^", IDHat}, + KeyPlus: {"+", IDPlus}, + KeyMinus: {"-", IDMinus}, + KeyStar: {"*", IDStar}, + KeySlash: {"/", IDSlash}, + KeyShiftL: {"<<", IDShiftL}, + KeyShiftR: {">>", IDShiftR}, + KeyAmp: {"&", IDAmp}, + KeyAmpHat: {"&^", IDAmpHat}, + KeyPipe: {"|", IDPipe}, + KeyHat: {"^", IDHat}, + KeyPercent: {"%", IDPercent}, // TODO: tilde-ops such as ~+ for ignoring overflow. Swift defines overflow // ops for +, - and *. Swift syntax uses a leading &. Puffs uses a leading // ~ as it is less ambiguous with the &^ operator (and any potential use of @@ -691,6 +699,10 @@ var lexers = [256][]suffixLexer{ {"=", IDSlashEq}, {"", IDSlash}, }, + '%': { + {"=", IDPercentEq}, + {"", IDPercent}, + }, '=': { {"=", IDEqEq}, {"", IDEq}, @@ -724,6 +736,7 @@ var ambiguousForms = [256]ID{ KeyXBinaryAmpHat: IDAmpHat, KeyXBinaryPipe: IDPipe, KeyXBinaryHat: IDHat, + KeyXBinaryPercent: IDPercent, KeyXBinaryNotEq: IDNotEq, KeyXBinaryLessThan: IDLessThan, KeyXBinaryLessEq: IDLessEq, @@ -750,16 +763,17 @@ var unaryForms = [256]ID{ } var binaryForms = [256]ID{ - KeyPlusEq: IDXBinaryPlus, - KeyMinusEq: IDXBinaryMinus, - KeyStarEq: IDXBinaryStar, - KeySlashEq: IDXBinarySlash, - KeyShiftLEq: IDXBinaryShiftL, - KeyShiftREq: IDXBinaryShiftR, - KeyAmpEq: IDXBinaryAmp, - KeyAmpHatEq: IDXBinaryAmpHat, - KeyPipeEq: IDXBinaryPipe, - KeyHatEq: IDXBinaryHat, + KeyPlusEq: IDXBinaryPlus, + KeyMinusEq: IDXBinaryMinus, + KeyStarEq: IDXBinaryStar, + KeySlashEq: IDXBinarySlash, + KeyShiftLEq: IDXBinaryShiftL, + KeyShiftREq: IDXBinaryShiftR, + KeyAmpEq: IDXBinaryAmp, + KeyAmpHatEq: IDXBinaryAmpHat, + KeyPipeEq: IDXBinaryPipe, + KeyHatEq: IDXBinaryHat, + KeyPercentEq: IDXBinaryPercent, KeyPlus: IDXBinaryPlus, KeyMinus: IDXBinaryMinus, @@ -771,6 +785,7 @@ var binaryForms = [256]ID{ KeyAmpHat: IDXBinaryAmpHat, KeyPipe: IDXBinaryPipe, KeyHat: IDXBinaryHat, + KeyPercent: IDXBinaryPercent, KeyNotEq: IDXBinaryNotEq, KeyLessThan: IDXBinaryLessThan, KeyLessEq: IDXBinaryLessEq, diff --git a/std/flate/decode_zlib.puffs b/std/flate/decode_zlib.puffs new file mode 100644 index 00000000..be9a61de --- /dev/null +++ b/std/flate/decode_zlib.puffs @@ -0,0 +1,31 @@ +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +pub error "invalid zlib compression method" +pub error "invalid zlib compression window size" +pub error "invalid zlib parity check" + +pub error "TODO: unsupported zlib preset dictionary" + +pub struct zlib_decoder?( + dec decoder, +) + +pub func zlib_decoder.decode?(dst writer1, src reader1)() { + var x u16 = in.src.read_u16be?() + if ((x >> 8) & 0x0F) != 0x08 { + return error "invalid zlib compression method" + } + if (x >> 12) > 0x07 { + return error "invalid zlib compression window size" + } + if (x & 0x20) != 0 { + return error "TODO: unsupported zlib preset dictionary" + } + if (x % 31) != 0 { + return error "invalid zlib parity check" + } + this.dec.decode?(dst:in.dst, src:in.src) + var checksum u32 = in.src.read_u32be?() + // TODO: verify the checksum. +} diff --git a/test/c/std/flate.c b/test/c/std/flate.c index 8f3f55f4..71a1d460 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -116,6 +116,20 @@ const char* puffs_flate_decode(puffs_base_buf1* dst, puffs_base_buf1* src) { return NULL; } +const char* puffs_zlib_decode(puffs_base_buf1* dst, puffs_base_buf1* src) { + puffs_flate_zlib_decoder dec; + puffs_flate_zlib_decoder_constructor(&dec, PUFFS_VERSION, 0); + puffs_base_writer1 dst_writer = {.buf = dst}; + puffs_base_reader1 src_reader = {.buf = src}; + puffs_flate_status s = + puffs_flate_zlib_decoder_decode(&dec, dst_writer, src_reader); + puffs_flate_zlib_decoder_destructor(&dec); + if (s) { + return puffs_flate_status_string(s); + } + return NULL; +} + void test_puffs_flate_decode_256_bytes() { proc_funcname = __func__; test_buf1_buf1(puffs_flate_decode, &flate_256_bytes_gt); @@ -339,6 +353,16 @@ void test_puffs_flate_table_redirect() { } } +void test_puffs_zlib_decode_midsummer() { + proc_funcname = __func__; + test_buf1_buf1(puffs_zlib_decode, &zlib_midsummer_gt); +} + +void test_puffs_zlib_decode_pi() { + proc_funcname = __func__; + test_buf1_buf1(puffs_zlib_decode, &zlib_pi_gt); +} + // ---------------- Mimic Tests #ifdef PUFFS_MIMIC @@ -482,6 +506,8 @@ proc tests[] = { test_puffs_flate_decode_romeo_fixed, // test_puffs_flate_decode_split_src, // test_puffs_flate_table_redirect, // + test_puffs_zlib_decode_midsummer, // + test_puffs_zlib_decode_pi, // #ifdef PUFFS_MIMIC