From 70c42bff105e0275fa70a3cd19678ee106d13464 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Wed, 22 Jul 2020 21:50:51 +1000 Subject: [PATCH] Break std/cbor string tokens on UTF-8 boundaries --- internal/cgen/builtin.go | 14 ++++++++ lang/builtin/builtin.go | 1 + lang/token/list.go | 48 ++++++++++++++------------ release/c/wuffs-unsupported-snapshot.c | 9 +++-- std/cbor/decode_cbor.wuffs | 10 +++--- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index 6eeccb8a..e4eff2ac 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -168,6 +168,20 @@ func (g *gen) writeBuiltinIOReader(b *buffer, recv *a.Expr, method t.ID, args [] } switch method { + case t.IDAvailableValidUTF8: + name, err := g.ioRecvName(recv) + if err != nil { + return err + } + b.printf("((uint64_t)(wuffs_base__utf_8__longest_valid_prefix(%s%s,\n"+ + "((size_t)(wuffs_base__u64__min(%s%s - %s%s, ", + iopPrefix, name, io2Prefix, name, iopPrefix, name) + if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil { + return err + } + b.writes("))))))") + return nil + case t.IDUndoByte: b.printf("(%s%s--, wuffs_base__make_empty_struct())", iopPrefix, name) return nil diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index edb7baed..52802a5d 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -388,6 +388,7 @@ var Funcs = []string{ "io_reader.peek_u64le_at(offset: u32[..= 0xFFF8]) u64", "io_reader.available() u64", + "io_reader.available_valid_utf_8(up_to: u64) u64", "io_reader.count_since(mark: u64) u64", "io_reader.is_closed() bool", "io_reader.mark() u64", diff --git a/lang/token/list.go b/lang/token/list.go index 2f48fc54..5600ad2d 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -651,17 +651,19 @@ const ( IDIsOK = ID(0x231) IDIsSuspension = ID(0x232) - IDAvailable = ID(0x240) - IDHeight = ID(0x241) - IDLength = ID(0x242) - IDPrefix = ID(0x243) - IDRow = ID(0x244) - IDStride = ID(0x245) - IDSuffix = ID(0x246) - IDWidth = ID(0x247) - IDIO = ID(0x248) - IDLimit = ID(0x249) - IDData = ID(0x24A) + IDAvailable = ID(0x240) + IDAvailableValidUTF8 = ID(0x241) + + IDData = ID(0x250) + IDHeight = ID(0x251) + IDIO = ID(0x252) + IDLength = ID(0x253) + IDLimit = ID(0x254) + IDPrefix = ID(0x255) + IDRow = ID(0x256) + IDStride = ID(0x257) + IDSuffix = ID(0x258) + IDWidth = ID(0x259) IDSwizzleInterleavedFromReader = ID(0x280) ) @@ -1014,17 +1016,19 @@ var builtInsByID = [nBuiltInIDs]string{ IDIsOK: "is_ok", IDIsSuspension: "is_suspension", - IDAvailable: "available", - IDHeight: "height", - IDLength: "length", - IDPrefix: "prefix", - IDRow: "row", - IDStride: "stride", - IDSuffix: "suffix", - IDWidth: "width", - IDIO: "io", - IDLimit: "limit", - IDData: "data", + IDAvailable: "available", + IDAvailableValidUTF8: "available_valid_utf_8", + + IDData: "data", + IDHeight: "height", + IDIO: "io", + IDLength: "length", + IDLimit: "limit", + IDPrefix: "prefix", + IDRow: "row", + IDStride: "stride", + IDSuffix: "suffix", + IDWidth: "width", IDSwizzleInterleavedFromReader: "swizzle_interleaved_from_reader", } diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 714c72c8..58b7011f 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -16686,13 +16686,12 @@ wuffs_cbor__decoder__decode_tokens( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6); goto label__1__continue; } - v_n64 = wuffs_base__u64__min(v_string_length, ((uint64_t)(io2_a_src - iop_a_src))); + v_n64 = wuffs_base__u64__min(v_string_length, 65535); + v_n64 = ((uint64_t)(wuffs_base__utf_8__longest_valid_prefix(iop_a_src, + ((size_t)(wuffs_base__u64__min(io2_a_src - iop_a_src, v_n64)))))); v_token_length = ((uint32_t)((v_n64 & 65535))); - if (v_n64 > 65535) { - v_token_length = 65535; - } if (v_token_length <= 0) { - if (a_src && a_src->meta.closed) { + if ((a_src && a_src->meta.closed) || (((uint64_t)(io2_a_src - iop_a_src)) >= 4)) { status = wuffs_base__make_status(wuffs_cbor__error__bad_input); goto exit; } diff --git a/std/cbor/decode_cbor.wuffs b/std/cbor/decode_cbor.wuffs index bc76df39..f0aeb888 100644 --- a/std/cbor/decode_cbor.wuffs +++ b/std/cbor/decode_cbor.wuffs @@ -361,14 +361,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor yield? base."$short write" continue } - n64 = string_length.min(a: args.src.available()) + n64 = string_length.min(a: 0xFFFF) + n64 = args.src.available_valid_utf_8(up_to: n64) token_length = (n64 & 0xFFFF) as base.u32 - if n64 > 0xFFFF { - token_length = 0xFFFF - } - // TODO: walk token_length back to a UTF-8 boundary. if token_length <= 0 { - if args.src.is_closed() { + // The longest UTF-8 code point is 4 bytes. + if args.src.is_closed() or (args.src.available() >= 4) { return "#bad input" } yield? base."$short read"