Break std/cbor string tokens on UTF-8 boundaries

This commit is contained in:
Nigel Tao committed 2020-07-22 21:50:51 +10:00
1 parent 702c7b299f
commit 70c42bff10
5 files changed
+49 -33

No files matched your search

+14
View File
@@ -168,6 +168,20 @@ func (g *gen) writeBuiltinIOReader(b *buffer, recv *a.Expr, method t.ID, args []
}
switch method {
case t.IDAvailableValidUTF8:
name, err := g.ioRecvName(recv)
if err != nil {
return err
}
b.printf("((uint64_t)(wuffs_base__utf_8__longest_valid_prefix(%s%s,\n"+
"((size_t)(wuffs_base__u64__min(%s%s - %s%s, ",
iopPrefix, name, io2Prefix, name, iopPrefix, name)
if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil {
return err
}
b.writes("))))))")
return nil
case t.IDUndoByte:
b.printf("(%s%s--, wuffs_base__make_empty_struct())", iopPrefix, name)
return nil
+1
View File
@@ -388,6 +388,7 @@ var Funcs = []string{
"io_reader.peek_u64le_at(offset: u32[..= 0xFFF8]) u64",
"io_reader.available() u64",
"io_reader.available_valid_utf_8(up_to: u64) u64",
"io_reader.count_since(mark: u64) u64",
"io_reader.is_closed() bool",
"io_reader.mark() u64",
+26 -22
View File
@@ -651,17 +651,19 @@ const (
IDIsOK = ID(0x231)
IDIsSuspension = ID(0x232)
IDAvailable = ID(0x240)
IDHeight = ID(0x241)
IDLength = ID(0x242)
IDPrefix = ID(0x243)
IDRow = ID(0x244)
IDStride = ID(0x245)
IDSuffix = ID(0x246)
IDWidth = ID(0x247)
IDIO = ID(0x248)
IDLimit = ID(0x249)
IDData = ID(0x24A)
IDAvailable = ID(0x240)
IDAvailableValidUTF8 = ID(0x241)
IDData = ID(0x250)
IDHeight = ID(0x251)
IDIO = ID(0x252)
IDLength = ID(0x253)
IDLimit = ID(0x254)
IDPrefix = ID(0x255)
IDRow = ID(0x256)
IDStride = ID(0x257)
IDSuffix = ID(0x258)
IDWidth = ID(0x259)
IDSwizzleInterleavedFromReader = ID(0x280)
)
@@ -1014,17 +1016,19 @@ var builtInsByID = [nBuiltInIDs]string{
IDIsOK: "is_ok",
IDIsSuspension: "is_suspension",
IDAvailable: "available",
IDHeight: "height",
IDLength: "length",
IDPrefix: "prefix",
IDRow: "row",
IDStride: "stride",
IDSuffix: "suffix",
IDWidth: "width",
IDIO: "io",
IDLimit: "limit",
IDData: "data",
IDAvailable: "available",
IDAvailableValidUTF8: "available_valid_utf_8",
IDData: "data",
IDHeight: "height",
IDIO: "io",
IDLength: "length",
IDLimit: "limit",
IDPrefix: "prefix",
IDRow: "row",
IDStride: "stride",
IDSuffix: "suffix",
IDWidth: "width",
IDSwizzleInterleavedFromReader: "swizzle_interleaved_from_reader",
}
+4 -5
View File
@@ -16686,13 +16686,12 @@ wuffs_cbor__decoder__decode_tokens(
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6);
goto label__1__continue;
}
v_n64 = wuffs_base__u64__min(v_string_length, ((uint64_t)(io2_a_src - iop_a_src)));
v_n64 = wuffs_base__u64__min(v_string_length, 65535);
v_n64 = ((uint64_t)(wuffs_base__utf_8__longest_valid_prefix(iop_a_src,
((size_t)(wuffs_base__u64__min(io2_a_src - iop_a_src, v_n64))))));
v_token_length = ((uint32_t)((v_n64 & 65535)));
if (v_n64 > 65535) {
v_token_length = 65535;
}
if (v_token_length <= 0) {
if (a_src && a_src->meta.closed) {
if ((a_src && a_src->meta.closed) || (((uint64_t)(io2_a_src - iop_a_src)) >= 4)) {
status = wuffs_base__make_status(wuffs_cbor__error__bad_input);
goto exit;
}
+4 -6
View File
@@ -361,14 +361,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
yield? base."$short write"
continue
}
n64 = string_length.min(a: args.src.available())
n64 = string_length.min(a: 0xFFFF)
n64 = args.src.available_valid_utf_8(up_to: n64)
token_length = (n64 & 0xFFFF) as base.u32
if n64 > 0xFFFF {
token_length = 0xFFFF
}
// TODO: walk token_length back to a UTF-8 boundary.
if token_length <= 0 {
if args.src.is_closed() {
// The longest UTF-8 code point is 4 bytes.
if args.src.is_closed() or (args.src.available() >= 4) {
return "#bad input"
}
yield? base."$short read"