diff --git a/internal/cgen/base/io-private.h b/internal/cgen/base/io-private.h index 83e8f892..a3081b90 100644 --- a/internal/cgen/base/io-private.h +++ b/internal/cgen/base/io-private.h @@ -161,6 +161,41 @@ wuffs_base__io_writer__copy_n32_from_slice(uint8_t** ptr_iop_w, return (uint32_t)(n); } +// wuffs_base__io_reader__match7 returns whether the io_reader's upcoming bytes +// start with the given prefix (up to 7 bytes long). It is peek-like, not +// read-like, in that there are no side-effects. +// +// The low 3 bits of a hold the prefix length, n. +// +// The high 56 bits of a hold the prefix itself, in little-endian order. The +// first prefix byte is in bits 8..=15, the second prefix byte is in bits +// 16..=23, etc. The high (8 * (7 - n)) bits are ignored. +// +// There are three possible return values: +// - 0 means success. +// - 1 means inconclusive, equivalent to "$short read". +// - 2 means failure. +static inline uint32_t // +wuffs_base__io_reader__match7(uint8_t* iop_r, uint8_t* io2_r, uint64_t a) { + uint32_t n = a & 7; + a >>= 8; + if ((io2_r - iop_r) >= 8) { + uint64_t x = wuffs_base__load_u64le(iop_r); + uint32_t shift = 8 * (8 - n); + return ((a << shift) == (x << shift)) ? 0 : 2; + } + for (; n > 0; n--) { + if (iop_r >= io2_r) { + return 1; + } else if (*iop_r != ((uint8_t)(a))) { + return 2; + } + iop_r++; + a >>= 8; + } + return 0; +} + static inline wuffs_base__io_buffer* // wuffs_base__io_reader__set(wuffs_base__io_buffer* b, uint8_t** ptr_iop_r, diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index 196ef8d9..19b38933 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -168,6 +168,14 @@ func (g *gen) writeBuiltinIOReader(b *buffer, recv *a.Expr, method t.ID, args [] b.printf("((uint64_t)(%s%s - %s%s))", iopPrefix, name, io0Prefix, name) return nil + case t.IDMatch7: + b.printf("wuffs_base__io_reader__match7(%s%s, %s%s,", iopPrefix, name, io2Prefix, name) + if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil { + return err + } + b.writeb(')') + return nil + case t.IDPosition: b.printf("wuffs_base__u64__sat_add(%s->meta.pos, ((uint64_t)(%s%s - %s%s)))", name, iopPrefix, name, io0Prefix, name) diff --git a/internal/cgen/data.go b/internal/cgen/data.go index f8aea32b..47b52ae7 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -211,8 +211,9 @@ const baseIOPrivateH = "" + " } else {\n n = (size_t)(length);\n }\n // TODO: unrolling by 3 seems best for the std/deflate benchmarks, but that\n // is mostly because 3 is the minimum length for the deflate format. This\n // function implementation shouldn't overfit to that one format. Perhaps the\n // copy_n32_from_history Wuffs method should also take an unroll hint\n // argument, and the cgen can look if that argument is the constant\n // expression '3'.\n //\n // See also wuffs_base__io_writer__copy_n32_from_history_fast below.\n //\n // Alternatively, or additionally, have a sloppy_copy_n32_from_history method\n // that copies 8 bytes at a time, possibly writing more than length bytes?\n for (; n >= 3; n -= 3) {\n *p++ = *q++;\n *p++ = *q++;\n *p++ = *q++;\n }\n for (; n; n--) {\n *p++ = *q++;\n }\n *ptr_iop_w = p;\n return length;\n}\n\n// wuffs_base__io_writer__copy_n32_from_history_fast is like the\n// wuffs_base__io_writer__copy_n32_from_history function above, but has\n// stronger pre-conditions. The caller needs to prove" + " that:\n// - distance > 0\n// - distance <= (*ptr_iop_w - io1_w)\n// - length <= (io2_w - *ptr_iop_w)\nstatic inline uint32_t //\nwuffs_base__io_writer__copy_n32_from_history_fast(uint8_t** ptr_iop_w,\n uint8_t* io1_w,\n uint8_t* io2_w,\n uint32_t length,\n uint32_t distance) {\n uint8_t* p = *ptr_iop_w;\n uint8_t* q = p - distance;\n uint32_t n = length;\n for (; n >= 3; n -= 3) {\n *p++ = *q++;\n *p++ = *q++;\n *p++ = *q++;\n }\n for (; n; n--) {\n *p++ = *q++;\n }\n *ptr_iop_w = p;\n return length;\n}\n\nstatic inline uint32_t //\nwuffs_base__io_writer__copy_n32_from_reader(uint8_t** ptr_iop_w,\n uint8_t* io2_w,\n uint32_t length,\n uint8_t** ptr_iop_r,\n " + " uint8_t* io2_r) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = length;\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n uint8_t* iop_r = *ptr_iop_r;\n if (n > ((size_t)(io2_r - iop_r))) {\n n = (size_t)(io2_r - iop_r);\n }\n if (n > 0) {\n memmove(iop_w, iop_r, n);\n *ptr_iop_w += n;\n *ptr_iop_r += n;\n }\n return (uint32_t)(n);\n}\n\nstatic inline uint64_t //\nwuffs_base__io_writer__copy_from_slice(uint8_t** ptr_iop_w,\n uint8_t* io2_w,\n wuffs_base__slice_u8 src) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = src.len;\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n if (n > 0) {\n memmove(iop_w, src.ptr, n);\n *ptr_iop_w += n;\n }\n return (uint64_t)(n);\n}\n\nstatic inline uint32_t //\nwuffs_base__io_writer__copy_n32_from_slice(uint8_t** ptr_iop_w,\n uint8_t* io2_w,\n uint32_t" + - " length,\n wuffs_base__slice_u8 src) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = src.len;\n if (n > length) {\n n = length;\n }\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n if (n > 0) {\n memmove(iop_w, src.ptr, n);\n *ptr_iop_w += n;\n }\n return (uint32_t)(n);\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_reader__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_r,\n uint8_t** ptr_io0_r,\n uint8_t** ptr_io1_r,\n uint8_t** ptr_io2_r,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = data.len;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_r = data.ptr;\n *ptr_io0_r = data.ptr;\n *ptr_io1_r = data.ptr;\n *ptr_io2_r = data.ptr + data.len;\n\n return b;\n}\n\nstatic inline wuffs_base__slice_u8 //\nwuffs_base__io_reader__take(uint8_t** ptr_iop_r, uint8_t* io2" + - "_r, uint64_t n) {\n if (n <= ((size_t)(io2_r - *ptr_iop_r))) {\n uint8_t* p = *ptr_iop_r;\n *ptr_iop_r += n;\n return wuffs_base__make_slice_u8(p, n);\n }\n return wuffs_base__make_slice_u8(NULL, 0);\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_writer__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_w,\n uint8_t** ptr_io0_w,\n uint8_t** ptr_io1_w,\n uint8_t** ptr_io2_w,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = 0;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_w = data.ptr;\n *ptr_io0_w = data.ptr;\n *ptr_io1_w = data.ptr;\n *ptr_io2_w = data.ptr + data.len;\n\n return b;\n}\n\n" + + " length,\n wuffs_base__slice_u8 src) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = src.len;\n if (n > length) {\n n = length;\n }\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n if (n > 0) {\n memmove(iop_w, src.ptr, n);\n *ptr_iop_w += n;\n }\n return (uint32_t)(n);\n}\n\n// wuffs_base__io_reader__match7 returns whether the io_reader's upcoming bytes\n// start with the given prefix (up to 7 bytes long). It is peek-like, not\n// read-like, in that there are no side-effects.\n//\n// The low 3 bits of a hold the prefix length, n.\n//\n// The high 56 bits of a hold the prefix itself, in little-endian order. The\n// first prefix byte is in bits 8..=15, the second prefix byte is in bits\n// 16..=23, etc. The high (8 * (7 - n)) bits are ignored.\n//\n// There are three possible return values:\n// - 0 means success.\n// - 1 means inconclusive, equivalent to \"$short read\".\n// - 2 means failure.\nstatic inline uint32_t //\nwuffs_base__io_reader__match7(ui" + + "nt8_t* iop_r, uint8_t* io2_r, uint64_t a) {\n uint32_t n = a & 7;\n a >>= 8;\n if ((io2_r - iop_r) >= 8) {\n uint64_t x = wuffs_base__load_u64le(iop_r);\n uint32_t shift = 8 * (8 - n);\n return ((a << shift) == (x << shift)) ? 0 : 2;\n }\n for (; n > 0; n--) {\n if (iop_r >= io2_r) {\n return 1;\n } else if (*iop_r != ((uint8_t)(a))) {\n return 2;\n }\n iop_r++;\n a >>= 8;\n }\n return 0;\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_reader__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_r,\n uint8_t** ptr_io0_r,\n uint8_t** ptr_io1_r,\n uint8_t** ptr_io2_r,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = data.len;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_r = data.ptr;\n *ptr_io0_r = data.ptr;\n *ptr_io1_r = data.ptr;\n *ptr_io2_r = data.ptr + data.len;\n\n return b;\n}\n\nstatic inline wuffs_base__sli" + + "ce_u8 //\nwuffs_base__io_reader__take(uint8_t** ptr_iop_r, uint8_t* io2_r, uint64_t n) {\n if (n <= ((size_t)(io2_r - *ptr_iop_r))) {\n uint8_t* p = *ptr_iop_r;\n *ptr_iop_r += n;\n return wuffs_base__make_slice_u8(p, n);\n }\n return wuffs_base__make_slice_u8(NULL, 0);\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_writer__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_w,\n uint8_t** ptr_io0_w,\n uint8_t** ptr_io1_w,\n uint8_t** ptr_io2_w,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = 0;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_w = data.ptr;\n *ptr_io0_w = data.ptr;\n *ptr_io1_w = data.ptr;\n *ptr_io2_w = data.ptr + data.len;\n\n return b;\n}\n\n" + "" + "// ---------------- I/O (Utility)\n\n#define wuffs_base__utility__empty_io_reader wuffs_base__empty_io_reader\n#define wuffs_base__utility__empty_io_writer wuffs_base__empty_io_writer\n" + "" diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 7d807cf8..1bf829f1 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -252,6 +252,7 @@ var Funcs = []string{ "io_reader.count_since(mark: u64) u64", "io_reader.is_closed() bool", "io_reader.mark() u64", + "io_reader.match7(a: u64) u32[..= 2]", "io_reader.position() u64", "io_reader.since(mark: u64) slice u8", "io_reader.take!(n: u64) slice u8", diff --git a/lang/token/list.go b/lang/token/list.go index d16773bd..3ade9bca 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -493,12 +493,15 @@ const ( IDHistoryAvailable = ID(0x162) IDIsClosed = ID(0x163) IDMark = ID(0x164) - IDPosition = ID(0x165) - IDSince = ID(0x166) - IDSkip = ID(0x167) - IDSkip32 = ID(0x168) - IDSkip32Fast = ID(0x169) - IDTake = ID(0x16A) + IDMatch15 = ID(0x165) + IDMatch31 = ID(0x166) + IDMatch7 = ID(0x167) + IDPosition = ID(0x168) + IDSince = ID(0x169) + IDSkip = ID(0x16A) + IDSkip32 = ID(0x16B) + IDSkip32Fast = ID(0x16C) + IDTake = ID(0x16D) IDCopyFromSlice = ID(0x170) IDCopyN32FromHistory = ID(0x171) @@ -844,6 +847,9 @@ var builtInsByID = [nBuiltInIDs]string{ IDHistoryAvailable: "history_available", IDIsClosed: "is_closed", IDMark: "mark", + IDMatch15: "match15", + IDMatch31: "match31", + IDMatch7: "match7", IDPosition: "position", IDSince: "since", IDSkip: "skip", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index f02f0c6f..a3d4dc46 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -6332,6 +6332,41 @@ wuffs_base__io_writer__copy_n32_from_slice(uint8_t** ptr_iop_w, return (uint32_t)(n); } +// wuffs_base__io_reader__match7 returns whether the io_reader's upcoming bytes +// start with the given prefix (up to 7 bytes long). It is peek-like, not +// read-like, in that there are no side-effects. +// +// The low 3 bits of a hold the prefix length, n. +// +// The high 56 bits of a hold the prefix itself, in little-endian order. The +// first prefix byte is in bits 8..=15, the second prefix byte is in bits +// 16..=23, etc. The high (8 * (7 - n)) bits are ignored. +// +// There are three possible return values: +// - 0 means success. +// - 1 means inconclusive, equivalent to "$short read". +// - 2 means failure. +static inline uint32_t // +wuffs_base__io_reader__match7(uint8_t* iop_r, uint8_t* io2_r, uint64_t a) { + uint32_t n = a & 7; + a >>= 8; + if ((io2_r - iop_r) >= 8) { + uint64_t x = wuffs_base__load_u64le(iop_r); + uint32_t shift = 8 * (8 - n); + return ((a << shift) == (x << shift)) ? 0 : 2; + } + for (; n > 0; n--) { + if (iop_r >= io2_r) { + return 1; + } else if (*iop_r != ((uint8_t)(a))) { + return 2; + } + iop_r++; + a >>= 8; + } + return 0; +} + static inline wuffs_base__io_buffer* // wuffs_base__io_reader__set(wuffs_base__io_buffer* b, uint8_t** ptr_iop_r, @@ -18508,6 +18543,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, uint32_t v_depth = 0; uint32_t v_stack_byte = 0; uint32_t v_stack_bit = 0; + uint32_t v_match = 0; uint8_t v_c = 0; uint8_t v_expect = 0; uint8_t v_expect_after_value = 0; @@ -18894,8 +18930,62 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } goto label_0_continue; } else { - status = wuffs_base__make_status(wuffs_json__error__bad_input); - goto exit; + if (0 == (v_expect & 1)) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + v_match = 2; + if (v_c == 110) { + v_match = + wuffs_base__io_reader__match7(iop_a_src, io2_a_src, 465676103172); + if (v_match == 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + status = wuffs_base__make_status( + wuffs_json__error__internal_error_inconsistent_i_o); + goto exit; + } + (iop_a_src += 4, wuffs_base__make_empty_struct()); + } + } else if (v_c == 102) { + v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src, + 111546413966853); + if (v_match == 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(4194369)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(5)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + if (((uint64_t)(io2_a_src - iop_a_src)) < 5) { + status = wuffs_base__make_status( + wuffs_json__error__internal_error_inconsistent_i_o); + goto exit; + } + (iop_a_src += 5, wuffs_base__make_empty_struct()); + } + } else if (v_c == 116) { + v_match = + wuffs_base__io_reader__match7(iop_a_src, io2_a_src, 435762131972); + if (v_match == 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(4194433)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + status = wuffs_base__make_status( + wuffs_json__error__internal_error_inconsistent_i_o); + goto exit; + } + (iop_a_src += 4, wuffs_base__make_empty_struct()); + } + } + if (v_match == 1) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8); + goto label_0_continue; + } else if (v_match == 2) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } } if (v_depth == 0) { goto label_0_break; diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index bbcb73df..f94280fa 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -61,6 +61,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var depth : base.u32[..= 1024] var stack_byte : base.u32[..= (1024 / 32) - 1] var stack_bit : base.u32[..= 31] + var match : base.u32[..= 2] var c : base.u8 // expect is a bitmask of what the next token can be: @@ -391,8 +392,48 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { continue.outer } else { - // TODO: literals (false, true, null). - return "#bad input" + if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE. + return "#bad input" + } + match = 2 + if c == 0x6E { // 0x6E is 'n'. + match = args.src.match7(a: 0x6C_6C75_6E04) // 4 bytes "null". + if match == 0 { + args.dst.write_fast_token!(value: 0x40_0021, length: 4) + if args.src.available() < 4 { + return "#internal error: inconsistent I/O" + } + args.src.skip32_fast!(actual: 4, worst_case: 4) + } + + } else if c == 0x66 { // 0x66 is 'f'. + match = args.src.match7(a: 0x6573_6C61_6605) // 5 bytes "false". + if match == 0 { + args.dst.write_fast_token!(value: 0x40_0041, length: 5) + if args.src.available() < 5 { + return "#internal error: inconsistent I/O" + } + args.src.skip32_fast!(actual: 5, worst_case: 5) + } + + } else if c == 0x74 { // 0x74 is 't'. + match = args.src.match7(a: 0x65_7572_7404) // 4 bytes "true". + if match == 0 { + args.dst.write_fast_token!(value: 0x40_0081, length: 4) + if args.src.available() < 4 { + return "#internal error: inconsistent I/O" + } + args.src.skip32_fast!(actual: 4, worst_case: 4) + } + + } + + if match == 1 { + yield? base."$short read" + continue.outer + } else if match == 2 { + return "#bad input" + } } // We've just parsed a value.