Have std/json decode null, false and true

This commit is contained in:
Nigel Tao committed 2020-02-21 11:16:27 +11:00
1 parent 9a5b635778
commit c92c0f50e4
7 files changed
+194 -12

No files matched your search

+35
View File
@@ -161,6 +161,41 @@ wuffs_base__io_writer__copy_n32_from_slice(uint8_t** ptr_iop_w,
return (uint32_t)(n);
}
// wuffs_base__io_reader__match7 returns whether the io_reader's upcoming bytes
// start with the given prefix (up to 7 bytes long). It is peek-like, not
// read-like, in that there are no side-effects.
//
// The low 3 bits of a hold the prefix length, n.
//
// The high 56 bits of a hold the prefix itself, in little-endian order. The
// first prefix byte is in bits 8..=15, the second prefix byte is in bits
// 16..=23, etc. The high (8 * (7 - n)) bits are ignored.
//
// There are three possible return values:
// - 0 means success.
// - 1 means inconclusive, equivalent to "$short read".
// - 2 means failure.
static inline uint32_t //
wuffs_base__io_reader__match7(uint8_t* iop_r, uint8_t* io2_r, uint64_t a) {
uint32_t n = a & 7;
a >>= 8;
if ((io2_r - iop_r) >= 8) {
uint64_t x = wuffs_base__load_u64le(iop_r);
uint32_t shift = 8 * (8 - n);
return ((a << shift) == (x << shift)) ? 0 : 2;
}
for (; n > 0; n--) {
if (iop_r >= io2_r) {
return 1;
} else if (*iop_r != ((uint8_t)(a))) {
return 2;
}
iop_r++;
a >>= 8;
}
return 0;
}
static inline wuffs_base__io_buffer* //
wuffs_base__io_reader__set(wuffs_base__io_buffer* b,
uint8_t** ptr_iop_r,
+8
View File
@@ -168,6 +168,14 @@ func (g *gen) writeBuiltinIOReader(b *buffer, recv *a.Expr, method t.ID, args []
b.printf("((uint64_t)(%s%s - %s%s))", iopPrefix, name, io0Prefix, name)
return nil
case t.IDMatch7:
b.printf("wuffs_base__io_reader__match7(%s%s, %s%s,", iopPrefix, name, io2Prefix, name)
if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil {
return err
}
b.writeb(')')
return nil
case t.IDPosition:
b.printf("wuffs_base__u64__sat_add(%s->meta.pos, ((uint64_t)(%s%s - %s%s)))",
name, iopPrefix, name, io0Prefix, name)
+3 -2
View File
@@ -211,8 +211,9 @@ const baseIOPrivateH = "" +
" } else {\n n = (size_t)(length);\n }\n // TODO: unrolling by 3 seems best for the std/deflate benchmarks, but that\n // is mostly because 3 is the minimum length for the deflate format. This\n // function implementation shouldn't overfit to that one format. Perhaps the\n // copy_n32_from_history Wuffs method should also take an unroll hint\n // argument, and the cgen can look if that argument is the constant\n // expression '3'.\n //\n // See also wuffs_base__io_writer__copy_n32_from_history_fast below.\n //\n // Alternatively, or additionally, have a sloppy_copy_n32_from_history method\n // that copies 8 bytes at a time, possibly writing more than length bytes?\n for (; n >= 3; n -= 3) {\n *p++ = *q++;\n *p++ = *q++;\n *p++ = *q++;\n }\n for (; n; n--) {\n *p++ = *q++;\n }\n *ptr_iop_w = p;\n return length;\n}\n\n// wuffs_base__io_writer__copy_n32_from_history_fast is like the\n// wuffs_base__io_writer__copy_n32_from_history function above, but has\n// stronger pre-conditions. The caller needs to prove" +
" that:\n// - distance > 0\n// - distance <= (*ptr_iop_w - io1_w)\n// - length <= (io2_w - *ptr_iop_w)\nstatic inline uint32_t //\nwuffs_base__io_writer__copy_n32_from_history_fast(uint8_t** ptr_iop_w,\n uint8_t* io1_w,\n uint8_t* io2_w,\n uint32_t length,\n uint32_t distance) {\n uint8_t* p = *ptr_iop_w;\n uint8_t* q = p - distance;\n uint32_t n = length;\n for (; n >= 3; n -= 3) {\n *p++ = *q++;\n *p++ = *q++;\n *p++ = *q++;\n }\n for (; n; n--) {\n *p++ = *q++;\n }\n *ptr_iop_w = p;\n return length;\n}\n\nstatic inline uint32_t //\nwuffs_base__io_writer__copy_n32_from_reader(uint8_t** ptr_iop_w,\n uint8_t* io2_w,\n uint32_t length,\n uint8_t** ptr_iop_r,\n " +
" uint8_t* io2_r) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = length;\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n uint8_t* iop_r = *ptr_iop_r;\n if (n > ((size_t)(io2_r - iop_r))) {\n n = (size_t)(io2_r - iop_r);\n }\n if (n > 0) {\n memmove(iop_w, iop_r, n);\n *ptr_iop_w += n;\n *ptr_iop_r += n;\n }\n return (uint32_t)(n);\n}\n\nstatic inline uint64_t //\nwuffs_base__io_writer__copy_from_slice(uint8_t** ptr_iop_w,\n uint8_t* io2_w,\n wuffs_base__slice_u8 src) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = src.len;\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n if (n > 0) {\n memmove(iop_w, src.ptr, n);\n *ptr_iop_w += n;\n }\n return (uint64_t)(n);\n}\n\nstatic inline uint32_t //\nwuffs_base__io_writer__copy_n32_from_slice(uint8_t** ptr_iop_w,\n uint8_t* io2_w,\n uint32_t" +
" length,\n wuffs_base__slice_u8 src) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = src.len;\n if (n > length) {\n n = length;\n }\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n if (n > 0) {\n memmove(iop_w, src.ptr, n);\n *ptr_iop_w += n;\n }\n return (uint32_t)(n);\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_reader__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_r,\n uint8_t** ptr_io0_r,\n uint8_t** ptr_io1_r,\n uint8_t** ptr_io2_r,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = data.len;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_r = data.ptr;\n *ptr_io0_r = data.ptr;\n *ptr_io1_r = data.ptr;\n *ptr_io2_r = data.ptr + data.len;\n\n return b;\n}\n\nstatic inline wuffs_base__slice_u8 //\nwuffs_base__io_reader__take(uint8_t** ptr_iop_r, uint8_t* io2" +
"_r, uint64_t n) {\n if (n <= ((size_t)(io2_r - *ptr_iop_r))) {\n uint8_t* p = *ptr_iop_r;\n *ptr_iop_r += n;\n return wuffs_base__make_slice_u8(p, n);\n }\n return wuffs_base__make_slice_u8(NULL, 0);\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_writer__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_w,\n uint8_t** ptr_io0_w,\n uint8_t** ptr_io1_w,\n uint8_t** ptr_io2_w,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = 0;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_w = data.ptr;\n *ptr_io0_w = data.ptr;\n *ptr_io1_w = data.ptr;\n *ptr_io2_w = data.ptr + data.len;\n\n return b;\n}\n\n" +
" length,\n wuffs_base__slice_u8 src) {\n uint8_t* iop_w = *ptr_iop_w;\n size_t n = src.len;\n if (n > length) {\n n = length;\n }\n if (n > ((size_t)(io2_w - iop_w))) {\n n = (size_t)(io2_w - iop_w);\n }\n if (n > 0) {\n memmove(iop_w, src.ptr, n);\n *ptr_iop_w += n;\n }\n return (uint32_t)(n);\n}\n\n// wuffs_base__io_reader__match7 returns whether the io_reader's upcoming bytes\n// start with the given prefix (up to 7 bytes long). It is peek-like, not\n// read-like, in that there are no side-effects.\n//\n// The low 3 bits of a hold the prefix length, n.\n//\n// The high 56 bits of a hold the prefix itself, in little-endian order. The\n// first prefix byte is in bits 8..=15, the second prefix byte is in bits\n// 16..=23, etc. The high (8 * (7 - n)) bits are ignored.\n//\n// There are three possible return values:\n// - 0 means success.\n// - 1 means inconclusive, equivalent to \"$short read\".\n// - 2 means failure.\nstatic inline uint32_t //\nwuffs_base__io_reader__match7(ui" +
"nt8_t* iop_r, uint8_t* io2_r, uint64_t a) {\n uint32_t n = a & 7;\n a >>= 8;\n if ((io2_r - iop_r) >= 8) {\n uint64_t x = wuffs_base__load_u64le(iop_r);\n uint32_t shift = 8 * (8 - n);\n return ((a << shift) == (x << shift)) ? 0 : 2;\n }\n for (; n > 0; n--) {\n if (iop_r >= io2_r) {\n return 1;\n } else if (*iop_r != ((uint8_t)(a))) {\n return 2;\n }\n iop_r++;\n a >>= 8;\n }\n return 0;\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_reader__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_r,\n uint8_t** ptr_io0_r,\n uint8_t** ptr_io1_r,\n uint8_t** ptr_io2_r,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = data.len;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_r = data.ptr;\n *ptr_io0_r = data.ptr;\n *ptr_io1_r = data.ptr;\n *ptr_io2_r = data.ptr + data.len;\n\n return b;\n}\n\nstatic inline wuffs_base__sli" +
"ce_u8 //\nwuffs_base__io_reader__take(uint8_t** ptr_iop_r, uint8_t* io2_r, uint64_t n) {\n if (n <= ((size_t)(io2_r - *ptr_iop_r))) {\n uint8_t* p = *ptr_iop_r;\n *ptr_iop_r += n;\n return wuffs_base__make_slice_u8(p, n);\n }\n return wuffs_base__make_slice_u8(NULL, 0);\n}\n\nstatic inline wuffs_base__io_buffer* //\nwuffs_base__io_writer__set(wuffs_base__io_buffer* b,\n uint8_t** ptr_iop_w,\n uint8_t** ptr_io0_w,\n uint8_t** ptr_io1_w,\n uint8_t** ptr_io2_w,\n wuffs_base__slice_u8 data) {\n b->data = data;\n b->meta.wi = 0;\n b->meta.ri = 0;\n b->meta.pos = 0;\n b->meta.closed = false;\n\n *ptr_iop_w = data.ptr;\n *ptr_io0_w = data.ptr;\n *ptr_io1_w = data.ptr;\n *ptr_io2_w = data.ptr + data.len;\n\n return b;\n}\n\n" +
"" +
"// ---------------- I/O (Utility)\n\n#define wuffs_base__utility__empty_io_reader wuffs_base__empty_io_reader\n#define wuffs_base__utility__empty_io_writer wuffs_base__empty_io_writer\n" +
""
+1
View File
@@ -252,6 +252,7 @@ var Funcs = []string{
"io_reader.count_since(mark: u64) u64",
"io_reader.is_closed() bool",
"io_reader.mark() u64",
"io_reader.match7(a: u64) u32[..= 2]",
"io_reader.position() u64",
"io_reader.since(mark: u64) slice u8",
"io_reader.take!(n: u64) slice u8",
+12 -6
View File
@@ -493,12 +493,15 @@ const (
IDHistoryAvailable = ID(0x162)
IDIsClosed = ID(0x163)
IDMark = ID(0x164)
IDPosition = ID(0x165)
IDSince = ID(0x166)
IDSkip = ID(0x167)
IDSkip32 = ID(0x168)
IDSkip32Fast = ID(0x169)
IDTake = ID(0x16A)
IDMatch15 = ID(0x165)
IDMatch31 = ID(0x166)
IDMatch7 = ID(0x167)
IDPosition = ID(0x168)
IDSince = ID(0x169)
IDSkip = ID(0x16A)
IDSkip32 = ID(0x16B)
IDSkip32Fast = ID(0x16C)
IDTake = ID(0x16D)
IDCopyFromSlice = ID(0x170)
IDCopyN32FromHistory = ID(0x171)
@@ -844,6 +847,9 @@ var builtInsByID = [nBuiltInIDs]string{
IDHistoryAvailable: "history_available",
IDIsClosed: "is_closed",
IDMark: "mark",
IDMatch15: "match15",
IDMatch31: "match31",
IDMatch7: "match7",
IDPosition: "position",
IDSince: "since",
IDSkip: "skip",
+92 -2
View File
@@ -6332,6 +6332,41 @@ wuffs_base__io_writer__copy_n32_from_slice(uint8_t** ptr_iop_w,
return (uint32_t)(n);
}
// wuffs_base__io_reader__match7 returns whether the io_reader's upcoming bytes
// start with the given prefix (up to 7 bytes long). It is peek-like, not
// read-like, in that there are no side-effects.
//
// The low 3 bits of a hold the prefix length, n.
//
// The high 56 bits of a hold the prefix itself, in little-endian order. The
// first prefix byte is in bits 8..=15, the second prefix byte is in bits
// 16..=23, etc. The high (8 * (7 - n)) bits are ignored.
//
// There are three possible return values:
// - 0 means success.
// - 1 means inconclusive, equivalent to "$short read".
// - 2 means failure.
static inline uint32_t //
wuffs_base__io_reader__match7(uint8_t* iop_r, uint8_t* io2_r, uint64_t a) {
uint32_t n = a & 7;
a >>= 8;
if ((io2_r - iop_r) >= 8) {
uint64_t x = wuffs_base__load_u64le(iop_r);
uint32_t shift = 8 * (8 - n);
return ((a << shift) == (x << shift)) ? 0 : 2;
}
for (; n > 0; n--) {
if (iop_r >= io2_r) {
return 1;
} else if (*iop_r != ((uint8_t)(a))) {
return 2;
}
iop_r++;
a >>= 8;
}
return 0;
}
static inline wuffs_base__io_buffer* //
wuffs_base__io_reader__set(wuffs_base__io_buffer* b,
uint8_t** ptr_iop_r,
@@ -18508,6 +18543,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
uint32_t v_depth = 0;
uint32_t v_stack_byte = 0;
uint32_t v_stack_bit = 0;
uint32_t v_match = 0;
uint8_t v_c = 0;
uint8_t v_expect = 0;
uint8_t v_expect_after_value = 0;
@@ -18894,8 +18930,62 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
goto label_0_continue;
} else {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
if (0 == (v_expect & 1)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
v_match = 2;
if (v_c == 110) {
v_match =
wuffs_base__io_reader__match7(iop_a_src, io2_a_src, 465676103172);
if (v_match == 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
status = wuffs_base__make_status(
wuffs_json__error__internal_error_inconsistent_i_o);
goto exit;
}
(iop_a_src += 4, wuffs_base__make_empty_struct());
}
} else if (v_c == 102) {
v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src,
111546413966853);
if (v_match == 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194369)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(5)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
if (((uint64_t)(io2_a_src - iop_a_src)) < 5) {
status = wuffs_base__make_status(
wuffs_json__error__internal_error_inconsistent_i_o);
goto exit;
}
(iop_a_src += 5, wuffs_base__make_empty_struct());
}
} else if (v_c == 116) {
v_match =
wuffs_base__io_reader__match7(iop_a_src, io2_a_src, 435762131972);
if (v_match == 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194433)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
status = wuffs_base__make_status(
wuffs_json__error__internal_error_inconsistent_i_o);
goto exit;
}
(iop_a_src += 4, wuffs_base__make_empty_struct());
}
}
if (v_match == 1) {
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8);
goto label_0_continue;
} else if (v_match == 2) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
}
if (v_depth == 0) {
goto label_0_break;
+43 -2
View File
@@ -61,6 +61,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var depth : base.u32[..= 1024]
var stack_byte : base.u32[..= (1024 / 32) - 1]
var stack_bit : base.u32[..= 31]
var match : base.u32[..= 2]
var c : base.u8
// expect is a bitmask of what the next token can be:
@@ -391,8 +392,48 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.outer
} else {
// TODO: literals (false, true, null).
return "#bad input"
if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE.
return "#bad input"
}
match = 2
if c == 0x6E { // 0x6E is 'n'.
match = args.src.match7(a: 0x6C_6C75_6E04) // 4 bytes "null".
if match == 0 {
args.dst.write_fast_token!(value: 0x40_0021, length: 4)
if args.src.available() < 4 {
return "#internal error: inconsistent I/O"
}
args.src.skip32_fast!(actual: 4, worst_case: 4)
}
} else if c == 0x66 { // 0x66 is 'f'.
match = args.src.match7(a: 0x6573_6C61_6605) // 5 bytes "false".
if match == 0 {
args.dst.write_fast_token!(value: 0x40_0041, length: 5)
if args.src.available() < 5 {
return "#internal error: inconsistent I/O"
}
args.src.skip32_fast!(actual: 5, worst_case: 5)
}
} else if c == 0x74 { // 0x74 is 't'.
match = args.src.match7(a: 0x65_7572_7404) // 4 bytes "true".
if match == 0 {
args.dst.write_fast_token!(value: 0x40_0081, length: 4)
if args.src.available() < 4 {
return "#internal error: inconsistent I/O"
}
args.src.skip32_fast!(actual: 4, worst_case: 4)
}
}
if match == 1 {
yield? base."$short read"
continue.outer
} else if match == 2 {
return "#bad input"
}
}
// We've just parsed a value.