Have std/json decode '[' and '{'

This commit is contained in:
Nigel Tao committed 2020-02-13 13:01:27 +11:00
1 parent e9aba8924a
commit ea91e5ab69
3 files changed
+325 -24

No files matched your search

+154 -10
View File
@@ -5080,6 +5080,9 @@ extern "C" {
// ---------------- Status Codes
extern const char* wuffs_json__error__bad_input;
extern const char* wuffs_json__error__unsupported_recursion_depth;
// ---------------- Public Consts
// ---------------- Struct Declarations
@@ -5137,6 +5140,15 @@ struct wuffs_json__decoder__struct {
uint32_t p_decode_tokens[1];
} private_impl;
struct {
uint32_t f_stack[32];
struct {
uint32_t v_depth;
uint8_t v_expect;
} s_decode_tokens[1];
} private_data;
#ifdef __cplusplus
#if (__cplusplus >= 201103L) && !defined(WUFFS_IMPLEMENTATION)
// Disallow constructing or copying an object via standard C++ mechanisms,
@@ -18118,6 +18130,10 @@ exit:
// ---------------- Status Codes Implementations
const char* wuffs_json__error__bad_input = "#json: bad input";
const char* wuffs_json__error__unsupported_recursion_depth =
"#json: unsupported recursion depth";
// ---------------- Private Consts
// ---------------- Private Initializer Prototypes
@@ -18207,7 +18223,12 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
self->private_impl.active_coroutine = 0;
wuffs_base__status status = wuffs_base__make_status(NULL);
uint64_t v_length = 0;
uint32_t v_whitespace_length = 0;
uint32_t v_depth = 0;
uint32_t v_stack_byte = 0;
uint32_t v_stack_bit = 0;
uint8_t v_c = 0;
uint8_t v_expect = 0;
wuffs_base__token* iop_a_dst = NULL;
wuffs_base__token* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
@@ -18222,23 +18243,141 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
io2_a_dst = iop_a_dst;
}
}
uint8_t* iop_a_src = NULL;
uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
if (a_src) {
io0_a_src = a_src->data.ptr;
io1_a_src = io0_a_src + a_src->meta.ri;
iop_a_src = io1_a_src;
io2_a_src = io0_a_src + a_src->meta.wi;
}
uint32_t coro_susp_point = self->private_impl.p_decode_tokens[0];
if (coro_susp_point) {
v_depth = self->private_data.s_decode_tokens[0].v_depth;
v_expect = self->private_data.s_decode_tokens[0].v_expect;
}
switch (coro_susp_point) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1);
v_expect = 3;
label_0_continue:;
while (true) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1);
goto label_0_continue;
}
v_whitespace_length = 0;
v_c = 0;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (v_whitespace_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_whitespace_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_whitespace_length = 0;
}
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2);
v_whitespace_length = 0;
goto label_0_continue;
}
v_c = wuffs_base__load_u8be(iop_a_src);
if (v_c > 32) {
goto label_1_break;
} else if ((v_c != 32) && (v_c != 10) && (v_c != 9) && (v_c != 13)) {
if (v_whitespace_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_whitespace_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
}
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_whitespace_length >= 65534) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_whitespace_length = 0;
goto label_0_continue;
}
v_whitespace_length += 1;
}
label_1_break:;
if (v_whitespace_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_whitespace_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_whitespace_length = 0;
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
goto label_0_continue;
}
}
if (v_c <= 57) {
if (v_c == 34) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
} else {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
} else if (v_c == 91) {
if (0 == (v_expect & 1)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
if (v_depth >= 1024) {
status = wuffs_base__make_status(
wuffs_json__error__unsupported_recursion_depth);
goto exit;
}
v_stack_byte = (v_depth / 32);
v_stack_bit = (v_depth & 31);
self->private_data.f_stack[v_stack_byte] &=
(0 ^ (((uint32_t)(1)) << v_stack_bit));
v_depth += 1;
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097168)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_expect = 19;
} else if (v_c == 123) {
if (0 == (v_expect & 1)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
if (v_depth >= 1024) {
status = wuffs_base__make_status(
wuffs_json__error__unsupported_recursion_depth);
goto exit;
}
v_stack_byte = (v_depth / 32);
v_stack_bit = (v_depth & 31);
self->private_data.f_stack[v_stack_byte] |=
(((uint32_t)(1)) << v_stack_bit);
v_depth += 1;
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097184)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_expect = 34;
} else {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
if (v_depth == 0) {
goto label_0_break;
}
}
v_length = 456;
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(123)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
status = wuffs_base__make_status(NULL);
goto ok;
label_0_break:;
goto ok;
ok:
self->private_impl.p_decode_tokens[0] = 0;
@@ -18251,12 +18390,17 @@ suspend:
wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0;
self->private_impl.active_coroutine =
wuffs_base__status__is_suspension(&status) ? 1 : 0;
self->private_data.s_decode_tokens[0].v_depth = v_depth;
self->private_data.s_decode_tokens[0].v_expect = v_expect;
goto exit;
exit:
if (a_dst) {
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
}
if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
if (wuffs_base__status__is_error(&status)) {
self->private_impl.magic = WUFFS_BASE__DISABLED;
+165 -9
View File
@@ -14,19 +14,175 @@
// TODO: "decoder implements base.token_decoder".
pub status "#bad input"
pub status "#unsupported recursion depth"
pub struct decoder?(
)(
// stack is conceptually an array of bits, implemented as an array of u32.
// The N'th bit being 0 or 1 means that we're in an array or object, where
// N is the recursion depth.
//
// Parsing JSON involves recursion: containers (arrays and objects) can
// hold other containers. As child elements are completed, the parser needs
// to remember 1 bit of state per recursion depth: whether the parent
// container was an array or an object. When continuing to parse the
// parent's elements, `, "key": value` is only valid for objects.
//
// Note that we explicitly track our own stack and depth. We do not use the
// call stack to hold this state and the decoder.decode_tokens function is
// not recursive per se.
//
// Wuffs code does not have the capability to dynamically allocate memory,
// so the maximum depth is hard-coded at compile time. In this case, the
// maximum is 1024 (stack is 1024 bits or 128 bytes).
//
// The [JSON spec](https://tools.ietf.org/html/rfc7159) clearly states, "an
// implementation may set limits on the maximum depth of nesting".
//
// In comparison, as of February 2020, the Chromium web browser's JSON
// parser's maximum recursion depth is 200:
// https://source.chromium.org/chromium/chromium/src/+/3dece34cde622faa0daac07156c25d92c9897d1e:base/json/json_common.h;l=18
//
// Other languages and libraries' maximum depths (determined empirically)
// are listed at https://github.com/lovasoa/bad_json_parsers#results
stack : array[1024 / 32] base.u32,
)
pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var length : base.u64[..= 0xFFFF]
var whitespace_length : base.u32[..= 0xFFFE]
var depth : base.u32[..= 1024]
var stack_byte : base.u32[..= (1024 / 32) - 1]
var stack_bit : base.u32[..= 31]
var c : base.u8
while args.dst.available() <= 0,
post args.dst.available() > 0,
{
yield? base."$short write"
// expect is a bitmask of what the next token can be:
// - 0x01 EXPECT_NON_STRING_VALUE.
// - 0x02 EXPECT_STRING_VALUE.
// - 0x04 EXPECT_COMMA ','.
// - 0x08 EXPECT_COLON ':'.
// - 0x10 EXPECT_CLOSE_BRACKET ']'.
// - 0x20 EXPECT_CLOSE_BRACE '}'.
//
// EXPECT_VALUE is also defined to be 0x03, equivalent to
// (EXPECT_NON_STRING_VALUE | EXPECT_STRING_VALUE).
//
// "Non-string value" includes numbers, literals (false, true, null),
// arrays ("[etc]") and objects ("{etc}").
//
// "String value" includes "this" and "th\u0061t".
var expect : base.u8
expect = 0x03 // EXPECT_VALUE
while.outer true {
if args.dst.available() <= 0 {
yield? base."$short write"
continue.outer
}
// Consume whitespace.
whitespace_length = 0
c = 0
while.ws true,
inv args.dst.available() > 0,
post args.src.available() > 0,
{
if args.src.available() <= 0 {
if whitespace_length > 0 {
args.dst.write_fast_token!(value: 0, length: whitespace_length as base.u64)
whitespace_length = 0
}
yield? base."$short read"
whitespace_length = 0
continue.outer
}
c = args.src.peek_u8()
if c > 0x20 { // 0x20 is ' '.
break.ws
} else if (c <> 0x20) and (c <> 0x0A) and (c <> 0x09) and (c <> 0x0D) {
if whitespace_length > 0 {
args.dst.write_fast_token!(value: 0, length: whitespace_length as base.u64)
}
return "#bad input"
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
if whitespace_length >= 0xFFFE {
args.dst.write_fast_token!(value: 0, length: 0xFFFF)
whitespace_length = 0
continue.outer
}
whitespace_length += 1
}
// Emit whitespace.
if whitespace_length > 0 {
args.dst.write_fast_token!(value: 0, length: whitespace_length as base.u64)
whitespace_length = 0
if args.dst.available() <= 0 {
continue.outer
}
}
// These assertions are redundant (the Wuffs compiler should already
// know these facts; deleting these assertions should still compile)
// but are listed explicitly to help guard against future edits to the
// code above inadvertently invalidating these assertions.
assert args.dst.available() > 0
assert args.src.available() > 0
if c <= 0x39 { // 0x39 is '9'.
if c == 0x22 { // 0x22 is '"'.
// TODO: strings.
return "#bad input"
} else {
// TODO: numbers.
return "#bad input"
}
} else if c == 0x5B { // 0x5B is '['.
if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE.
return "#bad input"
}
if depth >= 1024 {
return "#unsupported recursion depth"
}
stack_byte = depth / 32
stack_bit = depth & 31
this.stack[stack_byte] &= 0 ^ ((1 as base.u32) << stack_bit)
depth += 1
args.src.skip32_fast!(actual: 1, worst_case: 1)
args.dst.write_fast_token!(value: 0x20_0010, length: 1)
expect = 0x13 // 0x13 is (EXPECT_CLOSE_BRACKET | EXPECT_VALUE).
} else if c == 0x7B { // 0x7B is '{'.
if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE.
return "#bad input"
}
if depth >= 1024 {
return "#unsupported recursion depth"
}
stack_byte = depth / 32
stack_bit = depth & 31
this.stack[stack_byte] |= (1 as base.u32) << stack_bit
depth += 1
args.src.skip32_fast!(actual: 1, worst_case: 1)
args.dst.write_fast_token!(value: 0x20_0020, length: 1)
expect = 0x22 // 0x22 is (EXPECT_CLOSE_BRACE | EXPECT_STRING_VALUE).
} else {
// TODO: literals (false, true, null).
return "#bad input"
}
if depth == 0 {
break.outer
}
}
length = 456
args.dst.write_fast_token!(value: 123, length: length)
return ok
}
+6 -5
View File
@@ -84,9 +84,9 @@ const char* test_wuffs_json_decode_tokens() {
.data = global_src_slice,
});
CHECK_STRING(read_file(&src, "test/data/github-tags.json"));
CHECK_STATUS("decode_tokens",
wuffs_json__decoder__decode_tokens(&dec, &got, &src));
wuffs_base__status status =
wuffs_json__decoder__decode_tokens(&dec, &got, &src);
if (0) {
uint64_t pos = 0;
size_t i;
@@ -95,11 +95,12 @@ const char* test_wuffs_json_decode_tokens() {
uint64_t len = wuffs_base__token__length(t);
uint64_t bc = wuffs_base__token__value_base_category(t);
uint64_t bd = wuffs_base__token__value_base_detail(t);
printf("i=%3zu\tp=%3" PRIu64 "\tl=%3" PRIu64 "\tbc=%3" PRIu64
"\tbd=0x%06" PRIX64 "\n",
i, pos, len, bc, bd);
printf("i=%3zu\tp=%4" PRIu64 " (0x%03" PRIX64 ")\tl=%3" PRIu64
"\tbc=%3" PRIu64 "\tbd=0x%06" PRIX64 "\n",
i, pos, pos, len, bc, bd);
pos = wuffs_base__u64__sat_add(pos, len);
}
CHECK_STATUS("decode_tokens", status);
}
return NULL;