diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index ea1a3e59..fb86d266 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -5080,6 +5080,9 @@ extern "C" { // ---------------- Status Codes +extern const char* wuffs_json__error__bad_input; +extern const char* wuffs_json__error__unsupported_recursion_depth; + // ---------------- Public Consts // ---------------- Struct Declarations @@ -5137,6 +5140,15 @@ struct wuffs_json__decoder__struct { uint32_t p_decode_tokens[1]; } private_impl; + struct { + uint32_t f_stack[32]; + + struct { + uint32_t v_depth; + uint8_t v_expect; + } s_decode_tokens[1]; + } private_data; + #ifdef __cplusplus #if (__cplusplus >= 201103L) && !defined(WUFFS_IMPLEMENTATION) // Disallow constructing or copying an object via standard C++ mechanisms, @@ -18118,6 +18130,10 @@ exit: // ---------------- Status Codes Implementations +const char* wuffs_json__error__bad_input = "#json: bad input"; +const char* wuffs_json__error__unsupported_recursion_depth = + "#json: unsupported recursion depth"; + // ---------------- Private Consts // ---------------- Private Initializer Prototypes @@ -18207,7 +18223,12 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, self->private_impl.active_coroutine = 0; wuffs_base__status status = wuffs_base__make_status(NULL); - uint64_t v_length = 0; + uint32_t v_whitespace_length = 0; + uint32_t v_depth = 0; + uint32_t v_stack_byte = 0; + uint32_t v_stack_bit = 0; + uint8_t v_c = 0; + uint8_t v_expect = 0; wuffs_base__token* iop_a_dst = NULL; wuffs_base__token* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; @@ -18222,23 +18243,141 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, io2_a_dst = iop_a_dst; } } + uint8_t* iop_a_src = NULL; + uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } uint32_t coro_susp_point = self->private_impl.p_decode_tokens[0]; if (coro_susp_point) { + v_depth = self->private_data.s_decode_tokens[0].v_depth; + v_expect = self->private_data.s_decode_tokens[0].v_expect; } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { - status = wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1); + v_expect = 3; + label_0_continue:; + while (true) { + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + status = wuffs_base__make_status(wuffs_base__suspension__short_write); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1); + goto label_0_continue; + } + v_whitespace_length = 0; + v_c = 0; + while (true) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { + if (v_whitespace_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)(v_whitespace_length)))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_whitespace_length = 0; + } + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2); + v_whitespace_length = 0; + goto label_0_continue; + } + v_c = wuffs_base__load_u8be(iop_a_src); + if (v_c > 32) { + goto label_1_break; + } else if ((v_c != 32) && (v_c != 10) && (v_c != 9) && (v_c != 13)) { + if (v_whitespace_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)(v_whitespace_length)))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + } + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + (iop_a_src += 1, wuffs_base__make_empty_struct()); + if (v_whitespace_length >= 65534) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_whitespace_length = 0; + goto label_0_continue; + } + v_whitespace_length += 1; + } + label_1_break:; + if (v_whitespace_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)(v_whitespace_length)))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_whitespace_length = 0; + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + goto label_0_continue; + } + } + if (v_c <= 57) { + if (v_c == 34) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } else { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + } else if (v_c == 91) { + if (0 == (v_expect & 1)) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + if (v_depth >= 1024) { + status = wuffs_base__make_status( + wuffs_json__error__unsupported_recursion_depth); + goto exit; + } + v_stack_byte = (v_depth / 32); + v_stack_bit = (v_depth & 31); + self->private_data.f_stack[v_stack_byte] &= + (0 ^ (((uint32_t)(1)) << v_stack_bit)); + v_depth += 1; + (iop_a_src += 1, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2097168)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_expect = 19; + } else if (v_c == 123) { + if (0 == (v_expect & 1)) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + if (v_depth >= 1024) { + status = wuffs_base__make_status( + wuffs_json__error__unsupported_recursion_depth); + goto exit; + } + v_stack_byte = (v_depth / 32); + v_stack_bit = (v_depth & 31); + self->private_data.f_stack[v_stack_byte] |= + (((uint32_t)(1)) << v_stack_bit); + v_depth += 1; + (iop_a_src += 1, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2097184)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_expect = 34; + } else { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + if (v_depth == 0) { + goto label_0_break; + } } - v_length = 456; - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(123)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | - (((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - status = wuffs_base__make_status(NULL); - goto ok; + label_0_break:; + goto ok; ok: self->private_impl.p_decode_tokens[0] = 0; @@ -18251,12 +18390,17 @@ suspend: wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; self->private_impl.active_coroutine = wuffs_base__status__is_suspension(&status) ? 1 : 0; + self->private_data.s_decode_tokens[0].v_depth = v_depth; + self->private_data.s_decode_tokens[0].v_expect = v_expect; goto exit; exit: if (a_dst) { a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } if (wuffs_base__status__is_error(&status)) { self->private_impl.magic = WUFFS_BASE__DISABLED; diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 0d432c2b..38ebaeb1 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -14,19 +14,175 @@ // TODO: "decoder implements base.token_decoder". +pub status "#bad input" +pub status "#unsupported recursion depth" + pub struct decoder?( +)( + // stack is conceptually an array of bits, implemented as an array of u32. + // The N'th bit being 0 or 1 means that we're in an array or object, where + // N is the recursion depth. + // + // Parsing JSON involves recursion: containers (arrays and objects) can + // hold other containers. As child elements are completed, the parser needs + // to remember 1 bit of state per recursion depth: whether the parent + // container was an array or an object. When continuing to parse the + // parent's elements, `, "key": value` is only valid for objects. + // + // Note that we explicitly track our own stack and depth. We do not use the + // call stack to hold this state and the decoder.decode_tokens function is + // not recursive per se. + // + // Wuffs code does not have the capability to dynamically allocate memory, + // so the maximum depth is hard-coded at compile time. In this case, the + // maximum is 1024 (stack is 1024 bits or 128 bytes). + // + // The [JSON spec](https://tools.ietf.org/html/rfc7159) clearly states, "an + // implementation may set limits on the maximum depth of nesting". + // + // In comparison, as of February 2020, the Chromium web browser's JSON + // parser's maximum recursion depth is 200: + // https://source.chromium.org/chromium/chromium/src/+/3dece34cde622faa0daac07156c25d92c9897d1e:base/json/json_common.h;l=18 + // + // Other languages and libraries' maximum depths (determined empirically) + // are listed at https://github.com/lovasoa/bad_json_parsers#results + stack : array[1024 / 32] base.u32, ) pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { - var length : base.u64[..= 0xFFFF] + var whitespace_length : base.u32[..= 0xFFFE] + var depth : base.u32[..= 1024] + var stack_byte : base.u32[..= (1024 / 32) - 1] + var stack_bit : base.u32[..= 31] + var c : base.u8 - while args.dst.available() <= 0, - post args.dst.available() > 0, - { - yield? base."$short write" + // expect is a bitmask of what the next token can be: + // - 0x01 EXPECT_NON_STRING_VALUE. + // - 0x02 EXPECT_STRING_VALUE. + // - 0x04 EXPECT_COMMA ','. + // - 0x08 EXPECT_COLON ':'. + // - 0x10 EXPECT_CLOSE_BRACKET ']'. + // - 0x20 EXPECT_CLOSE_BRACE '}'. + // + // EXPECT_VALUE is also defined to be 0x03, equivalent to + // (EXPECT_NON_STRING_VALUE | EXPECT_STRING_VALUE). + // + // "Non-string value" includes numbers, literals (false, true, null), + // arrays ("[etc]") and objects ("{etc}"). + // + // "String value" includes "this" and "th\u0061t". + var expect : base.u8 + + expect = 0x03 // EXPECT_VALUE + + while.outer true { + if args.dst.available() <= 0 { + yield? base."$short write" + continue.outer + } + + // Consume whitespace. + whitespace_length = 0 + c = 0 + while.ws true, + inv args.dst.available() > 0, + post args.src.available() > 0, + { + if args.src.available() <= 0 { + if whitespace_length > 0 { + args.dst.write_fast_token!(value: 0, length: whitespace_length as base.u64) + whitespace_length = 0 + } + yield? base."$short read" + whitespace_length = 0 + continue.outer + } + + c = args.src.peek_u8() + if c > 0x20 { // 0x20 is ' '. + break.ws + } else if (c <> 0x20) and (c <> 0x0A) and (c <> 0x09) and (c <> 0x0D) { + if whitespace_length > 0 { + args.dst.write_fast_token!(value: 0, length: whitespace_length as base.u64) + } + return "#bad input" + } + args.src.skip32_fast!(actual: 1, worst_case: 1) + + if whitespace_length >= 0xFFFE { + args.dst.write_fast_token!(value: 0, length: 0xFFFF) + whitespace_length = 0 + continue.outer + } + whitespace_length += 1 + } + + // Emit whitespace. + if whitespace_length > 0 { + args.dst.write_fast_token!(value: 0, length: whitespace_length as base.u64) + whitespace_length = 0 + if args.dst.available() <= 0 { + continue.outer + } + } + + // These assertions are redundant (the Wuffs compiler should already + // know these facts; deleting these assertions should still compile) + // but are listed explicitly to help guard against future edits to the + // code above inadvertently invalidating these assertions. + assert args.dst.available() > 0 + assert args.src.available() > 0 + + if c <= 0x39 { // 0x39 is '9'. + if c == 0x22 { // 0x22 is '"'. + // TODO: strings. + + return "#bad input" + + } else { + // TODO: numbers. + return "#bad input" + } + + } else if c == 0x5B { // 0x5B is '['. + if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE. + return "#bad input" + } + if depth >= 1024 { + return "#unsupported recursion depth" + } + stack_byte = depth / 32 + stack_bit = depth & 31 + this.stack[stack_byte] &= 0 ^ ((1 as base.u32) << stack_bit) + depth += 1 + + args.src.skip32_fast!(actual: 1, worst_case: 1) + args.dst.write_fast_token!(value: 0x20_0010, length: 1) + expect = 0x13 // 0x13 is (EXPECT_CLOSE_BRACKET | EXPECT_VALUE). + + } else if c == 0x7B { // 0x7B is '{'. + if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE. + return "#bad input" + } + if depth >= 1024 { + return "#unsupported recursion depth" + } + stack_byte = depth / 32 + stack_bit = depth & 31 + this.stack[stack_byte] |= (1 as base.u32) << stack_bit + depth += 1 + + args.src.skip32_fast!(actual: 1, worst_case: 1) + args.dst.write_fast_token!(value: 0x20_0020, length: 1) + expect = 0x22 // 0x22 is (EXPECT_CLOSE_BRACE | EXPECT_STRING_VALUE). + + } else { + // TODO: literals (false, true, null). + return "#bad input" + } + + if depth == 0 { + break.outer + } } - - length = 456 - args.dst.write_fast_token!(value: 123, length: length) - return ok } diff --git a/test/c/std/json.c b/test/c/std/json.c index 79d0918c..5bea9af1 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -84,9 +84,9 @@ const char* test_wuffs_json_decode_tokens() { .data = global_src_slice, }); CHECK_STRING(read_file(&src, "test/data/github-tags.json")); - CHECK_STATUS("decode_tokens", - wuffs_json__decoder__decode_tokens(&dec, &got, &src)); + wuffs_base__status status = + wuffs_json__decoder__decode_tokens(&dec, &got, &src); if (0) { uint64_t pos = 0; size_t i; @@ -95,11 +95,12 @@ const char* test_wuffs_json_decode_tokens() { uint64_t len = wuffs_base__token__length(t); uint64_t bc = wuffs_base__token__value_base_category(t); uint64_t bd = wuffs_base__token__value_base_detail(t); - printf("i=%3zu\tp=%3" PRIu64 "\tl=%3" PRIu64 "\tbc=%3" PRIu64 - "\tbd=0x%06" PRIX64 "\n", - i, pos, len, bc, bd); + printf("i=%3zu\tp=%4" PRIu64 " (0x%03" PRIX64 ")\tl=%3" PRIu64 + "\tbc=%3" PRIu64 "\tbd=0x%06" PRIX64 "\n", + i, pos, pos, len, bc, bd); pos = wuffs_base__u64__sat_add(pos, len); } + CHECK_STATUS("decode_tokens", status); } return NULL;