diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 9aadd1ad..fe922b05 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -6022,6 +6022,7 @@ struct wuffs_json__decoder__struct { uint32_t p_decode_tokens[1]; uint32_t p_decode_leading[1]; + uint32_t p_decode_comment[1]; uint32_t p_decode_trailing_new_line[1]; } private_impl; @@ -6033,6 +6034,9 @@ struct wuffs_json__decoder__struct { uint32_t v_expect; uint32_t v_expect_after_value; } s_decode_tokens[1]; + struct { + uint32_t v_link_prev; + } s_decode_comment[1]; } private_data; #ifdef __cplusplus @@ -20366,6 +20370,11 @@ wuffs_json__decoder__decode_leading(wuffs_json__decoder* self, wuffs_base__token_buffer* a_dst, wuffs_base__io_buffer* a_src); +static wuffs_base__status // +wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, + wuffs_base__token_buffer* a_dst, + wuffs_base__io_buffer* a_src); + static wuffs_base__status // wuffs_json__decoder__decode_trailing_new_line(wuffs_json__decoder* self, wuffs_base__token_buffer* a_dst, @@ -21549,6 +21558,28 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(20); goto label__outer__continue; } + } else if (v_class == 12) { + if (self->private_impl.f_quirk_enabled_allow_comment_block || + self->private_impl.f_quirk_enabled_allow_comment_line) { + if (a_dst) { + a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(21); + status = wuffs_json__decoder__decode_comment(self, a_dst, a_src); + if (a_dst) { + iop_a_dst = a_dst->data.ptr + a_dst->meta.wi; + } + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + goto label__outer__continue; + } } status = wuffs_base__make_status(wuffs_json__error__bad_input); goto exit; @@ -21567,7 +21598,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(21); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(22); status = wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src); if (a_dst) { @@ -21913,6 +21944,209 @@ exit: return status; } +// -------- func json.decoder.decode_comment + +static wuffs_base__status // +wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, + wuffs_base__token_buffer* a_dst, + wuffs_base__io_buffer* a_src) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + uint8_t v_c8 = 0; + uint16_t v_c16 = 0; + uint32_t v_link_prev = 0; + uint32_t v_length = 0; + + wuffs_base__token* iop_a_dst = NULL; + wuffs_base__token* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + wuffs_base__token* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + wuffs_base__token* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_dst) { + io0_a_dst = a_dst->data.ptr; + io1_a_dst = io0_a_dst + a_dst->meta.wi; + iop_a_dst = io1_a_dst; + io2_a_dst = io0_a_dst + a_dst->data.len; + if (a_dst->meta.closed) { + io2_a_dst = iop_a_dst; + } + } + uint8_t* iop_a_src = NULL; + uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + uint32_t coro_susp_point = self->private_impl.p_decode_comment[0]; + if (coro_susp_point) { + v_link_prev = self->private_data.s_decode_comment[0].v_link_prev; + } + switch (coro_susp_point) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; + + label__0__continue:; + while ((((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) || + (((uint64_t)(io2_a_src - iop_a_src)) <= 1)) { + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + status = wuffs_base__make_status(wuffs_base__suspension__short_write); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1); + goto label__0__continue; + } + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2); + } + v_c16 = wuffs_base__load_u16le__no_bounds_check(iop_a_src); + if ((v_c16 == 10799) && + self->private_impl.f_quirk_enabled_allow_comment_block) { + (iop_a_src += 2, wuffs_base__make_empty_struct()); + v_length = 2; + label__comment_block__continue:; + while (true) { + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + status = wuffs_base__make_status(wuffs_base__suspension__short_write); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3); + v_length = 0; + goto label__comment_block__continue; + } + while (true) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 1) { + if (v_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((1 | v_link_prev))) + << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_link_prev = 2; + } + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + status = + wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4); + v_length = 0; + goto label__comment_block__continue; + } + v_c16 = wuffs_base__load_u16le__no_bounds_check(iop_a_src); + if (v_c16 == 12074) { + (iop_a_src += 2, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)((v_length + 2))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + status = wuffs_base__make_status(NULL); + goto ok; + } + (iop_a_src += 1, wuffs_base__make_empty_struct()); + if (v_length >= 65533) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((1 | v_link_prev))) + << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)((v_length + 1))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_length = 0; + v_link_prev = 2; + goto label__comment_block__continue; + } + v_length += 1; + } + } + } else if ((v_c16 == 12079) && + self->private_impl.f_quirk_enabled_allow_comment_line) { + (iop_a_src += 2, wuffs_base__make_empty_struct()); + v_length = 2; + label__comment_line__continue:; + while (true) { + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + status = wuffs_base__make_status(wuffs_base__suspension__short_write); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5); + v_length = 0; + goto label__comment_line__continue; + } + while (true) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { + if (v_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((1 | v_link_prev))) + << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_link_prev = 2; + } + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + status = + wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6); + v_length = 0; + goto label__comment_line__continue; + } + v_c8 = wuffs_base__load_u8be__no_bounds_check(iop_a_src); + if (v_c8 == 10) { + (iop_a_src += 1, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)((v_length + 1))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + status = wuffs_base__make_status(NULL); + goto ok; + } + (iop_a_src += 1, wuffs_base__make_empty_struct()); + if (v_length >= 65533) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((1 | v_link_prev))) + << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)((v_length + 1))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_length = 0; + v_link_prev = 2; + goto label__comment_line__continue; + } + v_length += 1; + } + } + } + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + goto ok; + ok: + self->private_impl.p_decode_comment[0] = 0; + goto exit; + } + + goto suspend; +suspend: + self->private_impl.p_decode_comment[0] = + wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + self->private_data.s_decode_comment[0].v_link_prev = v_link_prev; + + goto exit; +exit: + if (a_dst) { + a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + + return status; +} + // -------- func json.decoder.decode_trailing_new_line static wuffs_base__status // diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index e297898c..176fd095 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -1070,6 +1070,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { yield? base."$short read" continue.outer } + + } else if class == 0x0C { // 0x0C is CLASS_COMMENT. + if this.quirk_enabled_allow_comment_block or this.quirk_enabled_allow_comment_line { + this.decode_comment?(dst: args.dst, src: args.src) + continue.outer + } } return "#bad input" @@ -1310,6 +1316,145 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) { } } +pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { + var c8 : base.u8 + var c16 : base.u16 + var link_prev : base.u32[..= 0x2] + var length : base.u32[..= 0xFFFD] + + while (args.dst.available() <= 0) or (args.src.available() <= 1), + post args.dst.available() > 0, + post args.src.available() > 1, + { + if args.dst.available() <= 0 { + yield? base."$short write" + continue + } + if args.src.is_closed() { + return "#bad input" + } + yield? base."$short read" + } + c16 = args.src.peek_u16le() + + if (c16 == 0x2A2F) and this.quirk_enabled_allow_comment_block { + args.src.skip32_fast!(actual: 2, worst_case: 2) + length = 2 + + while.comment_block true { + if args.dst.available() <= 0 { + yield? base."$short write" + length = 0 + continue.comment_block + } + + while true, + pre args.dst.available() > 0, + { + if args.src.available() <= 1 { + if length > 0 { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x2, + link: 0x1 | link_prev, + length: length) + link_prev = 0x2 + } + if args.src.is_closed() { + return "#bad input" + } + yield? base."$short read" + length = 0 + continue.comment_block + } + + c16 = args.src.peek_u16le() + if c16 == 0x2F2A { // 0x2F2A is "*/" little-endian. + args.src.skip32_fast!(actual: 2, worst_case: 2) + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x2, + link: link_prev, + length: length + 2) + return ok + } + + args.src.skip32_fast!(actual: 1, worst_case: 1) + if length >= 0xFFFD { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x2, + link: 0x1 | link_prev, + length: length + 1) + length = 0 + link_prev = 0x2 + continue.comment_block + } + length += 1 + } + } endwhile.comment_block + + } else if (c16 == 0x2F2F) and this.quirk_enabled_allow_comment_line { + args.src.skip32_fast!(actual: 2, worst_case: 2) + length = 2 + + while.comment_line true { + if args.dst.available() <= 0 { + yield? base."$short write" + length = 0 + continue.comment_line + } + + while true, + pre args.dst.available() > 0, + { + if args.src.available() <= 0 { + if length > 0 { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x1, + link: 0x1 | link_prev, + length: length) + link_prev = 0x2 + } + if args.src.is_closed() { + return "#bad input" + } + yield? base."$short read" + length = 0 + continue.comment_line + } + + c8 = args.src.peek_u8() + if c8 == 0x0A { // 0x0A is '\n'. + args.src.skip32_fast!(actual: 1, worst_case: 1) + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x1, + link: link_prev, + length: length + 1) + return ok + } + + args.src.skip32_fast!(actual: 1, worst_case: 1) + if length >= 0xFFFD { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x1, + link: 0x1 | link_prev, + length: length + 1) + length = 0 + link_prev = 0x2 + continue.comment_line + } + length += 1 + } + } endwhile.comment_line + } + + return "#bad input" +} + pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_reader) { var c : base.u8 var whitespace_length : base.u32[..= 0xFFFE] diff --git a/std/json/decode_quirks.wuffs b/std/json/decode_quirks.wuffs index 4daf3104..c2bda1a0 100644 --- a/std/json/decode_quirks.wuffs +++ b/std/json/decode_quirks.wuffs @@ -68,18 +68,23 @@ pub const quirk_allow_backslash_zero base.u32 = 0x4909_9400 | 0x07 // anywhere whitespace would be, although see the quirk_allow_trailing_new_line // comment for additional interaction when combining multiple quirks. // -// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK tokens. +// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK tokens. The token +// chain's source bytes includes the starting "/*" and the ending "*/". pub const quirk_allow_comment_block base.u32 = 0x4909_9400 | 0x08 // When this quirk is enabled, "// C/C++ style line comments\n" are accepted // anywhere whitespace would be, although see the quirk_allow_trailing_new_line // comment for additional interaction when combining multiple quirks. // -// A line comment may omit the trailing '\n' if there is no input afterwards -// (i.e. the line comment ends with the end-of-file), but note the -// quirk_allow_trailing_new_line interaction already mentioned. +// A line comment may not omit the ending "\n", even if there is no input +// afterwards (i.e. the prospective line comment ends with the end-of-file). // -// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE tokens. +// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE tokens. The token +// chain's source bytes includes the starting "//" and the ending "\n". +// +// Even if the line comments are on consecutive lines, each line comment is a +// separate token chain. There may be whitespace tokens between one line +// comment's ending "\n" and the next one's starting "//". pub const quirk_allow_comment_line base.u32 = 0x4909_9400 | 0x09 // When this quirk is enabled, there may be a comma after the final array diff --git a/test/c/std/json.c b/test/c/std/json.c index 2d9cb1e6..e345344b 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -1329,6 +1329,78 @@ test_wuffs_json_decode_quirk_allow_backslash_x() { return NULL; } +const char* // +test_wuffs_json_decode_quirk_allow_comment_etc() { + CHECK_FOCUS(__func__); + + struct { + // want has 4 bytes, one for each possible q: + // - q&1 sets WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK. + // - q&2 sets WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE. + // An 'X', '+' or '-' means that decoding should succeed (and consume the + // entire input), succeed (without consuming the entire input) or fail. + const char* want; + const char* str; + } test_cases[] = { + {.want = "-X-X", .str = "[ /*com*/ 0]"}, + {.want = "--XX", .str = "//l\n //m\n0"}, + {.want = "---X", .str = "[ 0, /*com*/ 1 //l\n\n]"}, + {.want = "----", .str = "/*/0"}, // Not a valid slash-star comment. + {.want = "----", .str = "[4/5]"}, // Lone slash. + }; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + int q; + for (q = 0; q < 4; q++) { + wuffs_json__decoder dec; + CHECK_STATUS("initialize", wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + wuffs_json__decoder__set_quirk_enabled( + &dec, WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK, q & 1); + wuffs_json__decoder__set_quirk_enabled( + &dec, WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE, q & 2); + + wuffs_base__token_buffer tok = + wuffs_base__make_token_buffer_writer(global_have_token_slice); + wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader( + wuffs_base__make_slice_u8((void*)(test_cases[tc].str), + strlen(test_cases[tc].str)), + true); + const char* have = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr; + const char* want = + (test_cases[tc].want[q] != '-') ? NULL : wuffs_json__error__bad_input; + if (have != want) { + RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc, + q, have, want); + } + + size_t total_length = 0; + while (tok.meta.ri < tok.meta.wi) { + total_length += wuffs_base__token__length(&tok.data.ptr[tok.meta.ri++]); + } + if (total_length != src.meta.ri) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q, + total_length, src.meta.ri); + } + if (test_cases[tc].want[q] == 'X') { + if (total_length != src.data.len) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q, + total_length, src.data.len); + } + } else if (test_cases[tc].want[q] == '+') { + if (total_length >= src.data.len) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want < %zu", tc, q, + total_length, src.data.len); + } + } + } + } + return NULL; +} + const char* // test_wuffs_json_decode_quirk_allow_leading_etc() { CHECK_FOCUS(__func__); @@ -1973,6 +2045,7 @@ proc tests[] = { test_wuffs_json_decode_prior_valid_utf_8, test_wuffs_json_decode_quirk_allow_backslash_etc, test_wuffs_json_decode_quirk_allow_backslash_x, + test_wuffs_json_decode_quirk_allow_comment_etc, test_wuffs_json_decode_quirk_allow_leading_etc, test_wuffs_json_decode_quirk_allow_trailing_etc, test_wuffs_json_decode_quirk_replace_invalid_utf_8,