Add std/json quirk_allow_comment_etc

This commit is contained in:
Nigel Tao committed 2020-03-22 21:07:12 +11:00
1 parent 21f6a5df4a
commit eb06ed77ba
4 files changed
+463 -6

No files matched your search

+235 -1
View File
@@ -6022,6 +6022,7 @@ struct wuffs_json__decoder__struct {
uint32_t p_decode_tokens[1];
uint32_t p_decode_leading[1];
uint32_t p_decode_comment[1];
uint32_t p_decode_trailing_new_line[1];
} private_impl;
@@ -6033,6 +6034,9 @@ struct wuffs_json__decoder__struct {
uint32_t v_expect;
uint32_t v_expect_after_value;
} s_decode_tokens[1];
struct {
uint32_t v_link_prev;
} s_decode_comment[1];
} private_data;
#ifdef __cplusplus
@@ -20366,6 +20370,11 @@ wuffs_json__decoder__decode_leading(wuffs_json__decoder* self,
wuffs_base__token_buffer* a_dst,
wuffs_base__io_buffer* a_src);
static wuffs_base__status //
wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
wuffs_base__token_buffer* a_dst,
wuffs_base__io_buffer* a_src);
static wuffs_base__status //
wuffs_json__decoder__decode_trailing_new_line(wuffs_json__decoder* self,
wuffs_base__token_buffer* a_dst,
@@ -21549,6 +21558,28 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(20);
goto label__outer__continue;
}
} else if (v_class == 12) {
if (self->private_impl.f_quirk_enabled_allow_comment_block ||
self->private_impl.f_quirk_enabled_allow_comment_line) {
if (a_dst) {
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
}
if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(21);
status = wuffs_json__decoder__decode_comment(self, a_dst, a_src);
if (a_dst) {
iop_a_dst = a_dst->data.ptr + a_dst->meta.wi;
}
if (a_src) {
iop_a_src = a_src->data.ptr + a_src->meta.ri;
}
if (status.repr) {
goto suspend;
}
goto label__outer__continue;
}
}
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -21567,7 +21598,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(21);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(22);
status =
wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src);
if (a_dst) {
@@ -21913,6 +21944,209 @@ exit:
return status;
}
// -------- func json.decoder.decode_comment
static wuffs_base__status //
wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
wuffs_base__token_buffer* a_dst,
wuffs_base__io_buffer* a_src) {
wuffs_base__status status = wuffs_base__make_status(NULL);
uint8_t v_c8 = 0;
uint16_t v_c16 = 0;
uint32_t v_link_prev = 0;
uint32_t v_length = 0;
wuffs_base__token* iop_a_dst = NULL;
wuffs_base__token* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
wuffs_base__token* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
wuffs_base__token* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
if (a_dst) {
io0_a_dst = a_dst->data.ptr;
io1_a_dst = io0_a_dst + a_dst->meta.wi;
iop_a_dst = io1_a_dst;
io2_a_dst = io0_a_dst + a_dst->data.len;
if (a_dst->meta.closed) {
io2_a_dst = iop_a_dst;
}
}
uint8_t* iop_a_src = NULL;
uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
if (a_src) {
io0_a_src = a_src->data.ptr;
io1_a_src = io0_a_src + a_src->meta.ri;
iop_a_src = io1_a_src;
io2_a_src = io0_a_src + a_src->meta.wi;
}
uint32_t coro_susp_point = self->private_impl.p_decode_comment[0];
if (coro_susp_point) {
v_link_prev = self->private_data.s_decode_comment[0].v_link_prev;
}
switch (coro_susp_point) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
label__0__continue:;
while ((((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) ||
(((uint64_t)(io2_a_src - iop_a_src)) <= 1)) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1);
goto label__0__continue;
}
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2);
}
v_c16 = wuffs_base__load_u16le__no_bounds_check(iop_a_src);
if ((v_c16 == 10799) &&
self->private_impl.f_quirk_enabled_allow_comment_block) {
(iop_a_src += 2, wuffs_base__make_empty_struct());
v_length = 2;
label__comment_block__continue:;
while (true) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3);
v_length = 0;
goto label__comment_block__continue;
}
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 1) {
if (v_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)((1 | v_link_prev)))
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_link_prev = 2;
}
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4);
v_length = 0;
goto label__comment_block__continue;
}
v_c16 = wuffs_base__load_u16le__no_bounds_check(iop_a_src);
if (v_c16 == 12074) {
(iop_a_src += 2, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)((v_length + 2)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
status = wuffs_base__make_status(NULL);
goto ok;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_length >= 65533) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)((1 | v_link_prev)))
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)((v_length + 1)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_length = 0;
v_link_prev = 2;
goto label__comment_block__continue;
}
v_length += 1;
}
}
} else if ((v_c16 == 12079) &&
self->private_impl.f_quirk_enabled_allow_comment_line) {
(iop_a_src += 2, wuffs_base__make_empty_struct());
v_length = 2;
label__comment_line__continue:;
while (true) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5);
v_length = 0;
goto label__comment_line__continue;
}
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (v_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)((1 | v_link_prev)))
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_link_prev = 2;
}
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6);
v_length = 0;
goto label__comment_line__continue;
}
v_c8 = wuffs_base__load_u8be__no_bounds_check(iop_a_src);
if (v_c8 == 10) {
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)((v_length + 1)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
status = wuffs_base__make_status(NULL);
goto ok;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_length >= 65533) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)((1 | v_link_prev)))
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)((v_length + 1)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_length = 0;
v_link_prev = 2;
goto label__comment_line__continue;
}
v_length += 1;
}
}
}
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
goto ok;
ok:
self->private_impl.p_decode_comment[0] = 0;
goto exit;
}
goto suspend;
suspend:
self->private_impl.p_decode_comment[0] =
wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0;
self->private_data.s_decode_comment[0].v_link_prev = v_link_prev;
goto exit;
exit:
if (a_dst) {
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
}
if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
return status;
}
// -------- func json.decoder.decode_trailing_new_line
static wuffs_base__status //
+145
View File
@@ -1070,6 +1070,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
yield? base."$short read"
continue.outer
}
} else if class == 0x0C { // 0x0C is CLASS_COMMENT.
if this.quirk_enabled_allow_comment_block or this.quirk_enabled_allow_comment_line {
this.decode_comment?(dst: args.dst, src: args.src)
continue.outer
}
}
return "#bad input"
@@ -1310,6 +1316,145 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) {
}
}
pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
var c8 : base.u8
var c16 : base.u16
var link_prev : base.u32[..= 0x2]
var length : base.u32[..= 0xFFFD]
while (args.dst.available() <= 0) or (args.src.available() <= 1),
post args.dst.available() > 0,
post args.src.available() > 1,
{
if args.dst.available() <= 0 {
yield? base."$short write"
continue
}
if args.src.is_closed() {
return "#bad input"
}
yield? base."$short read"
}
c16 = args.src.peek_u16le()
if (c16 == 0x2A2F) and this.quirk_enabled_allow_comment_block {
args.src.skip32_fast!(actual: 2, worst_case: 2)
length = 2
while.comment_block true {
if args.dst.available() <= 0 {
yield? base."$short write"
length = 0
continue.comment_block
}
while true,
pre args.dst.available() > 0,
{
if args.src.available() <= 1 {
if length > 0 {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x2,
link: 0x1 | link_prev,
length: length)
link_prev = 0x2
}
if args.src.is_closed() {
return "#bad input"
}
yield? base."$short read"
length = 0
continue.comment_block
}
c16 = args.src.peek_u16le()
if c16 == 0x2F2A { // 0x2F2A is "*/" little-endian.
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x2,
link: link_prev,
length: length + 2)
return ok
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
if length >= 0xFFFD {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x2,
link: 0x1 | link_prev,
length: length + 1)
length = 0
link_prev = 0x2
continue.comment_block
}
length += 1
}
} endwhile.comment_block
} else if (c16 == 0x2F2F) and this.quirk_enabled_allow_comment_line {
args.src.skip32_fast!(actual: 2, worst_case: 2)
length = 2
while.comment_line true {
if args.dst.available() <= 0 {
yield? base."$short write"
length = 0
continue.comment_line
}
while true,
pre args.dst.available() > 0,
{
if args.src.available() <= 0 {
if length > 0 {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x1,
link: 0x1 | link_prev,
length: length)
link_prev = 0x2
}
if args.src.is_closed() {
return "#bad input"
}
yield? base."$short read"
length = 0
continue.comment_line
}
c8 = args.src.peek_u8()
if c8 == 0x0A { // 0x0A is '\n'.
args.src.skip32_fast!(actual: 1, worst_case: 1)
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x1,
link: link_prev,
length: length + 1)
return ok
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
if length >= 0xFFFD {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x1,
link: 0x1 | link_prev,
length: length + 1)
length = 0
link_prev = 0x2
continue.comment_line
}
length += 1
}
} endwhile.comment_line
}
return "#bad input"
}
pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_reader) {
var c : base.u8
var whitespace_length : base.u32[..= 0xFFFE]
+10 -5
View File
@@ -68,18 +68,23 @@ pub const quirk_allow_backslash_zero base.u32 = 0x4909_9400 | 0x07
// anywhere whitespace would be, although see the quirk_allow_trailing_new_line
// comment for additional interaction when combining multiple quirks.
//
// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK tokens.
// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK tokens. The token
// chain's source bytes includes the starting "/*" and the ending "*/".
pub const quirk_allow_comment_block base.u32 = 0x4909_9400 | 0x08
// When this quirk is enabled, "// C/C++ style line comments\n" are accepted
// anywhere whitespace would be, although see the quirk_allow_trailing_new_line
// comment for additional interaction when combining multiple quirks.
//
// A line comment may omit the trailing '\n' if there is no input afterwards
// (i.e. the line comment ends with the end-of-file), but note the
// quirk_allow_trailing_new_line interaction already mentioned.
// A line comment may not omit the ending "\n", even if there is no input
// afterwards (i.e. the prospective line comment ends with the end-of-file).
//
// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE tokens.
// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE tokens. The token
// chain's source bytes includes the starting "//" and the ending "\n".
//
// Even if the line comments are on consecutive lines, each line comment is a
// separate token chain. There may be whitespace tokens between one line
// comment's ending "\n" and the next one's starting "//".
pub const quirk_allow_comment_line base.u32 = 0x4909_9400 | 0x09
// When this quirk is enabled, there may be a comma after the final array
+73
View File
@@ -1329,6 +1329,78 @@ test_wuffs_json_decode_quirk_allow_backslash_x() {
return NULL;
}
const char* //
test_wuffs_json_decode_quirk_allow_comment_etc() {
CHECK_FOCUS(__func__);
struct {
// want has 4 bytes, one for each possible q:
// - q&1 sets WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK.
// - q&2 sets WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE.
// An 'X', '+' or '-' means that decoding should succeed (and consume the
// entire input), succeed (without consuming the entire input) or fail.
const char* want;
const char* str;
} test_cases[] = {
{.want = "-X-X", .str = "[ /*com*/ 0]"},
{.want = "--XX", .str = "//l\n //m\n0"},
{.want = "---X", .str = "[ 0, /*com*/ 1 //l\n\n]"},
{.want = "----", .str = "/*/0"}, // Not a valid slash-star comment.
{.want = "----", .str = "[4/5]"}, // Lone slash.
};
int tc;
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
int q;
for (q = 0; q < 4; q++) {
wuffs_json__decoder dec;
CHECK_STATUS("initialize", wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
wuffs_json__decoder__set_quirk_enabled(
&dec, WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK, q & 1);
wuffs_json__decoder__set_quirk_enabled(
&dec, WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE, q & 2);
wuffs_base__token_buffer tok =
wuffs_base__make_token_buffer_writer(global_have_token_slice);
wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader(
wuffs_base__make_slice_u8((void*)(test_cases[tc].str),
strlen(test_cases[tc].str)),
true);
const char* have =
wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr;
const char* want =
(test_cases[tc].want[q] != '-') ? NULL : wuffs_json__error__bad_input;
if (have != want) {
RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc,
q, have, want);
}
size_t total_length = 0;
while (tok.meta.ri < tok.meta.wi) {
total_length += wuffs_base__token__length(&tok.data.ptr[tok.meta.ri++]);
}
if (total_length != src.meta.ri) {
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q,
total_length, src.meta.ri);
}
if (test_cases[tc].want[q] == 'X') {
if (total_length != src.data.len) {
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q,
total_length, src.data.len);
}
} else if (test_cases[tc].want[q] == '+') {
if (total_length >= src.data.len) {
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want < %zu", tc, q,
total_length, src.data.len);
}
}
}
}
return NULL;
}
const char* //
test_wuffs_json_decode_quirk_allow_leading_etc() {
CHECK_FOCUS(__func__);
@@ -1973,6 +2045,7 @@ proc tests[] = {
test_wuffs_json_decode_prior_valid_utf_8,
test_wuffs_json_decode_quirk_allow_backslash_etc,
test_wuffs_json_decode_quirk_allow_backslash_x,
test_wuffs_json_decode_quirk_allow_comment_etc,
test_wuffs_json_decode_quirk_allow_leading_etc,
test_wuffs_json_decode_quirk_allow_trailing_etc,
test_wuffs_json_decode_quirk_replace_invalid_utf_8,