mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 08:38:14 +01:00
Add std/json quirk_allow_trailing_etc
This commit is contained in:
1 parent
dd8b315a6d
commit
4b186b0b6b
4 files changed
+327
-50
No files matched your search
+13
-29
@@ -670,8 +670,16 @@ initialize_globals(int argc, char** argv) {
|
||||
suppress_write_dst = query.next_fragment() ? 1 : 0;
|
||||
wrote_to_dst = false;
|
||||
|
||||
return dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0)
|
||||
.message();
|
||||
TRY(dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0)
|
||||
.message());
|
||||
|
||||
// Consume an optional whitespace trailer. This isn't part of the JSON spec,
|
||||
// but it works better with line oriented Unix tools (such as "echo 123 |
|
||||
// jsonptr" where it's "echo", not "echo -n") or hand-edited JSON files which
|
||||
// can accidentally contain trailing whitespace.
|
||||
dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE, true);
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// ----
|
||||
@@ -1051,30 +1059,6 @@ end_of_data:
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Consume an optional whitespace trailer. This isn't part of the JSON spec,
|
||||
// but it works better with line oriented Unix tools (such as "echo 123 |
|
||||
// jsonptr" where it's "echo", not "echo -n") or hand-edited JSON files which
|
||||
// can accidentally contain trailing whitespace.
|
||||
//
|
||||
// A whitespace trailer is zero or more ' ' and then zero or one '\n'.
|
||||
while (true) {
|
||||
if (src.meta.ri < src.meta.wi) {
|
||||
uint8_t c = src.data.ptr[src.meta.ri];
|
||||
if (c == ' ') {
|
||||
src.meta.ri++;
|
||||
continue;
|
||||
} else if (c == '\n') {
|
||||
src.meta.ri++;
|
||||
break;
|
||||
}
|
||||
// The "exhausted the input" check below will fail.
|
||||
break;
|
||||
} else if (src.meta.closed) {
|
||||
break;
|
||||
}
|
||||
TRY(read_src());
|
||||
}
|
||||
|
||||
// Check that we've exhausted the input.
|
||||
if ((src.meta.ri == src.meta.wi) && !src.meta.closed) {
|
||||
TRY(read_src());
|
||||
@@ -1084,9 +1068,9 @@ end_of_data:
|
||||
}
|
||||
|
||||
// Check that we've used all of the decoded tokens, other than trailing
|
||||
// filler tokens. For example, a bare `"foo"` string is valid JSON, but even
|
||||
// without a trailing '\n', the Wuffs JSON parser emits a filler token for
|
||||
// the final '\"'.
|
||||
// filler tokens. For example, "true\n" is valid JSON (and fully consumed
|
||||
// with WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE enabled) with a trailing
|
||||
// filler token for the "\n".
|
||||
for (; tok.meta.ri < tok.meta.wi; tok.meta.ri++) {
|
||||
if (tok.data.ptr[tok.meta.ri].value_base_category() !=
|
||||
WUFFS_BASE__TOKEN__VBC__FILLER) {
|
||||
|
||||
@@ -5999,6 +5999,7 @@ struct wuffs_json__decoder__struct {
|
||||
|
||||
uint32_t p_decode_tokens[1];
|
||||
uint32_t p_decode_leading[1];
|
||||
uint32_t p_decode_trailing_new_line[1];
|
||||
} private_impl;
|
||||
|
||||
struct {
|
||||
@@ -20296,6 +20297,11 @@ wuffs_json__decoder__decode_leading(wuffs_json__decoder* self,
|
||||
wuffs_base__token_buffer* a_dst,
|
||||
wuffs_base__io_buffer* a_src);
|
||||
|
||||
static wuffs_base__status //
|
||||
wuffs_json__decoder__decode_trailing_new_line(wuffs_json__decoder* self,
|
||||
wuffs_base__token_buffer* a_dst,
|
||||
wuffs_base__io_buffer* a_src);
|
||||
|
||||
// ---------------- VTables
|
||||
|
||||
const wuffs_base__token_decoder__func_ptrs
|
||||
@@ -21278,6 +21284,26 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
v_expect = v_expect_after_value;
|
||||
}
|
||||
label__outer__break:;
|
||||
if (self->private_impl.f_quirk_enabled_allow_trailing_new_line) {
|
||||
if (a_dst) {
|
||||
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
|
||||
}
|
||||
if (a_src) {
|
||||
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
|
||||
}
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(19);
|
||||
status =
|
||||
wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src);
|
||||
if (a_dst) {
|
||||
iop_a_dst = a_dst->data.ptr + a_dst->meta.wi;
|
||||
}
|
||||
if (a_src) {
|
||||
iop_a_src = a_src->data.ptr + a_src->meta.ri;
|
||||
}
|
||||
if (status.repr) {
|
||||
goto suspend;
|
||||
}
|
||||
}
|
||||
self->private_impl.f_end_of_data = true;
|
||||
|
||||
goto ok;
|
||||
@@ -21611,6 +21637,124 @@ exit:
|
||||
return status;
|
||||
}
|
||||
|
||||
// -------- func json.decoder.decode_trailing_new_line
|
||||
|
||||
static wuffs_base__status //
|
||||
wuffs_json__decoder__decode_trailing_new_line(wuffs_json__decoder* self,
|
||||
wuffs_base__token_buffer* a_dst,
|
||||
wuffs_base__io_buffer* a_src) {
|
||||
wuffs_base__status status = wuffs_base__make_status(NULL);
|
||||
|
||||
uint8_t v_c = 0;
|
||||
uint32_t v_whitespace_length = 0;
|
||||
|
||||
wuffs_base__token* iop_a_dst = NULL;
|
||||
wuffs_base__token* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
wuffs_base__token* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
wuffs_base__token* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
if (a_dst) {
|
||||
io0_a_dst = a_dst->data.ptr;
|
||||
io1_a_dst = io0_a_dst + a_dst->meta.wi;
|
||||
iop_a_dst = io1_a_dst;
|
||||
io2_a_dst = io0_a_dst + a_dst->data.len;
|
||||
if (a_dst->meta.closed) {
|
||||
io2_a_dst = iop_a_dst;
|
||||
}
|
||||
}
|
||||
uint8_t* iop_a_src = NULL;
|
||||
uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
if (a_src) {
|
||||
io0_a_src = a_src->data.ptr;
|
||||
io1_a_src = io0_a_src + a_src->meta.ri;
|
||||
iop_a_src = io1_a_src;
|
||||
io2_a_src = io0_a_src + a_src->meta.wi;
|
||||
}
|
||||
|
||||
uint32_t coro_susp_point = self->private_impl.p_decode_trailing_new_line[0];
|
||||
if (coro_susp_point) {
|
||||
}
|
||||
switch (coro_susp_point) {
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
|
||||
|
||||
label__outer__continue:;
|
||||
while (true) {
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1);
|
||||
v_whitespace_length = 0;
|
||||
goto label__outer__continue;
|
||||
}
|
||||
while (true) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
|
||||
if (v_whitespace_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_whitespace_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_whitespace_length = 0;
|
||||
}
|
||||
if (a_src && a_src->meta.closed) {
|
||||
goto label__outer__break;
|
||||
}
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2);
|
||||
v_whitespace_length = 0;
|
||||
goto label__outer__continue;
|
||||
}
|
||||
v_c = wuffs_base__load_u8be__no_bounds_check(iop_a_src);
|
||||
if (wuffs_json__lut_classes[v_c] != 0) {
|
||||
if (v_whitespace_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_whitespace_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_whitespace_length = 0;
|
||||
}
|
||||
status = wuffs_base__make_status(wuffs_json__error__bad_input);
|
||||
goto exit;
|
||||
}
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
if ((v_whitespace_length >= 65534) || (v_c == 10)) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)((v_whitespace_length + 1)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_whitespace_length = 0;
|
||||
if (v_c == 10) {
|
||||
goto label__outer__break;
|
||||
}
|
||||
goto label__outer__continue;
|
||||
}
|
||||
v_whitespace_length += 1;
|
||||
}
|
||||
}
|
||||
label__outer__break:;
|
||||
|
||||
goto ok;
|
||||
ok:
|
||||
self->private_impl.p_decode_trailing_new_line[0] = 0;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
goto suspend;
|
||||
suspend:
|
||||
self->private_impl.p_decode_trailing_new_line[0] =
|
||||
wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0;
|
||||
|
||||
goto exit;
|
||||
exit:
|
||||
if (a_dst) {
|
||||
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
|
||||
}
|
||||
if (a_src) {
|
||||
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
#endif // !defined(WUFFS_CONFIG__MODULES) ||
|
||||
// defined(WUFFS_CONFIG__MODULE__JSON)
|
||||
|
||||
|
||||
@@ -902,6 +902,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
expect = expect_after_value
|
||||
} endwhile.outer
|
||||
|
||||
if this.quirk_enabled_allow_trailing_new_line {
|
||||
this.decode_trailing_new_line?(dst: args.dst, src: args.src)
|
||||
}
|
||||
|
||||
this.end_of_data = true
|
||||
}
|
||||
|
||||
@@ -1124,3 +1128,57 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_reader) {
|
||||
var c : base.u8
|
||||
var whitespace_length : base.u32[..= 0xFFFE]
|
||||
|
||||
while.outer true {
|
||||
if args.dst.available() <= 0 {
|
||||
yield? base."$short write"
|
||||
whitespace_length = 0
|
||||
continue.outer
|
||||
}
|
||||
|
||||
while.inner true,
|
||||
pre args.dst.available() > 0,
|
||||
{
|
||||
if args.src.available() <= 0 {
|
||||
if whitespace_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
}
|
||||
if args.src.is_closed() {
|
||||
break.outer
|
||||
}
|
||||
yield? base."$short read"
|
||||
whitespace_length = 0
|
||||
continue.outer
|
||||
}
|
||||
|
||||
c = args.src.peek_u8()
|
||||
if lut_classes[c] <> 0x00 { // 0x00 is CLASS_WHITESPACE.
|
||||
if whitespace_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
}
|
||||
return "#bad input"
|
||||
}
|
||||
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if (whitespace_length >= 0xFFFE) or (c == 0x0A) {
|
||||
args.dst.write_fast_token!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length + 1)
|
||||
whitespace_length = 0
|
||||
if c == 0x0A {
|
||||
break.outer
|
||||
}
|
||||
continue.outer
|
||||
}
|
||||
whitespace_length += 1
|
||||
} endwhile.inner
|
||||
} endwhile.outer
|
||||
|
||||
}
|
||||
+112
-21
@@ -1113,21 +1113,22 @@ test_wuffs_json_decode_quirk_allow_leading_etc() {
|
||||
// want has 4 bytes, one for each possible q:
|
||||
// - q&1 sets WUFFS_JSON__QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR.
|
||||
// - q&2 sets WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK.
|
||||
// A '+' or '-' means that decoding should succeed or fail.
|
||||
// An 'X', '+' or '-' means that decoding should succeed (and consume the
|
||||
// entire input), succeed (without consuming the entire input) or fail.
|
||||
const char* want;
|
||||
const char* str;
|
||||
} test_cases[] = {
|
||||
{.want = "-+-+", .str = "\x1Etrue"},
|
||||
{.want = "--++", .str = "\xEF\xBB\xBFtrue"},
|
||||
{.want = "---+", .str = "\x1E\xEF\xBB\xBFtrue"},
|
||||
{.want = "---+", .str = "\xEF\xBB\xBF\x1Etrue"},
|
||||
{.want = "----", .str = " \x1Etrue"},
|
||||
{.want = "----", .str = "\x1E \xEF\xBB\xBFtrue"},
|
||||
{.want = "----", .str = "\x1E\x1Etrue"},
|
||||
{.want = "----", .str = "\xEF\xBB"},
|
||||
{.want = "----", .str = "\xEF\xBB\xBF"},
|
||||
{.want = "----", .str = "\xEF\xBB\xBF$"},
|
||||
{.want = "----", .str = "\xEFtrue"},
|
||||
{.want = "-X-X", .str = "\x1Etrue"}, //
|
||||
{.want = "--XX", .str = "\xEF\xBB\xBFtrue"}, //
|
||||
{.want = "---X", .str = "\x1E\xEF\xBB\xBFtrue"}, //
|
||||
{.want = "---X", .str = "\xEF\xBB\xBF\x1Etrue"}, //
|
||||
{.want = "----", .str = " \x1Etrue"}, //
|
||||
{.want = "----", .str = "\x1E \xEF\xBB\xBFtrue"}, //
|
||||
{.want = "----", .str = "\x1E\x1Etrue"}, //
|
||||
{.want = "----", .str = "\xEF\xBB"}, //
|
||||
{.want = "----", .str = "\xEF\xBB\xBF"}, //
|
||||
{.want = "----", .str = "\xEF\xBB\xBF$"}, //
|
||||
{.want = "----", .str = "\xEFtrue"}, //
|
||||
};
|
||||
|
||||
int tc;
|
||||
@@ -1152,7 +1153,7 @@ test_wuffs_json_decode_quirk_allow_leading_etc() {
|
||||
const char* have =
|
||||
wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr;
|
||||
const char* want =
|
||||
(test_cases[tc].want[q] == '+') ? NULL : wuffs_json__error__bad_input;
|
||||
(test_cases[tc].want[q] != '-') ? NULL : wuffs_json__error__bad_input;
|
||||
if (have != want) {
|
||||
RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc,
|
||||
q, have, want);
|
||||
@@ -1166,6 +1167,95 @@ test_wuffs_json_decode_quirk_allow_leading_etc() {
|
||||
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q,
|
||||
total_length, src.meta.ri);
|
||||
}
|
||||
if (test_cases[tc].want[q] == 'X') {
|
||||
if (total_length != src.data.len) {
|
||||
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q,
|
||||
total_length, src.data.len);
|
||||
}
|
||||
} else if (test_cases[tc].want[q] == '+') {
|
||||
if (total_length >= src.data.len) {
|
||||
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want < %zu", tc, q,
|
||||
total_length, src.data.len);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_json_decode_quirk_allow_trailing_etc() {
|
||||
CHECK_FOCUS(__func__);
|
||||
|
||||
struct {
|
||||
// want has 2 bytes, one for each possible q:
|
||||
// - q&1 sets WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE.
|
||||
// An 'X', '+' or '-' means that decoding should succeed (and consume the
|
||||
// entire input), succeed (without consuming the entire input) or fail.
|
||||
const char* want;
|
||||
const char* str;
|
||||
} test_cases[] = {
|
||||
{.want = "++", .str = "0 \n "}, //
|
||||
{.want = "++", .str = "0 \n\n"}, //
|
||||
{.want = "++", .str = "0\n\n"}, //
|
||||
{.want = "+-", .str = "0 true \n"}, //
|
||||
{.want = "+-", .str = "007"}, //
|
||||
{.want = "+-", .str = "007\n"}, //
|
||||
{.want = "+-", .str = "0true "}, //
|
||||
{.want = "+-", .str = "0true"}, //
|
||||
{.want = "+X", .str = "0 "}, //
|
||||
{.want = "+X", .str = "0 \n"}, //
|
||||
{.want = "+X", .str = "0\n"}, //
|
||||
{.want = "+X", .str = "0\t\r\n"}, //
|
||||
{.want = "--", .str = "\n"}, //
|
||||
{.want = "XX", .str = "0"}, //
|
||||
};
|
||||
|
||||
int tc;
|
||||
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
|
||||
int q;
|
||||
for (q = 0; q < 2; q++) {
|
||||
wuffs_json__decoder dec;
|
||||
CHECK_STATUS("initialize", wuffs_json__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
|
||||
wuffs_json__decoder__set_quirk_enabled(
|
||||
&dec, WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE, q & 1);
|
||||
|
||||
wuffs_base__token_buffer tok =
|
||||
wuffs_base__make_token_buffer_writer(global_have_token_slice);
|
||||
wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader(
|
||||
wuffs_base__make_slice_u8((void*)(test_cases[tc].str),
|
||||
strlen(test_cases[tc].str)),
|
||||
true);
|
||||
const char* have =
|
||||
wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr;
|
||||
const char* want =
|
||||
(test_cases[tc].want[q] != '-') ? NULL : wuffs_json__error__bad_input;
|
||||
if (have != want) {
|
||||
RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc,
|
||||
q, have, want);
|
||||
}
|
||||
|
||||
size_t total_length = 0;
|
||||
while (tok.meta.ri < tok.meta.wi) {
|
||||
total_length += wuffs_base__token__length(&tok.data.ptr[tok.meta.ri++]);
|
||||
}
|
||||
if (total_length != src.meta.ri) {
|
||||
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q,
|
||||
total_length, src.meta.ri);
|
||||
}
|
||||
if (test_cases[tc].want[q] == 'X') {
|
||||
if (total_length != src.data.len) {
|
||||
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q,
|
||||
total_length, src.data.len);
|
||||
}
|
||||
} else if (test_cases[tc].want[q] == '+') {
|
||||
if (total_length >= src.data.len) {
|
||||
RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want < %zu", tc, q,
|
||||
total_length, src.data.len);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
@@ -1519,14 +1609,15 @@ proc tests[] = {
|
||||
test_strconv_parse_number_u64, //
|
||||
test_strconv_utf_8_next, //
|
||||
|
||||
test_wuffs_json_decode_end_of_data, //
|
||||
test_wuffs_json_decode_interface, //
|
||||
test_wuffs_json_decode_long_numbers, //
|
||||
test_wuffs_json_decode_prior_valid_utf_8, //
|
||||
test_wuffs_json_decode_quirk_allow_leading_etc, //
|
||||
test_wuffs_json_decode_src_io_buffer_length, //
|
||||
test_wuffs_json_decode_string, //
|
||||
test_wuffs_json_decode_unicode4_escapes, //
|
||||
test_wuffs_json_decode_end_of_data, //
|
||||
test_wuffs_json_decode_interface, //
|
||||
test_wuffs_json_decode_long_numbers, //
|
||||
test_wuffs_json_decode_prior_valid_utf_8, //
|
||||
test_wuffs_json_decode_quirk_allow_leading_etc, //
|
||||
test_wuffs_json_decode_quirk_allow_trailing_etc, //
|
||||
test_wuffs_json_decode_src_io_buffer_length, //
|
||||
test_wuffs_json_decode_string, //
|
||||
test_wuffs_json_decode_unicode4_escapes, //
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
|
||||
Reference in new issue
Block a user