Add std/json quirk_replace_invalid_utf_8

This commit is contained in:
Nigel Tao committed 2020-03-19 10:37:18 +11:00
1 parent c5c10e95b7
commit 31cfaaeb4e
3 files changed
+289 -1

No files matched your search

+81
View File
@@ -20767,6 +20767,18 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else {
if (((uint64_t)(io2_a_src - iop_a_src)) < 12) {
if (a_src && a_src->meta.closed) {
if (self->private_impl
.f_quirk_enabled_replace_invalid_utf_8) {
(iop_a_src += 6, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3))
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(6))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
}
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
goto exit;
@@ -20837,6 +20849,21 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
}
}
if (self->private_impl
.f_quirk_enabled_replace_invalid_utf_8) {
if (((uint64_t)(io2_a_src - iop_a_src)) < 6) {
status = wuffs_base__make_status(
wuffs_json__error__internal_error_inconsistent_i_o);
goto exit;
}
(iop_a_src += 6, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
}
}
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
@@ -20851,8 +20878,22 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
goto label__string_loop_outer__continue;
}
}
if (a_src && a_src->meta.closed) {
if (self->private_impl
.f_quirk_enabled_replace_invalid_utf_8) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(1))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
goto label__string_loop_inner__continue;
}
status =
wuffs_base__make_status(wuffs_json__error__bad_utf_8);
goto exit;
@@ -20893,8 +20934,22 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
goto label__string_loop_outer__continue;
}
}
if (a_src && a_src->meta.closed) {
if (self->private_impl
.f_quirk_enabled_replace_invalid_utf_8) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(1))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
goto label__string_loop_inner__continue;
}
status =
wuffs_base__make_status(wuffs_json__error__bad_utf_8);
goto exit;
@@ -20940,8 +20995,22 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
goto label__string_loop_outer__continue;
}
}
if (a_src && a_src->meta.closed) {
if (self->private_impl
.f_quirk_enabled_replace_invalid_utf_8) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(1))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
goto label__string_loop_inner__continue;
}
status =
wuffs_base__make_status(wuffs_json__error__bad_utf_8);
goto exit;
@@ -20986,12 +21055,24 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
goto label__string_loop_outer__continue;
}
}
if (v_char == 128) {
status = wuffs_base__make_status(
wuffs_json__error__bad_c0_control_code);
goto exit;
}
if (self->private_impl.f_quirk_enabled_replace_invalid_utf_8) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
goto label__string_loop_inner__continue;
}
status = wuffs_base__make_status(wuffs_json__error__bad_utf_8);
goto exit;
}
+73 -1
View File
@@ -400,6 +400,15 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// high surrogate. We need another 6.
if args.src.available() < 12 {
if args.src.is_closed() {
if this.quirk_enabled_replace_invalid_utf_8 {
args.src.skip32_fast!(actual: 6, worst_case: 6)
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x60_FFFD,
link: 0x3,
length: 6)
continue.string_loop_outer
}
return "#bad backslash-escape"
}
yield? base."$short read"
@@ -460,7 +469,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// through to "#invalid
// backslash-escape".
uni4_rollback = 4
while uni4_rollback > 0 {
while uni4_rollback > 0,
inv args.dst.available() > 0,
{
uni4_rollback -= 1
if args.src.can_undo_byte() {
args.src.undo_byte!()
@@ -470,6 +481,19 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
}
}
if this.quirk_enabled_replace_invalid_utf_8 {
if args.src.available() < 6 {
return "#internal error: inconsistent I/O"
}
args.src.skip32_fast!(actual: 6, worst_case: 6)
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x60_FFFD,
link: 0x3,
length: 6)
continue.string_loop_outer
}
// -------- END backslash-u.
}
@@ -484,8 +508,20 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
link: 0x3,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
continue.string_loop_outer
}
}
if args.src.is_closed() {
if this.quirk_enabled_replace_invalid_utf_8 {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x60_FFFD,
link: 0x3,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_inner
}
return "#bad UTF-8"
}
yield? base."$short read"
@@ -520,8 +556,20 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
link: 0x3,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
continue.string_loop_outer
}
}
if args.src.is_closed() {
if this.quirk_enabled_replace_invalid_utf_8 {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x60_FFFD,
link: 0x3,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_inner
}
return "#bad UTF-8"
}
yield? base."$short read"
@@ -561,8 +609,20 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
link: 0x3,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
continue.string_loop_outer
}
}
if args.src.is_closed() {
if this.quirk_enabled_replace_invalid_utf_8 {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x60_FFFD,
link: 0x3,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_inner
}
return "#bad UTF-8"
}
yield? base."$short read"
@@ -600,10 +660,22 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
link: 0x3,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
continue.string_loop_outer
}
}
if char == 0x80 {
return "#bad C0 control code"
}
if this.quirk_enabled_replace_invalid_utf_8 {
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x60_FFFD,
link: 0x3,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_inner
}
return "#bad UTF-8"
} endwhile.string_loop_inner
} endwhile.string_loop_outer
+135
View File
@@ -1261,6 +1261,140 @@ test_wuffs_json_decode_quirk_allow_trailing_etc() {
return NULL;
}
const char* //
test_wuffs_json_decode_quirk_replace_invalid_utf_8() {
CHECK_FOCUS(__func__);
// Decoding str should produce want, with invalid UTF-8 replaced by "?". A
// proper JSON decoder (with the quirk enabled) would replace with
// "\xEF\xBF\xBD", the UTF-8 encoding of U+FFFD, but using "?" leads to
// clearer, shorter test cases.
struct {
const char* want;
const char* str;
} test_cases[] = {
// Valid UTF-8.
{.want = "abc", .str = "\"abc\""},
{.want = "del\xCE\x94ta", .str = "\"del\\u0394ta\""},
{.want = "del\xCE\x94ta", .str = "\"del\xCE\x94ta\""},
// Invalid UTF-8: right byte lengths, wrong bytes.
{.want = "1byte?yz", .str = "\"1byte\xFFyz\""},
{.want = "2byte??yz", .str = "\"2byte\xCE\xFFyz\""},
{.want = "3byte???yz", .str = "\"3byte\xE2\x98\xFFyz\""},
{.want = "4byte????yz", .str = "\"4byte\xF0\x9F\x92\xFFyz\""},
// Invalid UTF-8: wrong byte lengths.
{.want = "?", .str = "\"\xCE\""},
{.want = "?g", .str = "\"\xCEg\""},
{.want = "?gh", .str = "\"\xCEgh\""},
{.want = "j?", .str = "\"j\xE2\""},
{.want = "j?l", .str = "\"j\xE2l\""},
{.want = "j?lm", .str = "\"j\xE2lm\""},
{.want = "?", .str = "\"\xF0\""},
{.want = "?r", .str = "\"\xF0r\""},
{.want = "?rs", .str = "\"\xF0rs\""},
// U+DC00 (as an unpaired surrogate) is either 1 or 3 '?'s depending on
// whether it's backslash-u or backslash-x.
{.want = "a?z", .str = "\"a\\uDC00z\""},
{.want = "a?zzzzzz", .str = "\"a\\uDC00zzzzzz\""},
{.want = "a???z", .str = "\"a\xED\xB0\x80z\""},
{.want = "a???zzzzzz", .str = "\"a\xED\xB0\x80zzzzzz\""},
// 1 or 2 unpaired surrogates each become '?'s, but for 3 surrogates
// where consecutive surrogates make a valid pair, there's only 1 '?'.
{.want = "a?z", .str = "\"a\\uD800z\""},
{.want = "a??z", .str = "\"a\\uD800\\uDBFFz\""},
{.want = "a?\xF4\x8F\xBF\xBFz", .str = "\"a\\uD800\\uDBFF\\uDFFFz\""},
{.want = "a\xF0\x90\x80\x80?z", .str = "\"a\\uD800\\uDC00\\uDFFFz\""},
};
int tc;
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
wuffs_json__decoder dec;
CHECK_STATUS("initialize", wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
wuffs_json__decoder__set_quirk_enabled(
&dec, WUFFS_JSON__QUIRK_REPLACE_INVALID_UTF_8, true);
wuffs_base__io_buffer have =
wuffs_base__make_io_buffer_writer(global_have_slice);
wuffs_base__token_buffer tok =
wuffs_base__make_token_buffer_writer(global_have_token_slice);
wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader(
wuffs_base__make_slice_u8((void*)(test_cases[tc].str),
strlen(test_cases[tc].str)),
true);
CHECK_STATUS("decode_tokens",
wuffs_json__decoder__decode_tokens(&dec, &tok, &src));
uint64_t src_index = 0;
while (tok.meta.ri < tok.meta.wi) {
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
uint64_t vbc = wuffs_base__token__value_base_category(t);
uint64_t vbd = wuffs_base__token__value_base_detail(t);
uint64_t token_length = wuffs_base__token__length(t);
if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) {
uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL];
size_t n = wuffs_base__utf_8__encode(
wuffs_base__make_slice_u8(&u[0],
WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL),
vbd);
if (vbd == 0xFFFD) {
u[0] = '?';
n = 1;
}
if ((have.data.len - have.meta.wi) < n) {
RETURN_FAIL("tc=%d: token too long", tc);
}
memcpy(&have.data.ptr[have.meta.wi], &u[0], n);
have.meta.wi += n;
} else if (vbc == WUFFS_BASE__TOKEN__VBC__STRING) {
if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) {
// No-op.
} else if (vbd &
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) {
if ((have.data.len - have.meta.wi) < token_length) {
RETURN_FAIL("tc=%d: token too long", tc);
}
memcpy(&have.data.ptr[have.meta.wi], &test_cases[tc].str[src_index],
token_length);
have.meta.wi += token_length;
} else {
RETURN_FAIL("tc=%d: unexpected string-token conversion", tc);
}
} else {
RETURN_FAIL("tc=%d: unexpected token", tc);
}
src_index += token_length;
}
if (src_index != src.meta.ri) {
RETURN_FAIL("tc=%d: src_index: have %zu, want %zu", tc, src_index,
src.meta.ri);
}
if (have.meta.wi >= have.data.len) {
RETURN_FAIL("tc=%d: too many have bytes", tc);
}
have.data.ptr[have.meta.wi] = '\x00';
size_t len = strlen(test_cases[tc].want);
if ((len != have.meta.wi) ||
(memcmp(have.data.ptr, test_cases[tc].want, len) != 0)) {
RETURN_FAIL("tc=%d: have \"%s\", want \"%s\"", tc, have.data.ptr,
test_cases[tc].want);
}
}
return NULL;
}
const char* //
test_wuffs_json_decode_unicode4_escapes() {
CHECK_FOCUS(__func__);
@@ -1614,6 +1748,7 @@ proc tests[] = {
test_wuffs_json_decode_prior_valid_utf_8,
test_wuffs_json_decode_quirk_allow_leading_etc,
test_wuffs_json_decode_quirk_allow_trailing_etc,
test_wuffs_json_decode_quirk_replace_invalid_utf_8,
test_wuffs_json_decode_src_io_buffer_length,
test_wuffs_json_decode_string,
test_wuffs_json_decode_unicode4_escapes,