Add std/json quirk_allow_backslash_x

This commit is contained in:
Nigel Tao committed 2020-03-21 21:42:30 +11:00
1 parent c6b2aded30
commit 502c8eff56
4 files changed
+239 -18

No files matched your search

+39
View File
@@ -272,6 +272,13 @@ class TokenStream {
m_curr_token_end_src_index(0) {
m_status =
m_dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0);
// Uncomment these lines to enable the WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X
// option, discussed in a separate comment.
//
// if (m_status.is_ok()) {
// m_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true);
// }
}
Result peek() { return peek_or_next(false); }
@@ -584,11 +591,43 @@ JsonThing::parse_string(TokenStream& ts, TokenStream::Result tsr) {
case WUFFS_BASE__TOKEN__VBC__STRING: {
if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) {
// No-op.
} else if (vbd &
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) {
const char* ptr = // Convert from (uint8_t*).
static_cast<const char*>(static_cast<void*>(tsr.src_data.ptr));
jt.value.s.append(ptr, tsr.src_data.len);
} else if (
vbd &
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X) {
// We shouldn't get here unless we enable the
// WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X option. The jsonfindptrs
// program doesn't enable that by default, but if you're copy/pasting
// this JsonThing code and your program does enable that option,
// here's how to handle it.
wuffs_base__slice_u8 encoded = tsr.src_data;
if (encoded.len & 3) {
return Result(
"main: internal error: \\x token length not a multiple of 4",
JsonThing());
}
while (encoded.len) {
uint8_t decoded[64];
size_t len = wuffs_base__hexadecimal__decode4(
wuffs_base__make_slice_u8(&decoded[0], 64), encoded);
if ((len > 64) || ((len * 4) > encoded.len)) {
return Result(
"main: internal error: inconsistent hexadecimal decoding",
JsonThing());
}
const char* ptr = // Convert from (uint8_t*).
static_cast<const char*>(static_cast<void*>(&decoded[0]));
jt.value.s.append(ptr, len);
encoded.ptr += len * 4;
encoded.len -= len * 4;
}
} else {
return Result(
"main: internal error: unexpected string-token conversion",
+69 -18
View File
@@ -20532,6 +20532,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
uint8_t v_char = 0;
uint8_t v_class = 0;
uint32_t v_multi_byte_utf8 = 0;
uint32_t v_backslash_x_length = 0;
uint8_t v_backslash_x_ok = 0;
uint32_t v_backslash_x_string = 0;
uint8_t v_uni4_ok = 0;
uint64_t v_uni4_string = 0;
uint32_t v_uni4_value = 0;
@@ -21008,6 +21011,54 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
}
} else if ((v_c == 120) &&
self->private_impl
.f_quirk_enabled_allow_backslash_x) {
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
goto exit;
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
v_string_length = 0;
v_char = 0;
goto label__string_loop_outer__continue;
}
v_backslash_x_length = 0;
while ((v_backslash_x_length <= 65531) &&
(((uint64_t)(io2_a_src - iop_a_src)) >= 4)) {
v_backslash_x_string =
wuffs_base__load_u32le__no_bounds_check(iop_a_src);
v_backslash_x_ok = 128;
v_c = wuffs_json__lut_hexadecimal_digits[(
255 & (v_backslash_x_string >> 16))];
v_backslash_x_ok &= v_c;
v_c = wuffs_json__lut_hexadecimal_digits[(
255 & (v_backslash_x_string >> 24))];
v_backslash_x_ok &= v_c;
if ((v_backslash_x_ok == 0) ||
((v_backslash_x_string & 65535) != 30812)) {
goto label__1__break;
}
(iop_a_src += 4, wuffs_base__make_empty_struct());
v_backslash_x_length += 4;
}
label__1__break:;
if (v_backslash_x_length == 0) {
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
goto exit;
}
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194432))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(v_backslash_x_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
}
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
@@ -21044,7 +21095,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
v_string_length = 0;
v_char = 0;
goto label__string_loop_outer__continue;
@@ -21100,7 +21151,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
v_string_length = 0;
v_char = 0;
goto label__string_loop_outer__continue;
@@ -21161,7 +21212,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
v_string_length = 0;
v_char = 0;
goto label__string_loop_outer__continue;
@@ -21222,7 +21273,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
}
label__string_loop_outer__break:;
label__1__continue:;
label__2__continue:;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (a_src && a_src->meta.closed) {
@@ -21231,14 +21282,14 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
goto label__1__continue;
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
goto label__2__continue;
}
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
goto label__1__continue;
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
goto label__2__continue;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
@@ -21246,9 +21297,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__1__break;
goto label__2__break;
}
label__1__break:;
label__2__break:;
if (0 == (v_expect & 16)) {
v_expect = 8;
goto label__outer__continue;
@@ -21293,7 +21344,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(v_number_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__2__break;
goto label__3__break;
}
while (v_number_length > 0) {
v_number_length -= 1;
@@ -21315,15 +21366,15 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16);
while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
}
}
}
label__2__break:;
label__3__break:;
goto label__goto_parsed_a_leaf_value__break;
} else if (v_class == 5) {
v_vminor = 2113553;
@@ -21453,7 +21504,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_match == 1) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18);
goto label__outer__continue;
}
} else if (v_class == 10) {
@@ -21474,7 +21525,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_match == 1) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19);
goto label__outer__continue;
}
} else if (v_class == 11) {
@@ -21495,7 +21546,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_match == 1) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(20);
goto label__outer__continue;
}
}
@@ -21516,7 +21567,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(20);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(21);
status =
wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src);
if (a_dst) {
+50
View File
@@ -117,6 +117,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var class : base.u8[..= 0x0F]
var multi_byte_utf8 : base.u32
var backslash_x_length : base.u32[..= 0xFFFF]
var backslash_x_ok : base.u8
var backslash_x_string : base.u32
var uni4_ok : base.u8
var uni4_string : base.u64
var uni4_value : base.u32[..= 0xFFFF]
@@ -554,6 +558,52 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.string_loop_outer
}
// -------- END backslash-capital-u.
} else if (c == 0x78) and
this.quirk_enabled_allow_backslash_x { // 0x78 is 'x'.
// -------- BEGIN backslash-x
if args.src.available() < 4 {
if args.src.is_closed() {
return "#bad backslash-escape"
}
yield? base."$short read"
string_length = 0
char = 0
continue.string_loop_outer
}
backslash_x_length = 0
while (backslash_x_length <= 0xFFFB) and (args.src.available() >= 4),
inv args.dst.available() > 0,
{
backslash_x_string = args.src.peek_u32le()
backslash_x_ok = 0x80
c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 16)]
backslash_x_ok &= c
c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 24)]
backslash_x_ok &= c
if (backslash_x_ok == 0) or
((backslash_x_string & 0xFFFF) <> 0x785C) {
// It wasn't "\\x34", for some
// hexadecimal digits "34".
break
}
args.src.skip32_fast!(actual: 4, worst_case: 4)
backslash_x_length += 4
}
if backslash_x_length == 0 {
return "#bad backslash-escape"
}
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x40_0080,
link: 0x3,
length: backslash_x_length)
continue.string_loop_outer
// -------- END backslash-x
}
return "#bad backslash-escape"
+81
View File
@@ -1249,6 +1249,86 @@ test_wuffs_json_decode_quirk_allow_backslash_etc() {
return NULL;
}
const char* //
test_wuffs_json_decode_quirk_allow_backslash_x() {
CHECK_FOCUS(__func__);
struct {
uint64_t want_bytes;
const char* want_status_repr;
const char* str;
} test_cases[] = {
{.want_bytes = 0x12789A,
.want_status_repr = NULL,
.str = "\"\\x12\\u3456\\x78\\x9A\""},
{.want_bytes = 0x00,
.want_status_repr = wuffs_json__error__bad_backslash_escape,
.str = "\"a\\X6A\""},
{.want_bytes = 0x6A6B,
.want_status_repr = NULL,
.str = "\"a\\x6A\\x6bz\""},
{.want_bytes = 0x6A,
.want_status_repr = wuffs_json__error__bad_backslash_escape,
.str = "\"a\\x6A\\x6yz\""},
{.want_bytes = 0x00,
.want_status_repr = wuffs_json__error__bad_backslash_escape,
.str = "\"a\\x\""},
};
int tc;
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
wuffs_json__decoder dec;
CHECK_STATUS("initialize", wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
wuffs_json__decoder__set_quirk_enabled(
&dec, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true);
wuffs_base__token_buffer tok =
wuffs_base__make_token_buffer_writer(global_have_token_slice);
wuffs_base__slice_u8 src_slice = wuffs_base__make_slice_u8(
(void*)(test_cases[tc].str), strlen(test_cases[tc].str));
wuffs_base__io_buffer src =
wuffs_base__make_io_buffer_reader(src_slice, true);
const char* have_status_repr =
wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr;
if (have_status_repr != test_cases[tc].want_status_repr) {
RETURN_FAIL("tc=%d: decode_tokens: have \"%s\", want \"%s\"", tc,
have_status_repr, test_cases[tc].want_status_repr);
}
uint64_t src_index = 0;
uint64_t have_bytes = 0;
while (tok.meta.ri < tok.meta.wi) {
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
uint64_t vbc = wuffs_base__token__value_base_category(t);
uint64_t vbd = wuffs_base__token__value_base_detail(t);
uint64_t token_length = wuffs_base__token__length(t);
if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) &&
(vbd ==
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) {
uint8_t b[8];
size_t n = wuffs_base__hexadecimal__decode4(
wuffs_base__make_slice_u8(&b[0], 8),
wuffs_base__make_slice_u8(src_slice.ptr + src_index, token_length));
size_t i = 0;
for (; i < n; i++) {
have_bytes <<= 8;
have_bytes |= b[i];
}
}
src_index += token_length;
}
if (have_bytes != test_cases[tc].want_bytes) {
RETURN_FAIL("tc=%d: have U+%08" PRIX64 ", want U+%08" PRIX64, tc,
have_bytes, test_cases[tc].want_bytes);
}
}
return NULL;
}
const char* //
test_wuffs_json_decode_quirk_allow_leading_etc() {
CHECK_FOCUS(__func__);
@@ -1892,6 +1972,7 @@ proc tests[] = {
test_wuffs_json_decode_long_numbers,
test_wuffs_json_decode_prior_valid_utf_8,
test_wuffs_json_decode_quirk_allow_backslash_etc,
test_wuffs_json_decode_quirk_allow_backslash_x,
test_wuffs_json_decode_quirk_allow_leading_etc,
test_wuffs_json_decode_quirk_allow_trailing_etc,
test_wuffs_json_decode_quirk_replace_invalid_utf_8,