mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Add std/json quirk_allow_backslash_x
This commit is contained in:
1 parent
c6b2aded30
commit
502c8eff56
4 files changed
+239
-18
No files matched your search
@@ -272,6 +272,13 @@ class TokenStream {
|
||||
m_curr_token_end_src_index(0) {
|
||||
m_status =
|
||||
m_dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0);
|
||||
|
||||
// Uncomment these lines to enable the WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X
|
||||
// option, discussed in a separate comment.
|
||||
//
|
||||
// if (m_status.is_ok()) {
|
||||
// m_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true);
|
||||
// }
|
||||
}
|
||||
|
||||
Result peek() { return peek_or_next(false); }
|
||||
@@ -584,11 +591,43 @@ JsonThing::parse_string(TokenStream& ts, TokenStream::Result tsr) {
|
||||
case WUFFS_BASE__TOKEN__VBC__STRING: {
|
||||
if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) {
|
||||
// No-op.
|
||||
|
||||
} else if (vbd &
|
||||
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) {
|
||||
const char* ptr = // Convert from (uint8_t*).
|
||||
static_cast<const char*>(static_cast<void*>(tsr.src_data.ptr));
|
||||
jt.value.s.append(ptr, tsr.src_data.len);
|
||||
|
||||
} else if (
|
||||
vbd &
|
||||
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X) {
|
||||
// We shouldn't get here unless we enable the
|
||||
// WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X option. The jsonfindptrs
|
||||
// program doesn't enable that by default, but if you're copy/pasting
|
||||
// this JsonThing code and your program does enable that option,
|
||||
// here's how to handle it.
|
||||
wuffs_base__slice_u8 encoded = tsr.src_data;
|
||||
if (encoded.len & 3) {
|
||||
return Result(
|
||||
"main: internal error: \\x token length not a multiple of 4",
|
||||
JsonThing());
|
||||
}
|
||||
while (encoded.len) {
|
||||
uint8_t decoded[64];
|
||||
size_t len = wuffs_base__hexadecimal__decode4(
|
||||
wuffs_base__make_slice_u8(&decoded[0], 64), encoded);
|
||||
if ((len > 64) || ((len * 4) > encoded.len)) {
|
||||
return Result(
|
||||
"main: internal error: inconsistent hexadecimal decoding",
|
||||
JsonThing());
|
||||
}
|
||||
const char* ptr = // Convert from (uint8_t*).
|
||||
static_cast<const char*>(static_cast<void*>(&decoded[0]));
|
||||
jt.value.s.append(ptr, len);
|
||||
encoded.ptr += len * 4;
|
||||
encoded.len -= len * 4;
|
||||
}
|
||||
|
||||
} else {
|
||||
return Result(
|
||||
"main: internal error: unexpected string-token conversion",
|
||||
|
||||
@@ -20532,6 +20532,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
uint8_t v_char = 0;
|
||||
uint8_t v_class = 0;
|
||||
uint32_t v_multi_byte_utf8 = 0;
|
||||
uint32_t v_backslash_x_length = 0;
|
||||
uint8_t v_backslash_x_ok = 0;
|
||||
uint32_t v_backslash_x_string = 0;
|
||||
uint8_t v_uni4_ok = 0;
|
||||
uint64_t v_uni4_string = 0;
|
||||
uint32_t v_uni4_value = 0;
|
||||
@@ -21008,6 +21011,54 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
}
|
||||
} else if ((v_c == 120) &&
|
||||
self->private_impl
|
||||
.f_quirk_enabled_allow_backslash_x) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
|
||||
if (a_src && a_src->meta.closed) {
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_json__error__bad_backslash_escape);
|
||||
goto exit;
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop_outer__continue;
|
||||
}
|
||||
v_backslash_x_length = 0;
|
||||
while ((v_backslash_x_length <= 65531) &&
|
||||
(((uint64_t)(io2_a_src - iop_a_src)) >= 4)) {
|
||||
v_backslash_x_string =
|
||||
wuffs_base__load_u32le__no_bounds_check(iop_a_src);
|
||||
v_backslash_x_ok = 128;
|
||||
v_c = wuffs_json__lut_hexadecimal_digits[(
|
||||
255 & (v_backslash_x_string >> 16))];
|
||||
v_backslash_x_ok &= v_c;
|
||||
v_c = wuffs_json__lut_hexadecimal_digits[(
|
||||
255 & (v_backslash_x_string >> 24))];
|
||||
v_backslash_x_ok &= v_c;
|
||||
if ((v_backslash_x_ok == 0) ||
|
||||
((v_backslash_x_string & 65535) != 30812)) {
|
||||
goto label__1__break;
|
||||
}
|
||||
(iop_a_src += 4, wuffs_base__make_empty_struct());
|
||||
v_backslash_x_length += 4;
|
||||
}
|
||||
label__1__break:;
|
||||
if (v_backslash_x_length == 0) {
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_json__error__bad_backslash_escape);
|
||||
goto exit;
|
||||
}
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194432))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(v_backslash_x_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_json__error__bad_backslash_escape);
|
||||
@@ -21044,7 +21095,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21100,7 +21151,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21161,7 +21212,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21222,7 +21273,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
}
|
||||
label__string_loop_outer__break:;
|
||||
label__1__continue:;
|
||||
label__2__continue:;
|
||||
while (true) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
|
||||
if (a_src && a_src->meta.closed) {
|
||||
@@ -21231,14 +21282,14 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
|
||||
goto label__1__continue;
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
|
||||
goto label__2__continue;
|
||||
}
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
|
||||
goto label__1__continue;
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
|
||||
goto label__2__continue;
|
||||
}
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
@@ -21246,9 +21297,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__1__break;
|
||||
goto label__2__break;
|
||||
}
|
||||
label__1__break:;
|
||||
label__2__break:;
|
||||
if (0 == (v_expect & 16)) {
|
||||
v_expect = 8;
|
||||
goto label__outer__continue;
|
||||
@@ -21293,7 +21344,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_number_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__2__break;
|
||||
goto label__3__break;
|
||||
}
|
||||
while (v_number_length > 0) {
|
||||
v_number_length -= 1;
|
||||
@@ -21315,15 +21366,15 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16);
|
||||
while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
|
||||
}
|
||||
}
|
||||
}
|
||||
label__2__break:;
|
||||
label__3__break:;
|
||||
goto label__goto_parsed_a_leaf_value__break;
|
||||
} else if (v_class == 5) {
|
||||
v_vminor = 2113553;
|
||||
@@ -21453,7 +21504,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_match == 1) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18);
|
||||
goto label__outer__continue;
|
||||
}
|
||||
} else if (v_class == 10) {
|
||||
@@ -21474,7 +21525,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_match == 1) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19);
|
||||
goto label__outer__continue;
|
||||
}
|
||||
} else if (v_class == 11) {
|
||||
@@ -21495,7 +21546,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_match == 1) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(20);
|
||||
goto label__outer__continue;
|
||||
}
|
||||
}
|
||||
@@ -21516,7 +21567,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
if (a_src) {
|
||||
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
|
||||
}
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(20);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(21);
|
||||
status =
|
||||
wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src);
|
||||
if (a_dst) {
|
||||
|
||||
@@ -117,6 +117,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
var class : base.u8[..= 0x0F]
|
||||
var multi_byte_utf8 : base.u32
|
||||
|
||||
var backslash_x_length : base.u32[..= 0xFFFF]
|
||||
var backslash_x_ok : base.u8
|
||||
var backslash_x_string : base.u32
|
||||
|
||||
var uni4_ok : base.u8
|
||||
var uni4_string : base.u64
|
||||
var uni4_value : base.u32[..= 0xFFFF]
|
||||
@@ -554,6 +558,52 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
continue.string_loop_outer
|
||||
}
|
||||
// -------- END backslash-capital-u.
|
||||
|
||||
} else if (c == 0x78) and
|
||||
this.quirk_enabled_allow_backslash_x { // 0x78 is 'x'.
|
||||
// -------- BEGIN backslash-x
|
||||
if args.src.available() < 4 {
|
||||
if args.src.is_closed() {
|
||||
return "#bad backslash-escape"
|
||||
}
|
||||
yield? base."$short read"
|
||||
string_length = 0
|
||||
char = 0
|
||||
continue.string_loop_outer
|
||||
}
|
||||
|
||||
backslash_x_length = 0
|
||||
while (backslash_x_length <= 0xFFFB) and (args.src.available() >= 4),
|
||||
inv args.dst.available() > 0,
|
||||
{
|
||||
backslash_x_string = args.src.peek_u32le()
|
||||
backslash_x_ok = 0x80
|
||||
|
||||
c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 16)]
|
||||
backslash_x_ok &= c
|
||||
c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 24)]
|
||||
backslash_x_ok &= c
|
||||
|
||||
if (backslash_x_ok == 0) or
|
||||
((backslash_x_string & 0xFFFF) <> 0x785C) {
|
||||
// It wasn't "\\x34", for some
|
||||
// hexadecimal digits "34".
|
||||
break
|
||||
}
|
||||
args.src.skip32_fast!(actual: 4, worst_case: 4)
|
||||
backslash_x_length += 4
|
||||
}
|
||||
|
||||
if backslash_x_length == 0 {
|
||||
return "#bad backslash-escape"
|
||||
}
|
||||
args.dst.write_fast_token!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0080,
|
||||
link: 0x3,
|
||||
length: backslash_x_length)
|
||||
continue.string_loop_outer
|
||||
// -------- END backslash-x
|
||||
}
|
||||
|
||||
return "#bad backslash-escape"
|
||||
|
||||
@@ -1249,6 +1249,86 @@ test_wuffs_json_decode_quirk_allow_backslash_etc() {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_json_decode_quirk_allow_backslash_x() {
|
||||
CHECK_FOCUS(__func__);
|
||||
|
||||
struct {
|
||||
uint64_t want_bytes;
|
||||
const char* want_status_repr;
|
||||
const char* str;
|
||||
} test_cases[] = {
|
||||
{.want_bytes = 0x12789A,
|
||||
.want_status_repr = NULL,
|
||||
.str = "\"\\x12\\u3456\\x78\\x9A\""},
|
||||
{.want_bytes = 0x00,
|
||||
.want_status_repr = wuffs_json__error__bad_backslash_escape,
|
||||
.str = "\"a\\X6A\""},
|
||||
{.want_bytes = 0x6A6B,
|
||||
.want_status_repr = NULL,
|
||||
.str = "\"a\\x6A\\x6bz\""},
|
||||
{.want_bytes = 0x6A,
|
||||
.want_status_repr = wuffs_json__error__bad_backslash_escape,
|
||||
.str = "\"a\\x6A\\x6yz\""},
|
||||
{.want_bytes = 0x00,
|
||||
.want_status_repr = wuffs_json__error__bad_backslash_escape,
|
||||
.str = "\"a\\x\""},
|
||||
};
|
||||
|
||||
int tc;
|
||||
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
|
||||
wuffs_json__decoder dec;
|
||||
CHECK_STATUS("initialize", wuffs_json__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
|
||||
wuffs_json__decoder__set_quirk_enabled(
|
||||
&dec, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true);
|
||||
|
||||
wuffs_base__token_buffer tok =
|
||||
wuffs_base__make_token_buffer_writer(global_have_token_slice);
|
||||
wuffs_base__slice_u8 src_slice = wuffs_base__make_slice_u8(
|
||||
(void*)(test_cases[tc].str), strlen(test_cases[tc].str));
|
||||
wuffs_base__io_buffer src =
|
||||
wuffs_base__make_io_buffer_reader(src_slice, true);
|
||||
const char* have_status_repr =
|
||||
wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr;
|
||||
if (have_status_repr != test_cases[tc].want_status_repr) {
|
||||
RETURN_FAIL("tc=%d: decode_tokens: have \"%s\", want \"%s\"", tc,
|
||||
have_status_repr, test_cases[tc].want_status_repr);
|
||||
}
|
||||
|
||||
uint64_t src_index = 0;
|
||||
uint64_t have_bytes = 0;
|
||||
while (tok.meta.ri < tok.meta.wi) {
|
||||
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
|
||||
uint64_t vbc = wuffs_base__token__value_base_category(t);
|
||||
uint64_t vbd = wuffs_base__token__value_base_detail(t);
|
||||
uint64_t token_length = wuffs_base__token__length(t);
|
||||
if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) &&
|
||||
(vbd ==
|
||||
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) {
|
||||
uint8_t b[8];
|
||||
size_t n = wuffs_base__hexadecimal__decode4(
|
||||
wuffs_base__make_slice_u8(&b[0], 8),
|
||||
wuffs_base__make_slice_u8(src_slice.ptr + src_index, token_length));
|
||||
size_t i = 0;
|
||||
for (; i < n; i++) {
|
||||
have_bytes <<= 8;
|
||||
have_bytes |= b[i];
|
||||
}
|
||||
}
|
||||
|
||||
src_index += token_length;
|
||||
}
|
||||
if (have_bytes != test_cases[tc].want_bytes) {
|
||||
RETURN_FAIL("tc=%d: have U+%08" PRIX64 ", want U+%08" PRIX64, tc,
|
||||
have_bytes, test_cases[tc].want_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_json_decode_quirk_allow_leading_etc() {
|
||||
CHECK_FOCUS(__func__);
|
||||
@@ -1892,6 +1972,7 @@ proc tests[] = {
|
||||
test_wuffs_json_decode_long_numbers,
|
||||
test_wuffs_json_decode_prior_valid_utf_8,
|
||||
test_wuffs_json_decode_quirk_allow_backslash_etc,
|
||||
test_wuffs_json_decode_quirk_allow_backslash_x,
|
||||
test_wuffs_json_decode_quirk_allow_leading_etc,
|
||||
test_wuffs_json_decode_quirk_allow_trailing_etc,
|
||||
test_wuffs_json_decode_quirk_replace_invalid_utf_8,
|
||||
|
||||
Reference in new issue
Block a user