Have std/json decode Unicode surrogate pairs

This commit is contained in:
Nigel Tao committed 2020-02-25 10:45:31 +11:00
1 parent 7b199116b0
commit 16b0c4652a
4 files changed
+305 -20

No files matched your search

+14 -2
View File
@@ -327,19 +327,31 @@ const char* handle_string(parsed_token pt) {
} else if (vbd <= 0x07FF) {
// The UTF-8 encoding takes 2 bytes.
uint8_t esc2[6];
uint8_t esc2[2];
esc2[0] = 0xC0 | (uint8_t)((vbd >> 6));
esc2[1] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F);
TRY(write_dst(&esc2[0], 2));
} else if (vbd <= 0xFFFF) {
if ((0xD800 <= vbd) && (vbd <= 0xDFFF)) {
return "main: unexpected Unicode surrogate";
}
// The UTF-8 encoding takes 3 bytes.
uint8_t esc3[6];
uint8_t esc3[3];
esc3[0] = 0xE0 | (uint8_t)((vbd >> 12));
esc3[1] = 0x80 | (uint8_t)((vbd >> 6) & 0x3F);
esc3[2] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F);
TRY(write_dst(&esc3[0], 3));
} else if (vbd <= 0x10FFFF) {
// The UTF-8 encoding takes 4 bytes.
uint8_t esc4[4];
esc4[0] = 0xF0 | (uint8_t)((vbd >> 18));
esc4[1] = 0x80 | (uint8_t)((vbd >> 12) & 0x3F);
esc4[2] = 0x80 | (uint8_t)((vbd >> 6) & 0x3F);
esc4[3] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F);
TRY(write_dst(&esc4[0], 4));
} else {
return "main: unexpected Unicode code point";
}
+84 -11
View File
@@ -18648,8 +18648,10 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
uint8_t v_class = 0;
uint8_t v_backslash = 0;
uint8_t v_uni4_ok = 0;
uint32_t v_uni4_string = 0;
uint64_t v_uni4_string = 0;
uint32_t v_uni4_value = 0;
uint32_t v_uni4_high_surrogate = 0;
uint32_t v_uni4_rollback = 0;
uint32_t v_expect = 0;
uint32_t v_expect_after_value = 0;
@@ -18880,8 +18882,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
v_string_length = 0;
goto label__string_loop__continue;
}
v_uni4_string = ((uint32_t)(
(((uint64_t)(wuffs_base__load_u48le(iop_a_src))) >> 16)));
v_uni4_string =
(((uint64_t)(wuffs_base__load_u48le(iop_a_src))) >> 16);
v_uni4_value = 0;
v_uni4_ok = 128;
v_c = wuffs_json__hexadecimal_digits[(255 &
@@ -18900,13 +18902,84 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(v_uni4_string >> 24))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 0);
if (v_uni4_ok != 0) {
if (v_uni4_ok == 0) {
} else if ((v_uni4_value < 55296) || (57343 < v_uni4_value)) {
(iop_a_src += 6, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((8388608 | ((uint64_t)(v_uni4_value)))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop__continue;
} else if (v_uni4_value >= 56320) {
} else {
if (((uint64_t)(io2_a_src - iop_a_src)) < 12) {
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(
wuffs_json__error__bad_input);
goto exit;
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9);
v_string_length = 0;
v_uni4_value = 0;
goto label__string_loop__continue;
}
(iop_a_src += 4, wuffs_base__make_empty_struct());
v_uni4_string = (wuffs_base__load_u64le(iop_a_src) >> 16);
if (((255 & (v_uni4_string >> 0)) != 92) ||
((255 & (v_uni4_string >> 8)) != 117)) {
v_uni4_high_surrogate = 0;
v_uni4_value = 0;
v_uni4_ok = 0;
} else {
v_uni4_high_surrogate =
(65536 + ((v_uni4_value - 55296) << 10));
v_uni4_value = 0;
v_uni4_ok = 128;
v_uni4_string >>= 16;
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 0))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 12);
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 8))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 8);
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 16))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 4);
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 24))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 0);
}
if ((v_uni4_ok != 0) && (56320 <= v_uni4_value) &&
(v_uni4_value <= 57343)) {
v_uni4_value -= 56320;
(iop_a_src += 8, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(
(8388608 | ((uint64_t)((v_uni4_high_surrogate +
v_uni4_value))))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(12))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop__continue;
} else {
v_uni4_rollback = 4;
while (v_uni4_rollback > 0) {
v_uni4_rollback -= 1;
if (iop_a_src > io1_a_src) {
(iop_a_src--, wuffs_base__make_empty_struct());
} else {
status = wuffs_base__make_status(
wuffs_json__error__internal_error_inconsistent_i_o);
goto exit;
}
}
}
}
}
status = wuffs_base__make_status(wuffs_json__error__bad_input);
@@ -18933,13 +19006,13 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
goto label__1__continue;
}
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
goto label__1__continue;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
@@ -19015,11 +19088,11 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
}
}
}
@@ -19144,7 +19217,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_match == 1) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
goto label__outer__continue;
}
} else if (v_class == 10) {
@@ -19164,7 +19237,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_match == 1) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
goto label__outer__continue;
}
} else if (v_class == 11) {
@@ -19184,7 +19257,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_match == 1) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16);
goto label__outer__continue;
}
}
+96 -6
View File
@@ -59,9 +59,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var c : base.u8
var class : base.u8[..= 0x0F]
var backslash : base.u8
var uni4_ok : base.u8
var uni4_string : base.u32
var uni4_value : base.u32[..= 0xFFFF]
var uni4_ok : base.u8
var uni4_string : base.u64
var uni4_value : base.u32[..= 0xFFFF]
var uni4_high_surrogate : base.u32[..= 0x10_FC00]
var uni4_rollback : base.u32
// expect is a bitmask of what the next character class can be.
//
@@ -250,6 +253,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.string_loop
} else if backslash == 0x75 { // 0x75 is 'u'.
// -------- BEGIN backslash-u.
if args.src.available() < 6 {
if args.src.is_closed() {
return "#bad input"
@@ -258,9 +262,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
string_length = 0
continue.string_loop
}
uni4_string = (args.src.peek_u48le_as_u64() >> 16) as base.u32
uni4_string = args.src.peek_u48le_as_u64() >> 16
uni4_value = 0
uni4_ok = 0x80
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 12
@@ -273,15 +279,99 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 0
if uni4_ok <> 0 {
if uni4_ok == 0 {
// It wasn't 4 hexadecimal digits. No-op
// (and fall through to "#bad input").
} else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) {
// Not a Unicode surrogate. We're good.
args.src.skip32_fast!(actual: 6, worst_case: 6)
args.dst.write_fast_token!(
value: 0x80_0000 | (uni4_value as base.u64),
length: 6)
continue.string_loop
}
} else if uni4_value >= 0xDC00 {
// Low surrogate. No-op (and fall through
// to "#bad input").
} else {
// High surrogate, which needs to be
// followed by a "\\u1234" low surrogate.
// We've already peeked 6 bytes for the
// high surrogate. We need another 6.
if args.src.available() < 12 {
if args.src.is_closed() {
return "#bad input"
}
yield? base."$short read"
string_length = 0
uni4_value = 0
continue.string_loop
}
// Roll forward 4 bytes, so that calling
// peek_u64le will pick up the last 8.
args.src.skip32_fast!(actual: 4, worst_case: 4)
uni4_string = args.src.peek_u64le() >> 16
// Look for the low surrogate's "\\u".
if ((0xFF & (uni4_string >> 0)) <> 0x5C) or
((0xFF & (uni4_string >> 8)) <> 0x75) {
uni4_high_surrogate = 0
uni4_value = 0
uni4_ok = 0
} else {
uni4_high_surrogate =
0x1_0000 + ((uni4_value - 0xD800) << 10)
uni4_value = 0
uni4_ok = 0x80
uni4_string >>= 16
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 12
c = hexadecimal_digits[0xFF & (uni4_string >> 8)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 8
c = hexadecimal_digits[0xFF & (uni4_string >> 16)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 4
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 0
}
if (uni4_ok <> 0) and
(0xDC00 <= uni4_value) and (uni4_value <= 0xDFFF) {
// Emit a single token for the surrogate pair.
uni4_value -= 0xDC00
args.src.skip32_fast!(actual: 8, worst_case: 8)
args.dst.write_fast_token!(
value: 0x80_0000 | ((uni4_high_surrogate + uni4_value) as base.u64),
length: 12)
continue.string_loop
} else {
// Roll back the 4 bytes, then fall
// through to "#bad input".
uni4_rollback = 4
while uni4_rollback > 0 {
uni4_rollback -= 1
if args.src.can_undo_byte() {
args.src.undo_byte!()
} else {
return "#internal error: inconsistent I/O"
}
}
}
}
// -------- END backslash-u.
}
return "#bad input"
}
+111 -1
View File
@@ -187,6 +187,115 @@ const char* test_wuffs_json_decode_tokens() {
return NULL;
}
const char* test_wuffs_json_decode_unicode4_escapes() {
CHECK_FOCUS(__func__);
const uint32_t fail = 0xDEADBEEF;
struct {
uint32_t want;
const char* str;
} test_cases[] = {
// Simple (non-surrogate) successes.
{.want = 0x0000000A, .str = "\"\\u000a\""},
{.want = 0x0000005C, .str = "\"\\\\u1234\""}, // U+005C is '\\'.
{.want = 0x00001000, .str = "\"\\u10002345\""},
{.want = 0x00001000, .str = "\"\\u1000234\""},
{.want = 0x00001000, .str = "\"\\u100023\""},
{.want = 0x00001000, .str = "\"\\u10002\""},
{.want = 0x00001234, .str = "\"\\u1234\""},
{.want = 0x0000D7FF, .str = "\"\\ud7ff\""},
{.want = 0x0000E000, .str = "\"\\uE000\""},
{.want = 0x0000FFFF, .str = "\"\\uFffF\""},
// Unicode surrogate pair. U+0001F4A9 PILE OF POO is (U+D83D, U+DCA9),
// because ((0x03D << 10) | 0x0A9) is 0xF4A9:
// - High surrogates are in the range U+D800 ..= U+DBFF.
// - Low surrogates are in the range U+DC00 ..= U+DFFF.
{.want = 0x0001F4A9, .str = "\"\\uD83D\\udca9\""},
// More surrogate pairs.
{.want = 0x00010000, .str = "\"\\uD800\\uDC00\""},
{.want = 0x0010FFFF, .str = "\"\\uDBFF\\uDFFF\""},
// Simple (non-surrogate) failures.
{.want = fail, .str = "\"\\U1234\""},
{.want = fail, .str = "\"\\u123"},
{.want = fail, .str = "\"\\u123\""},
{.want = fail, .str = "\"\\u123x\""},
{.want = fail, .str = "\"u1234\""},
// Invalid surrogate pairs.
{.want = fail, .str = "\"\\uD800\""}, // High alone.
{.want = fail, .str = "\"\\uD83D?udca9\""}, // High then not "\\u".
{.want = fail, .str = "\"\\uD83D\\ud7ff\""}, // High then non-surrogate.
{.want = fail, .str = "\"\\uD83D\\udbff\""}, // High then high.
{.want = fail, .str = "\"\\uD83D\\ue000\""}, // High then non-surrogate.
{.want = fail, .str = "\"\\uDC00\""}, // Low alone.
{.want = fail, .str = "\"\\uDC00\\u0000\""}, // Low then non-surrogate.
{.want = fail, .str = "\"\\uDC00\\ud800\""}, // Low then high.
{.want = fail, .str = "\"\\uDC00\\udfff\""}, // Low then low.
{.want = fail, .str = "\"\\uDFFF1234\""}, // Low alone.
};
wuffs_json__decoder dec;
int tc;
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){
.data = global_have_token_slice,
});
size_t n = strlen(test_cases[tc].str);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8((void*)(test_cases[tc].str), n),
.meta = wuffs_base__make_io_buffer_meta(n, 0, 0, true),
});
wuffs_json__decoder__decode_tokens(&dec, &tok, &src);
uint32_t have = fail;
uint64_t total_length = 0;
size_t i;
for (i = tok.meta.ri; i < tok.meta.wi; i++) {
wuffs_base__token* t = &tok.data.ptr[i];
total_length =
wuffs_base__u64__sat_add(total_length, wuffs_base__token__length(t));
// Set have to the first Unicode code point token.
if ((have == fail) && ((wuffs_base__token__value_base_category(t) ==
WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT))) {
have = wuffs_base__token__value_base_detail(t);
if (have > 0x10FFFF) { // This also catches "have == fail".
RETURN_FAIL("%s: invalid Unicode code point", test_cases[tc].str);
}
uint64_t have_length = wuffs_base__token__length(t);
uint64_t want_length = (have == 0x5C) ? 2 : ((have <= 0xFFFF) ? 6 : 12);
if (have_length != want_length) {
RETURN_FAIL("%s: token length: have %" PRIu64 ", want %" PRIu64,
test_cases[tc].str, have_length, want_length);
}
}
}
if (have != test_cases[tc].want) {
RETURN_FAIL("%s: have 0x%" PRIX32 ", want 0x%" PRIX32, test_cases[tc].str,
have, test_cases[tc].want);
}
if (total_length != src.meta.ri) {
RETURN_FAIL("%s: total length: have 0x%" PRIu64 ", want 0x%" PRIu64,
test_cases[tc].str, total_length, src.meta.ri);
}
}
return NULL;
}
// ---------------- Mimic Tests
#ifdef WUFFS_MIMIC
@@ -217,7 +326,8 @@ proc tests[] = {
// as any other place.
test_strconv_parse_number_u64, //
test_wuffs_json_decode_tokens, //
test_wuffs_json_decode_tokens, //
test_wuffs_json_decode_unicode4_escapes, //
#ifdef WUFFS_MIMIC