Have std/json decode Unicode surrogate pairs

This commit is contained in:
Nigel Tao committed 2020-02-25 10:45:31 +11:00
1 parent 7b199116b0
commit 16b0c4652a
4 files changed
+305 -20

No files matched your search

+96 -6
View File
@@ -59,9 +59,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var c : base.u8
var class : base.u8[..= 0x0F]
var backslash : base.u8
var uni4_ok : base.u8
var uni4_string : base.u32
var uni4_value : base.u32[..= 0xFFFF]
var uni4_ok : base.u8
var uni4_string : base.u64
var uni4_value : base.u32[..= 0xFFFF]
var uni4_high_surrogate : base.u32[..= 0x10_FC00]
var uni4_rollback : base.u32
// expect is a bitmask of what the next character class can be.
//
@@ -250,6 +253,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.string_loop
} else if backslash == 0x75 { // 0x75 is 'u'.
// -------- BEGIN backslash-u.
if args.src.available() < 6 {
if args.src.is_closed() {
return "#bad input"
@@ -258,9 +262,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
string_length = 0
continue.string_loop
}
uni4_string = (args.src.peek_u48le_as_u64() >> 16) as base.u32
uni4_string = args.src.peek_u48le_as_u64() >> 16
uni4_value = 0
uni4_ok = 0x80
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 12
@@ -273,15 +279,99 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 0
if uni4_ok <> 0 {
if uni4_ok == 0 {
// It wasn't 4 hexadecimal digits. No-op
// (and fall through to "#bad input").
} else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) {
// Not a Unicode surrogate. We're good.
args.src.skip32_fast!(actual: 6, worst_case: 6)
args.dst.write_fast_token!(
value: 0x80_0000 | (uni4_value as base.u64),
length: 6)
continue.string_loop
}
} else if uni4_value >= 0xDC00 {
// Low surrogate. No-op (and fall through
// to "#bad input").
} else {
// High surrogate, which needs to be
// followed by a "\\u1234" low surrogate.
// We've already peeked 6 bytes for the
// high surrogate. We need another 6.
if args.src.available() < 12 {
if args.src.is_closed() {
return "#bad input"
}
yield? base."$short read"
string_length = 0
uni4_value = 0
continue.string_loop
}
// Roll forward 4 bytes, so that calling
// peek_u64le will pick up the last 8.
args.src.skip32_fast!(actual: 4, worst_case: 4)
uni4_string = args.src.peek_u64le() >> 16
// Look for the low surrogate's "\\u".
if ((0xFF & (uni4_string >> 0)) <> 0x5C) or
((0xFF & (uni4_string >> 8)) <> 0x75) {
uni4_high_surrogate = 0
uni4_value = 0
uni4_ok = 0
} else {
uni4_high_surrogate =
0x1_0000 + ((uni4_value - 0xD800) << 10)
uni4_value = 0
uni4_ok = 0x80
uni4_string >>= 16
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 12
c = hexadecimal_digits[0xFF & (uni4_string >> 8)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 8
c = hexadecimal_digits[0xFF & (uni4_string >> 16)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 4
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 0
}
if (uni4_ok <> 0) and
(0xDC00 <= uni4_value) and (uni4_value <= 0xDFFF) {
// Emit a single token for the surrogate pair.
uni4_value -= 0xDC00
args.src.skip32_fast!(actual: 8, worst_case: 8)
args.dst.write_fast_token!(
value: 0x80_0000 | ((uni4_high_surrogate + uni4_value) as base.u64),
length: 12)
continue.string_loop
} else {
// Roll back the 4 bytes, then fall
// through to "#bad input".
uni4_rollback = 4
while uni4_rollback > 0 {
uni4_rollback -= 1
if args.src.can_undo_byte() {
args.src.undo_byte!()
} else {
return "#internal error: inconsistent I/O"
}
}
}
}
// -------- END backslash-u.
}
return "#bad input"
}