mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 16:47:28 +01:00
Have std/json decode Unicode surrogate pairs
This commit is contained in:
1 parent
7b199116b0
commit
16b0c4652a
4 files changed
+305
-20
No files matched your search
@@ -59,9 +59,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
var c : base.u8
|
||||
var class : base.u8[..= 0x0F]
|
||||
var backslash : base.u8
|
||||
var uni4_ok : base.u8
|
||||
var uni4_string : base.u32
|
||||
var uni4_value : base.u32[..= 0xFFFF]
|
||||
|
||||
var uni4_ok : base.u8
|
||||
var uni4_string : base.u64
|
||||
var uni4_value : base.u32[..= 0xFFFF]
|
||||
var uni4_high_surrogate : base.u32[..= 0x10_FC00]
|
||||
var uni4_rollback : base.u32
|
||||
|
||||
// expect is a bitmask of what the next character class can be.
|
||||
//
|
||||
@@ -250,6 +253,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
continue.string_loop
|
||||
|
||||
} else if backslash == 0x75 { // 0x75 is 'u'.
|
||||
// -------- BEGIN backslash-u.
|
||||
if args.src.available() < 6 {
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
@@ -258,9 +262,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
string_length = 0
|
||||
continue.string_loop
|
||||
}
|
||||
uni4_string = (args.src.peek_u48le_as_u64() >> 16) as base.u32
|
||||
|
||||
uni4_string = args.src.peek_u48le_as_u64() >> 16
|
||||
uni4_value = 0
|
||||
uni4_ok = 0x80
|
||||
|
||||
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
|
||||
uni4_ok &= c
|
||||
uni4_value |= ((c & 0x0F) as base.u32) << 12
|
||||
@@ -273,15 +279,99 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
|
||||
uni4_ok &= c
|
||||
uni4_value |= ((c & 0x0F) as base.u32) << 0
|
||||
if uni4_ok <> 0 {
|
||||
|
||||
if uni4_ok == 0 {
|
||||
// It wasn't 4 hexadecimal digits. No-op
|
||||
// (and fall through to "#bad input").
|
||||
|
||||
} else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) {
|
||||
// Not a Unicode surrogate. We're good.
|
||||
args.src.skip32_fast!(actual: 6, worst_case: 6)
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_0000 | (uni4_value as base.u64),
|
||||
length: 6)
|
||||
continue.string_loop
|
||||
}
|
||||
|
||||
} else if uni4_value >= 0xDC00 {
|
||||
// Low surrogate. No-op (and fall through
|
||||
// to "#bad input").
|
||||
|
||||
} else {
|
||||
// High surrogate, which needs to be
|
||||
// followed by a "\\u1234" low surrogate.
|
||||
// We've already peeked 6 bytes for the
|
||||
// high surrogate. We need another 6.
|
||||
if args.src.available() < 12 {
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
}
|
||||
yield? base."$short read"
|
||||
string_length = 0
|
||||
uni4_value = 0
|
||||
continue.string_loop
|
||||
}
|
||||
|
||||
// Roll forward 4 bytes, so that calling
|
||||
// peek_u64le will pick up the last 8.
|
||||
args.src.skip32_fast!(actual: 4, worst_case: 4)
|
||||
|
||||
uni4_string = args.src.peek_u64le() >> 16
|
||||
|
||||
// Look for the low surrogate's "\\u".
|
||||
if ((0xFF & (uni4_string >> 0)) <> 0x5C) or
|
||||
((0xFF & (uni4_string >> 8)) <> 0x75) {
|
||||
uni4_high_surrogate = 0
|
||||
uni4_value = 0
|
||||
uni4_ok = 0
|
||||
} else {
|
||||
uni4_high_surrogate =
|
||||
0x1_0000 + ((uni4_value - 0xD800) << 10)
|
||||
uni4_value = 0
|
||||
uni4_ok = 0x80
|
||||
uni4_string >>= 16
|
||||
|
||||
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
|
||||
uni4_ok &= c
|
||||
uni4_value |= ((c & 0x0F) as base.u32) << 12
|
||||
c = hexadecimal_digits[0xFF & (uni4_string >> 8)]
|
||||
uni4_ok &= c
|
||||
uni4_value |= ((c & 0x0F) as base.u32) << 8
|
||||
c = hexadecimal_digits[0xFF & (uni4_string >> 16)]
|
||||
uni4_ok &= c
|
||||
uni4_value |= ((c & 0x0F) as base.u32) << 4
|
||||
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
|
||||
uni4_ok &= c
|
||||
uni4_value |= ((c & 0x0F) as base.u32) << 0
|
||||
}
|
||||
|
||||
if (uni4_ok <> 0) and
|
||||
(0xDC00 <= uni4_value) and (uni4_value <= 0xDFFF) {
|
||||
|
||||
// Emit a single token for the surrogate pair.
|
||||
uni4_value -= 0xDC00
|
||||
args.src.skip32_fast!(actual: 8, worst_case: 8)
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_0000 | ((uni4_high_surrogate + uni4_value) as base.u64),
|
||||
length: 12)
|
||||
continue.string_loop
|
||||
|
||||
} else {
|
||||
// Roll back the 4 bytes, then fall
|
||||
// through to "#bad input".
|
||||
uni4_rollback = 4
|
||||
while uni4_rollback > 0 {
|
||||
uni4_rollback -= 1
|
||||
if args.src.can_undo_byte() {
|
||||
args.src.undo_byte!()
|
||||
} else {
|
||||
return "#internal error: inconsistent I/O"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// -------- END backslash-u.
|
||||
}
|
||||
|
||||
return "#bad input"
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user