mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-21 08:47:28 +01:00
Have std/json reject some invalid UTF-8
This commit is contained in:
1 parent
9f0eb266e8
commit
3b17d5b28a
4 files changed
+303
-79
No files matched your search
@@ -12,6 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub status "#bad C0 control code"
|
||||
pub status "#bad UTF-8"
|
||||
pub status "#bad backslash-escape"
|
||||
pub status "#bad input"
|
||||
pub status "#unsupported number length"
|
||||
pub status "#unsupported recursion depth"
|
||||
@@ -62,6 +65,68 @@ pri const lut_backslashes array[256] base.u8 = [
|
||||
// 8 9 A B C D E F
|
||||
]
|
||||
|
||||
// lut_chars helps decode bytes within a string:
|
||||
// - 0x00 is 1-byte UTF-8 (ASCII) but not '"', '\\' or a C0 control code.
|
||||
// - 0x01 is '"'.
|
||||
// - 0x02 is '\\'.
|
||||
// - 0x03 is the start of 2-byte UTF-8.
|
||||
// - 0x04 is the start of 3-byte UTF-8.
|
||||
// - 0x05 is the start of 4-byte UTF-8.
|
||||
// - 0x10 is a UTF-8 tail byte.
|
||||
// - 0x80 is invalid JSON (C0 control codes).
|
||||
// - 0x81 is invalid UTF-8.
|
||||
//
|
||||
// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:
|
||||
// UTF8-1 = %x00-7F
|
||||
// UTF8-2 = %xC2-DF UTF8-tail
|
||||
// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
|
||||
// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
|
||||
// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
|
||||
// %xF4 %x80-8F 2( UTF8-tail )
|
||||
// UTF8-tail = %x80-BF
|
||||
pri const lut_chars array[256] base.u8 = [
|
||||
// 0 1 2 3 4 5 6 7
|
||||
// 8 9 A B C D E F
|
||||
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 ..= 0x07. C0 control codes.
|
||||
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 ..= 0x0F. C0 control codes.
|
||||
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 ..= 0x17. C0 control codes.
|
||||
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 ..= 0x1F. C0 control codes.
|
||||
0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. UTF-8-1; '"'.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. UTF-8-1.
|
||||
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. UTF-8-1; '\\'.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. UTF-8-1.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. UTF-8-1.
|
||||
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x80 ..= 0x87. UTF-8 tail.
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x88 ..= 0x8F. UTF-8 tail.
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x90 ..= 0x97. UTF-8 tail.
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x98 ..= 0x9F. UTF-8 tail.
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA0 ..= 0xA7. UTF-8 tail.
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA8 ..= 0xAF. UTF-8 tail.
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB0 ..= 0xB7. UTF-8 tail.
|
||||
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB8 ..= 0xBF. UTF-8 tail.
|
||||
|
||||
0x81, 0x81, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2.
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC8 ..= 0xCF. UTF-8-2.
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD0 ..= 0xD7. UTF-8-2.
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD8 ..= 0xDF. UTF-8-2.
|
||||
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE0 ..= 0xE7. UTF-8-3.
|
||||
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE8 ..= 0xEF. UTF-8-3.
|
||||
0x05, 0x05, 0x05, 0x05, 0x05, 0x81, 0x81, 0x81, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8.
|
||||
0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, // 0xF8 ..= 0xFF. Invalid UTF-8.
|
||||
// 0 1 2 3 4 5 6 7
|
||||
// 8 9 A B C D E F
|
||||
]
|
||||
|
||||
// lut_classes is:
|
||||
// - 0x00 (bitmask 0x0001) is CLASS_WHITESPACE.
|
||||
// - 0x01 (bitmask 0x0002) is CLASS_STRING.
|
||||
// - 0x02 (bitmask 0x0004) is CLASS_COMMA.
|
||||
|
||||
+55
-33
@@ -55,8 +55,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
var stack_bit : base.u32[..= 31]
|
||||
var match : base.u32[..= 2]
|
||||
var c : base.u8
|
||||
var class : base.u8[..= 0x0F]
|
||||
var backslash : base.u8
|
||||
var char : base.u8
|
||||
var class : base.u8[..= 0x0F]
|
||||
|
||||
var uni4_ok : base.u8
|
||||
var uni4_string : base.u64
|
||||
@@ -203,26 +204,28 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
continue.string_loop
|
||||
}
|
||||
c = args.src.peek_u8()
|
||||
char = lut_chars[c]
|
||||
|
||||
if c <= 0x22 { // 0x22 is '"'.
|
||||
if c == 0x22 { // 0x22 is '"'.
|
||||
if char == 0x00 { // Non-special ASCII.
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if string_length >= 0xFFFE {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0220,
|
||||
length: string_length as base.u64)
|
||||
value: 0x20_0221,
|
||||
length: 0xFFFF)
|
||||
string_length = 0
|
||||
break.string_loop
|
||||
|
||||
} else if c < 0x20 { // 0x20 is ' '.
|
||||
if string_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: string_length as base.u64)
|
||||
string_length = 0
|
||||
}
|
||||
return "#bad input"
|
||||
continue.string_loop
|
||||
}
|
||||
string_length += 1
|
||||
continue
|
||||
|
||||
} else if c == 0x5C { // 0x5C is '\\'.
|
||||
} else if char == 0x01 { // '"'
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0220,
|
||||
length: string_length as base.u64)
|
||||
string_length = 0
|
||||
break.string_loop
|
||||
|
||||
} else if char == 0x02 { // '\\'.
|
||||
if string_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
@@ -236,29 +239,31 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
if args.src.available() < 2 {
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
return "#bad backslash-escape"
|
||||
}
|
||||
yield? base."$short read"
|
||||
string_length = 0
|
||||
char = 0
|
||||
continue.string_loop
|
||||
}
|
||||
backslash = (args.src.peek_u16le() >> 8) as base.u8
|
||||
c = lut_backslashes[backslash]
|
||||
if c > 0 {
|
||||
c = (args.src.peek_u16le() >> 8) as base.u8
|
||||
backslash = lut_backslashes[c]
|
||||
if backslash > 0 {
|
||||
args.src.skip32_fast!(actual: 2, worst_case: 2)
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x40_0000 | ((c & 0x7F) as base.u64),
|
||||
value: 0x40_0000 | ((backslash & 0x7F) as base.u64),
|
||||
length: 2)
|
||||
continue.string_loop
|
||||
|
||||
} else if backslash == 0x75 { // 0x75 is 'u'.
|
||||
} else if c == 0x75 { // 0x75 is 'u'.
|
||||
// -------- BEGIN backslash-u.
|
||||
if args.src.available() < 6 {
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
return "#bad backslash-escape"
|
||||
}
|
||||
yield? base."$short read"
|
||||
string_length = 0
|
||||
char = 0
|
||||
continue.string_loop
|
||||
}
|
||||
|
||||
@@ -281,7 +286,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
if uni4_ok == 0 {
|
||||
// It wasn't 4 hexadecimal digits. No-op
|
||||
// (and fall through to "#bad input").
|
||||
// (and fall through to "#bad
|
||||
// backslash-escape").
|
||||
|
||||
} else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) {
|
||||
// Not a Unicode surrogate. We're good.
|
||||
@@ -293,7 +299,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
} else if uni4_value >= 0xDC00 {
|
||||
// Low surrogate. No-op (and fall through
|
||||
// to "#bad input").
|
||||
// to "#bad backslash-escape").
|
||||
|
||||
} else {
|
||||
// High surrogate, which needs to be
|
||||
@@ -302,11 +308,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
// high surrogate. We need another 6.
|
||||
if args.src.available() < 12 {
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
return "#bad backslash-escape"
|
||||
}
|
||||
yield? base."$short read"
|
||||
string_length = 0
|
||||
uni4_value = 0
|
||||
char = 0
|
||||
continue.string_loop
|
||||
}
|
||||
|
||||
@@ -356,7 +363,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
} else {
|
||||
// Roll back the 4 bytes, then fall
|
||||
// through to "#bad input".
|
||||
// through to "#invalid
|
||||
// backslash-escape".
|
||||
uni4_rollback = 4
|
||||
while uni4_rollback > 0 {
|
||||
uni4_rollback -= 1
|
||||
@@ -371,18 +379,32 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
// -------- END backslash-u.
|
||||
}
|
||||
|
||||
return "#bad input"
|
||||
return "#bad backslash-escape"
|
||||
|
||||
} else if char <= 0x10 {
|
||||
// TODO: reject invalid UTF-8.
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if string_length >= 0xFFFE {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: 0xFFFF)
|
||||
string_length = 0
|
||||
continue.string_loop
|
||||
}
|
||||
string_length += 1
|
||||
continue
|
||||
}
|
||||
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if string_length >= 0xFFFE {
|
||||
if string_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: 0xFFFF)
|
||||
length: string_length as base.u64)
|
||||
string_length = 0
|
||||
continue.string_loop
|
||||
}
|
||||
string_length += 1
|
||||
if char == 0x80 {
|
||||
return "#bad C0 control code"
|
||||
}
|
||||
return "#bad UTF-8"
|
||||
}
|
||||
} endwhile.string_loop
|
||||
|
||||
|
||||
Reference in new issue
Block a user