Have std/json reject some invalid UTF-8

This commit is contained in:
Nigel Tao committed 2020-02-26 14:36:23 +11:00
1 parent 9f0eb266e8
commit 3b17d5b28a
4 files changed
+303 -79

No files matched your search

+65
View File
@@ -12,6 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub status "#bad C0 control code"
pub status "#bad UTF-8"
pub status "#bad backslash-escape"
pub status "#bad input"
pub status "#unsupported number length"
pub status "#unsupported recursion depth"
@@ -62,6 +65,68 @@ pri const lut_backslashes array[256] base.u8 = [
// 8 9 A B C D E F
]
// lut_chars helps decode bytes within a string:
// - 0x00 is 1-byte UTF-8 (ASCII) but not '"', '\\' or a C0 control code.
// - 0x01 is '"'.
// - 0x02 is '\\'.
// - 0x03 is the start of 2-byte UTF-8.
// - 0x04 is the start of 3-byte UTF-8.
// - 0x05 is the start of 4-byte UTF-8.
// - 0x10 is a UTF-8 tail byte.
// - 0x80 is invalid JSON (C0 control codes).
// - 0x81 is invalid UTF-8.
//
// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:
// UTF8-1 = %x00-7F
// UTF8-2 = %xC2-DF UTF8-tail
// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
// %xF4 %x80-8F 2( UTF8-tail )
// UTF8-tail = %x80-BF
pri const lut_chars array[256] base.u8 = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 ..= 0x07. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 ..= 0x0F. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 ..= 0x17. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 ..= 0x1F. C0 control codes.
0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. UTF-8-1; '"'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. UTF-8-1; '\\'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. UTF-8-1.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x80 ..= 0x87. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x88 ..= 0x8F. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x90 ..= 0x97. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x98 ..= 0x9F. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA0 ..= 0xA7. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA8 ..= 0xAF. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB0 ..= 0xB7. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB8 ..= 0xBF. UTF-8 tail.
0x81, 0x81, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC8 ..= 0xCF. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD0 ..= 0xD7. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD8 ..= 0xDF. UTF-8-2.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE0 ..= 0xE7. UTF-8-3.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE8 ..= 0xEF. UTF-8-3.
0x05, 0x05, 0x05, 0x05, 0x05, 0x81, 0x81, 0x81, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8.
0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, // 0xF8 ..= 0xFF. Invalid UTF-8.
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
]
// lut_classes is:
// - 0x00 (bitmask 0x0001) is CLASS_WHITESPACE.
// - 0x01 (bitmask 0x0002) is CLASS_STRING.
// - 0x02 (bitmask 0x0004) is CLASS_COMMA.
+55 -33
View File
@@ -55,8 +55,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var stack_bit : base.u32[..= 31]
var match : base.u32[..= 2]
var c : base.u8
var class : base.u8[..= 0x0F]
var backslash : base.u8
var char : base.u8
var class : base.u8[..= 0x0F]
var uni4_ok : base.u8
var uni4_string : base.u64
@@ -203,26 +204,28 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.string_loop
}
c = args.src.peek_u8()
char = lut_chars[c]
if c <= 0x22 { // 0x22 is '"'.
if c == 0x22 { // 0x22 is '"'.
if char == 0x00 { // Non-special ASCII.
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
args.dst.write_fast_token!(
value: 0x20_0220,
length: string_length as base.u64)
value: 0x20_0221,
length: 0xFFFF)
string_length = 0
break.string_loop
} else if c < 0x20 { // 0x20 is ' '.
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x20_0221,
length: string_length as base.u64)
string_length = 0
}
return "#bad input"
continue.string_loop
}
string_length += 1
continue
} else if c == 0x5C { // 0x5C is '\\'.
} else if char == 0x01 { // '"'
args.dst.write_fast_token!(
value: 0x20_0220,
length: string_length as base.u64)
string_length = 0
break.string_loop
} else if char == 0x02 { // '\\'.
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x20_0221,
@@ -236,29 +239,31 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
if args.src.available() < 2 {
if args.src.is_closed() {
return "#bad input"
return "#bad backslash-escape"
}
yield? base."$short read"
string_length = 0
char = 0
continue.string_loop
}
backslash = (args.src.peek_u16le() >> 8) as base.u8
c = lut_backslashes[backslash]
if c > 0 {
c = (args.src.peek_u16le() >> 8) as base.u8
backslash = lut_backslashes[c]
if backslash > 0 {
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value: 0x40_0000 | ((c & 0x7F) as base.u64),
value: 0x40_0000 | ((backslash & 0x7F) as base.u64),
length: 2)
continue.string_loop
} else if backslash == 0x75 { // 0x75 is 'u'.
} else if c == 0x75 { // 0x75 is 'u'.
// -------- BEGIN backslash-u.
if args.src.available() < 6 {
if args.src.is_closed() {
return "#bad input"
return "#bad backslash-escape"
}
yield? base."$short read"
string_length = 0
char = 0
continue.string_loop
}
@@ -281,7 +286,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
if uni4_ok == 0 {
// It wasn't 4 hexadecimal digits. No-op
// (and fall through to "#bad input").
// (and fall through to "#bad
// backslash-escape").
} else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) {
// Not a Unicode surrogate. We're good.
@@ -293,7 +299,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
} else if uni4_value >= 0xDC00 {
// Low surrogate. No-op (and fall through
// to "#bad input").
// to "#bad backslash-escape").
} else {
// High surrogate, which needs to be
@@ -302,11 +308,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// high surrogate. We need another 6.
if args.src.available() < 12 {
if args.src.is_closed() {
return "#bad input"
return "#bad backslash-escape"
}
yield? base."$short read"
string_length = 0
uni4_value = 0
char = 0
continue.string_loop
}
@@ -356,7 +363,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
} else {
// Roll back the 4 bytes, then fall
// through to "#bad input".
// through to "#invalid
// backslash-escape".
uni4_rollback = 4
while uni4_rollback > 0 {
uni4_rollback -= 1
@@ -371,18 +379,32 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// -------- END backslash-u.
}
return "#bad input"
return "#bad backslash-escape"
} else if char <= 0x10 {
// TODO: reject invalid UTF-8.
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
args.dst.write_fast_token!(
value: 0x20_0221,
length: 0xFFFF)
string_length = 0
continue.string_loop
}
string_length += 1
continue
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x20_0221,
length: 0xFFFF)
length: string_length as base.u64)
string_length = 0
continue.string_loop
}
string_length += 1
if char == 0x80 {
return "#bad C0 control code"
}
return "#bad UTF-8"
}
} endwhile.string_loop