From 550ef90114fd63c64995a610d1fa5699918393db Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Wed, 26 Feb 2020 16:01:04 +1100 Subject: [PATCH] Have std/json reject invalid UTF-8 --- release/c/wuffs-unsupported-snapshot.c | 131 +++++++++++++++++++++---- std/json/decode_json.wuffs | 99 ++++++++++++++++--- test/c/std/json.c | 22 ++++- 3 files changed, 220 insertions(+), 32 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index ba8c0573..77efc361 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -18940,6 +18940,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, uint8_t v_backslash = 0; uint8_t v_char = 0; uint8_t v_class = 0; + uint32_t v_multi_byte_utf8 = 0; uint8_t v_uni4_ok = 0; uint64_t v_uni4_string = 0; uint32_t v_uni4_value = 0; @@ -19110,11 +19111,11 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, v_char = wuffs_json__lut_chars[v_c]; if (v_char == 0) { (iop_a_src += 1, wuffs_base__make_empty_struct()); - if (v_string_length >= 65534) { + if (v_string_length >= 65531) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | - (((uint64_t)(65535)) + (((uint64_t)(65532)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; goto label__string_loop__continue; @@ -19281,19 +19282,111 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, status = wuffs_base__make_status( wuffs_json__error__bad_backslash_escape); goto exit; - } else if (v_char <= 16) { - (iop_a_src += 1, wuffs_base__make_empty_struct()); - if (v_string_length >= 65534) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097697)) - << WUFFS_BASE__TOKEN__VALUE__SHIFT) | - (((uint64_t)(65535)) - << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + } else if (v_char == 3) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 2) { + if (a_src && a_src->meta.closed) { + status = + wuffs_base__make_status(wuffs_json__error__bad_utf_8); + goto exit; + } + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); v_string_length = 0; + v_char = 0; goto label__string_loop__continue; } - v_string_length += 1; - goto label__1__continue; + v_multi_byte_utf8 = + ((uint32_t)(wuffs_base__load_u16le(iop_a_src))); + if ((v_multi_byte_utf8 & 49152) == 32768) { + v_multi_byte_utf8 = ((1984 & (v_multi_byte_utf8 << 6)) | + (63 & (v_multi_byte_utf8 >> 8))); + (iop_a_src += 2, wuffs_base__make_empty_struct()); + if (v_string_length >= 65528) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2097697)) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)((v_string_length + 2))))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + goto label__string_loop__continue; + } + v_string_length += 2; + goto label__1__continue; + } + } else if (v_char == 4) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 3) { + if (a_src && a_src->meta.closed) { + status = + wuffs_base__make_status(wuffs_json__error__bad_utf_8); + goto exit; + } + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); + v_string_length = 0; + v_char = 0; + goto label__string_loop__continue; + } + v_multi_byte_utf8 = + ((uint32_t)(wuffs_base__load_u24le(iop_a_src))); + if ((v_multi_byte_utf8 & 12632064) == 8421376) { + v_multi_byte_utf8 = ((61440 & (v_multi_byte_utf8 << 12)) | + (4032 & (v_multi_byte_utf8 >> 2)) | + (63 & (v_multi_byte_utf8 >> 16))); + if ((2047 < v_multi_byte_utf8) && + ((v_multi_byte_utf8 < 55296) || + (57343 < v_multi_byte_utf8))) { + (iop_a_src += 3, wuffs_base__make_empty_struct()); + if (v_string_length >= 65528) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2097697)) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)((v_string_length + 3))))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + goto label__string_loop__continue; + } + v_string_length += 3; + goto label__1__continue; + } + } + } else if (v_char == 5) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + if (a_src && a_src->meta.closed) { + status = + wuffs_base__make_status(wuffs_json__error__bad_utf_8); + goto exit; + } + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); + v_string_length = 0; + v_char = 0; + goto label__string_loop__continue; + } + v_multi_byte_utf8 = wuffs_base__load_u32le(iop_a_src); + if ((v_multi_byte_utf8 & 3233857536) == 2155905024) { + v_multi_byte_utf8 = ((1835008 & (v_multi_byte_utf8 << 18)) | + (258048 & (v_multi_byte_utf8 << 4)) | + (4032 & (v_multi_byte_utf8 >> 10)) | + (63 & (v_multi_byte_utf8 >> 24))); + if ((65535 < v_multi_byte_utf8) && + (v_multi_byte_utf8 <= 1114111)) { + (iop_a_src += 4, wuffs_base__make_empty_struct()); + if (v_string_length >= 65528) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2097697)) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)((v_string_length + 4))))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + goto label__string_loop__continue; + } + v_string_length += 4; + goto label__1__continue; + } + } } if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( @@ -19321,13 +19414,13 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); goto label__2__continue; } if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); goto label__2__continue; } (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -19403,11 +19496,11 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status( wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); } } } @@ -19532,7 +19625,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17); goto label__outer__continue; } } else if (v_class == 10) { @@ -19552,7 +19645,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18); goto label__outer__continue; } } else if (v_class == 11) { @@ -19572,7 +19665,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19); goto label__outer__continue; } } diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 64746cfb..959589e8 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -48,7 +48,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var token_value : base.u64[..= 0xFF_FFFF] var number_length : base.u32[..= 0x3FF] var number_status : base.u32[..= 0x3] - var string_length : base.u32[..= 0xFFFE] + var string_length : base.u32[..= 0xFFFB] var whitespace_length : base.u32[..= 0xFFFE] var depth : base.u32[..= 1024] var stack_byte : base.u32[..= (1024 / 32) - 1] @@ -58,6 +58,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var backslash : base.u8 var char : base.u8 var class : base.u8[..= 0x0F] + var multi_byte_utf8 : base.u32 var uni4_ok : base.u8 var uni4_string : base.u64 @@ -208,10 +209,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if char == 0x00 { // Non-special ASCII. args.src.skip32_fast!(actual: 1, worst_case: 1) - if string_length >= 0xFFFE { + if string_length >= 0xFFFB { args.dst.write_fast_token!( value: 0x20_0221, - length: 0xFFFF) + length: 0xFFFC) string_length = 0 continue.string_loop } @@ -381,18 +382,92 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { return "#bad backslash-escape" - } else if char <= 0x10 { - // TODO: reject invalid UTF-8. - args.src.skip32_fast!(actual: 1, worst_case: 1) - if string_length >= 0xFFFE { - args.dst.write_fast_token!( - value: 0x20_0221, - length: 0xFFFF) + } else if char == 0x03 { // 2-byte UTF-8. + if args.src.available() < 2 { + if args.src.is_closed() { + return "#bad UTF-8" + } + yield? base."$short read" string_length = 0 + char = 0 continue.string_loop } - string_length += 1 - continue + multi_byte_utf8 = args.src.peek_u16le_as_u32() + if (multi_byte_utf8 & 0xC000) == 0x8000 { + multi_byte_utf8 = (0x00_07C0 & (multi_byte_utf8 ~mod<< 6)) | + (0x00_003F & (multi_byte_utf8 >> 8)) + args.src.skip32_fast!(actual: 2, worst_case: 2) + if string_length >= 0xFFF8 { + args.dst.write_fast_token!( + value: 0x20_0221, + length: (string_length + 2) as base.u64) + string_length = 0 + continue.string_loop + } + string_length += 2 + continue + } + + } else if char == 0x04 { // 3-byte UTF-8. + if args.src.available() < 3 { + if args.src.is_closed() { + return "#bad UTF-8" + } + yield? base."$short read" + string_length = 0 + char = 0 + continue.string_loop + } + multi_byte_utf8 = args.src.peek_u24le_as_u32() + if (multi_byte_utf8 & 0xC0_C000) == 0x80_8000 { + multi_byte_utf8 = (0x00_F000 & (multi_byte_utf8 ~mod<< 12)) | + (0x00_0FC0 & (multi_byte_utf8 >> 2)) | + (0x00_003F & (multi_byte_utf8 >> 16)) + if (0x07FF < multi_byte_utf8) and + ((multi_byte_utf8 < 0xD800) or (0xDFFF < multi_byte_utf8)) { + + args.src.skip32_fast!(actual: 3, worst_case: 3) + if string_length >= 0xFFF8 { + args.dst.write_fast_token!( + value: 0x20_0221, + length: (string_length + 3) as base.u64) + string_length = 0 + continue.string_loop + } + string_length += 3 + continue + } + } + + } else if char == 0x05 { // 4-byte UTF-8. + if args.src.available() < 4 { + if args.src.is_closed() { + return "#bad UTF-8" + } + yield? base."$short read" + string_length = 0 + char = 0 + continue.string_loop + } + multi_byte_utf8 = args.src.peek_u32le() + if (multi_byte_utf8 & 0xC0C0_C000) == 0x8080_8000 { + multi_byte_utf8 = (0x1C_0000 & (multi_byte_utf8 ~mod<< 18)) | + (0x03_F000 & (multi_byte_utf8 ~mod<< 4)) | + (0x00_0FC0 & (multi_byte_utf8 >> 10)) | + (0x00_003F & (multi_byte_utf8 >> 24)) + if (0xFFFF < multi_byte_utf8) and (multi_byte_utf8 <= 0x10_FFFF) { + args.src.skip32_fast!(actual: 4, worst_case: 4) + if string_length >= 0xFFF8 { + args.dst.write_fast_token!( + value: 0x20_0221, + length: (string_length + 4) as base.u64) + string_length = 0 + continue.string_loop + } + string_length += 4 + continue + } + } } if string_length > 0 { diff --git a/test/c/std/json.c b/test/c/std/json.c index 8e522622..d913d119 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -401,10 +401,15 @@ test_wuffs_json_decode_string() { {.want_status_repr = NULL, .str = "\"\xC2\x80\""}, // U+00000080. {.want_status_repr = NULL, .str = "\"\xDF\xBF\""}, // U+000007FF. {.want_status_repr = NULL, .str = "\"\xE0\xA0\x80\""}, // U+00000800. - {.want_status_repr = NULL, .str = "\"\xED\xAF\xBF\""}, // U+0000DBFF. + {.want_status_repr = NULL, .str = "\"\xED\x80\x80\""}, // U+0000D000. + {.want_status_repr = NULL, .str = "\"\xED\x9F\xBF\""}, // U+0000D7FF. {.want_status_repr = NULL, .str = "\"\xEE\x80\x80\""}, // U+0000E000. {.want_status_repr = NULL, .str = "\"\xEF\xBF\xBF\""}, // U+0000FFFF. {.want_status_repr = NULL, .str = "\"\xF0\x90\x80\x80\""}, // U+00010000. + {.want_status_repr = NULL, .str = "\"\xF0\xB0\x80\x81\""}, // U+00030001. + {.want_status_repr = NULL, .str = "\"\xF1\xB0\x80\x82\""}, // U+00070002. + {.want_status_repr = NULL, .str = "\"\xF3\xB0\x80\x83\""}, // U+000F0003. + {.want_status_repr = NULL, .str = "\"\xF4\x80\x80\x84\""}, // U+00100004. {.want_status_repr = NULL, .str = "\"\xF4\x8F\xBF\xBF\""}, // U+0010FFFF. {.want_status_repr = NULL, .str = "\"abc\""}, {.want_status_repr = NULL, .str = "\"i\x6Ak\""}, @@ -418,6 +423,21 @@ test_wuffs_json_decode_string() { {.want_status_repr = bad_ccc, .str = "\"\x1F\""}, {.want_status_repr = bad_ccc, .str = "\"tab+\t+tab\""}, + {.want_status_repr = bad_utf, .str = "\"\x80\""}, + {.want_status_repr = bad_utf, .str = "\"\xBF\""}, + {.want_status_repr = bad_utf, .str = "\"\xC1\x80\""}, + {.want_status_repr = bad_utf, .str = "\"\xC2\x7F\""}, + {.want_status_repr = bad_utf, .str = "\"\xDF\xC0\""}, + {.want_status_repr = bad_utf, .str = "\"\xDF\xFF\""}, + {.want_status_repr = bad_utf, .str = "\"\xE0\x9F\xBF\""}, + {.want_status_repr = bad_utf, .str = "\"\xED\xA0\xB0\""}, // U+0000D800. + {.want_status_repr = bad_utf, .str = "\"\xED\xBF\xBF\""}, // U+0000DFFF. + {.want_status_repr = bad_utf, .str = "\"\xF0\x80\x80\""}, + {.want_status_repr = bad_utf, .str = "\"\xF0\x8F\xBF\xBF\""}, + {.want_status_repr = bad_utf, .str = "\"\xF2\x7F\x80\x80\""}, + {.want_status_repr = bad_utf, .str = "\"\xF2\x80\x7F\x80\""}, + {.want_status_repr = bad_utf, .str = "\"\xF2\x80\x80\x7F\""}, + {.want_status_repr = bad_utf, .str = "\"\xF4\x90\x80\x80\""}, {.want_status_repr = bad_utf, .str = "\"\xF5\""}, {.want_status_repr = bad_utf, .str = "\"\xFF\xFF\xFF\xFF\""}, };