diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3a8bc179..e59aa6da 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -20468,6 +20468,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, uint64_t v_uni4_string = 0; uint32_t v_uni4_value = 0; uint32_t v_uni4_high_surrogate = 0; + uint8_t v_uni8_ok = 0; + uint64_t v_uni8_string = 0; + uint32_t v_uni8_value = 0; uint32_t v_expect = 0; uint32_t v_expect_after_value = 0; @@ -20864,6 +20867,79 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } + } else if ((v_c == 85) && + self->private_impl + .f_quirk_enabled_allow_backslash_capital_u) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 10) { + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status( + wuffs_json__error__bad_backslash_escape); + goto exit; + } + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9); + v_string_length = 0; + v_char = 0; + goto label__string_loop_outer__continue; + } + v_uni8_string = + wuffs_base__load_u64le__no_bounds_check(iop_a_src + 2); + v_uni8_value = 0; + v_uni8_ok = 128; + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 0))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 28); + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 8))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 24); + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 16))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 20); + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 24))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 16); + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 32))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 12); + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 40))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 8); + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 48))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 4); + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_uni8_string >> 56))]; + v_uni8_ok &= v_c; + v_uni8_value |= (((uint32_t)((v_c & 15))) << 0); + if (v_uni8_ok == 0) { + } else if ((v_uni8_value < 55296) || + ((57343 < v_uni8_value) && + (v_uni8_value <= 1114111))) { + (iop_a_src += 10, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((6291456 | (v_uni8_value & 2097151)))) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; + } else if (self->private_impl + .f_quirk_enabled_replace_invalid_utf_8) { + (iop_a_src += 10, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; + } } status = wuffs_base__make_status( wuffs_json__error__bad_backslash_escape); @@ -20900,7 +20976,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status( wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); v_string_length = 0; v_char = 0; goto label__string_loop_outer__continue; @@ -20956,7 +21032,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status( wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); v_string_length = 0; v_char = 0; goto label__string_loop_outer__continue; @@ -21017,7 +21093,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status( wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); v_string_length = 0; v_char = 0; goto label__string_loop_outer__continue; @@ -21087,13 +21163,13 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); goto label__1__continue; } if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); goto label__1__continue; } (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -21171,11 +21247,11 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status( wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); } } } @@ -21309,7 +21385,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17); goto label__outer__continue; } } else if (v_class == 10) { @@ -21330,7 +21406,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18); goto label__outer__continue; } } else if (v_class == 11) { @@ -21351,7 +21427,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19); goto label__outer__continue; } } @@ -21372,7 +21448,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(19); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(20); status = wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src); if (a_dst) { diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 51edc4fe..017e6b20 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -122,6 +122,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var uni4_value : base.u32[..= 0xFFFF] var uni4_high_surrogate : base.u32[..= 0x10_FC00] + var uni8_ok : base.u8 + var uni8_string : base.u64 + var uni8_value : base.u32[..= 0xFFFF_FFFF] + // expect is a bitmask of what the next character class can be. // // expect_after_value is what to expect after seeing a value (a literal, @@ -483,6 +487,73 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { continue.string_loop_outer } // -------- END backslash-u. + + } else if (c == 0x55) and + this.quirk_enabled_allow_backslash_capital_u { // 0x55 is 'U'. + // -------- BEGIN backslash-capital-u. + if args.src.available() < 10 { + if args.src.is_closed() { + return "#bad backslash-escape" + } + yield? base."$short read" + string_length = 0 + char = 0 + continue.string_loop_outer + } + uni8_string = args.src.peek_u64le_at(offset: 2) + uni8_value = 0 + uni8_ok = 0x80 + + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 0)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 28 + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 8)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 24 + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 16)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 20 + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 24)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 16 + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 32)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 12 + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 40)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 8 + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 48)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 4 + c = lut_hexadecimal_digits[0xFF & (uni8_string >> 56)] + uni8_ok &= c + uni8_value |= ((c & 0x0F) as base.u32) << 0 + + if uni8_ok == 0 { + // It wasn't 8 hexadecimal digits. No-op + // (and fall through to "#bad + // backslash-escape"). + + } else if (uni8_value < 0xD800) or ( + (0xDFFF < uni8_value) and (uni8_value <= 0x10_FFFF)) { + // Not a Unicode surrogate. We're good. + args.src.skip32_fast!(actual: 10, worst_case: 10) + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_0000 | (uni8_value & 0x1F_FFFF), + link: 0x3, + length: 10) + continue.string_loop_outer + } else if this.quirk_enabled_replace_invalid_utf_8 { + args.src.skip32_fast!(actual: 10, worst_case: 10) + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_FFFD, + link: 0x3, + length: 6) + continue.string_loop_outer + } + // -------- END backslash-capital-u. } return "#bad backslash-escape" diff --git a/std/json/decode_quirks.wuffs b/std/json/decode_quirks.wuffs index c354505b..4daf3104 100644 --- a/std/json/decode_quirks.wuffs +++ b/std/json/decode_quirks.wuffs @@ -28,6 +28,8 @@ pub const quirk_allow_backslash_a base.u32 = 0x4909_9400 | 0x00 // When this quirk is enabled, e.g. "abc\U0001F4A9z" is accepted as a JSON // string, equivalent to "abc\uD83D\uDCA9z", containing the U+0001F4A9 PILE OF // POO Unicode code point. There are exactly 8 encoded bytes after each "\U". +// +// This quirk can combine with quirk_replace_invalid_utf_8. pub const quirk_allow_backslash_capital_u base.u32 = 0x4909_9400 | 0x01 // When this quirk is enabled, e.g. "abc\ez" is accepted as a JSON string, @@ -51,7 +53,9 @@ pub const quirk_allow_backslash_v base.u32 = 0x4909_9400 | 0x05 // decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A. There are exactly 2 // encoded bytes after each "\x". // -// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8. +// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8 and +// even if the overall byte string is valid UTF-8, tokens are also no longer +// guaranteed to split on UTF-8 boundaries. // // "\x", "\x9", "\x9$" and "\X99" are all still rejected. pub const quirk_allow_backslash_x base.u32 = 0x4909_9400 | 0x06 @@ -149,10 +153,16 @@ pub const quirk_allow_trailing_new_line base.u32 = 0x4909_9400 | 0x0E // When this quirk is enabled, invalid UTF-8 inside a JSON string is accepted. // Each byte of invalid UTF-8 is equivalent to "\uFFFD", the Unicode -// Replacement Character. Invalid UTF-8 outside a JSON string remains an error. +// Replacement Character. The UTF-8 encoding of U+FFFD is "\xEF\xBF\xBD". +// +// Invalid UTF-8 outside a JSON string remains an error. // // Similarly, for backslash-u escapes featuring incorrectly paired Unicode // surrogates, each backslash-u 6-byte unit is replaced. For example, // "abc\uDC00z" and "ijk\uD800\uDBFFz" are equivalent to "abc\uFFFDz" and // "ijk\uFFFD\uFFFDz". +// +// When combined with quirk_allow_backslash_capital_u, a "\U12345678" 10-byte +// unit that is an invalid Unicode code point (i.e. in the range U+D800 ..= +// U+DFFF or above U+10FFFF) is similarly replaced with U+FFFD. pub const quirk_replace_invalid_utf_8 base.u32 = 0x4909_9400 | 0x0F diff --git a/test/c/std/json.c b/test/c/std/json.c index eb5d018d..9de587de 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -1119,6 +1119,11 @@ test_wuffs_json_decode_quirk_allow_backslash_etc() { .str = "\"\\a\"", .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_A, }, + { + .want = 0x0001F4A9, + .str = "\"\\U0001F4A9\"", + .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_CAPITAL_U, + }, { .want = 0x1B, .str = "\"\\e\"",