diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index 3e667d75..783faa24 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -327,19 +327,31 @@ const char* handle_string(parsed_token pt) { } else if (vbd <= 0x07FF) { // The UTF-8 encoding takes 2 bytes. - uint8_t esc2[6]; + uint8_t esc2[2]; esc2[0] = 0xC0 | (uint8_t)((vbd >> 6)); esc2[1] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F); TRY(write_dst(&esc2[0], 2)); } else if (vbd <= 0xFFFF) { + if ((0xD800 <= vbd) && (vbd <= 0xDFFF)) { + return "main: unexpected Unicode surrogate"; + } // The UTF-8 encoding takes 3 bytes. - uint8_t esc3[6]; + uint8_t esc3[3]; esc3[0] = 0xE0 | (uint8_t)((vbd >> 12)); esc3[1] = 0x80 | (uint8_t)((vbd >> 6) & 0x3F); esc3[2] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F); TRY(write_dst(&esc3[0], 3)); + } else if (vbd <= 0x10FFFF) { + // The UTF-8 encoding takes 4 bytes. + uint8_t esc4[4]; + esc4[0] = 0xF0 | (uint8_t)((vbd >> 18)); + esc4[1] = 0x80 | (uint8_t)((vbd >> 12) & 0x3F); + esc4[2] = 0x80 | (uint8_t)((vbd >> 6) & 0x3F); + esc4[3] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F); + TRY(write_dst(&esc4[0], 4)); + } else { return "main: unexpected Unicode code point"; } diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 25dc723f..79f6ab6c 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -18648,8 +18648,10 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, uint8_t v_class = 0; uint8_t v_backslash = 0; uint8_t v_uni4_ok = 0; - uint32_t v_uni4_string = 0; + uint64_t v_uni4_string = 0; uint32_t v_uni4_value = 0; + uint32_t v_uni4_high_surrogate = 0; + uint32_t v_uni4_rollback = 0; uint32_t v_expect = 0; uint32_t v_expect_after_value = 0; @@ -18880,8 +18882,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, v_string_length = 0; goto label__string_loop__continue; } - v_uni4_string = ((uint32_t)( - (((uint64_t)(wuffs_base__load_u48le(iop_a_src))) >> 16))); + v_uni4_string = + (((uint64_t)(wuffs_base__load_u48le(iop_a_src))) >> 16); v_uni4_value = 0; v_uni4_ok = 128; v_c = wuffs_json__hexadecimal_digits[(255 & @@ -18900,13 +18902,84 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (v_uni4_string >> 24))]; v_uni4_ok &= v_c; v_uni4_value |= (((uint32_t)((v_c & 15))) << 0); - if (v_uni4_ok != 0) { + if (v_uni4_ok == 0) { + } else if ((v_uni4_value < 55296) || (57343 < v_uni4_value)) { (iop_a_src += 6, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)((8388608 | ((uint64_t)(v_uni4_value))))) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop__continue; + } else if (v_uni4_value >= 56320) { + } else { + if (((uint64_t)(io2_a_src - iop_a_src)) < 12) { + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status( + wuffs_json__error__bad_input); + goto exit; + } + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9); + v_string_length = 0; + v_uni4_value = 0; + goto label__string_loop__continue; + } + (iop_a_src += 4, wuffs_base__make_empty_struct()); + v_uni4_string = (wuffs_base__load_u64le(iop_a_src) >> 16); + if (((255 & (v_uni4_string >> 0)) != 92) || + ((255 & (v_uni4_string >> 8)) != 117)) { + v_uni4_high_surrogate = 0; + v_uni4_value = 0; + v_uni4_ok = 0; + } else { + v_uni4_high_surrogate = + (65536 + ((v_uni4_value - 55296) << 10)); + v_uni4_value = 0; + v_uni4_ok = 128; + v_uni4_string >>= 16; + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 0))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 12); + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 8))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 8); + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 16))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 4); + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 24))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 0); + } + if ((v_uni4_ok != 0) && (56320 <= v_uni4_value) && + (v_uni4_value <= 57343)) { + v_uni4_value -= 56320; + (iop_a_src += 8, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)( + (8388608 | ((uint64_t)((v_uni4_high_surrogate + + v_uni4_value)))))) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(12)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop__continue; + } else { + v_uni4_rollback = 4; + while (v_uni4_rollback > 0) { + v_uni4_rollback -= 1; + if (iop_a_src > io1_a_src) { + (iop_a_src--, wuffs_base__make_empty_struct()); + } else { + status = wuffs_base__make_status( + wuffs_json__error__internal_error_inconsistent_i_o); + goto exit; + } + } + } } } status = wuffs_base__make_status(wuffs_json__error__bad_input); @@ -18933,13 +19006,13 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); goto label__1__continue; } if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); goto label__1__continue; } (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -19015,11 +19088,11 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status( wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); } } } @@ -19144,7 +19217,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); goto label__outer__continue; } } else if (v_class == 10) { @@ -19164,7 +19237,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); goto label__outer__continue; } } else if (v_class == 11) { @@ -19184,7 +19257,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); goto label__outer__continue; } } diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 7cb6a770..8c1ae4b1 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -59,9 +59,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var c : base.u8 var class : base.u8[..= 0x0F] var backslash : base.u8 - var uni4_ok : base.u8 - var uni4_string : base.u32 - var uni4_value : base.u32[..= 0xFFFF] + + var uni4_ok : base.u8 + var uni4_string : base.u64 + var uni4_value : base.u32[..= 0xFFFF] + var uni4_high_surrogate : base.u32[..= 0x10_FC00] + var uni4_rollback : base.u32 // expect is a bitmask of what the next character class can be. // @@ -250,6 +253,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { continue.string_loop } else if backslash == 0x75 { // 0x75 is 'u'. + // -------- BEGIN backslash-u. if args.src.available() < 6 { if args.src.is_closed() { return "#bad input" @@ -258,9 +262,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { string_length = 0 continue.string_loop } - uni4_string = (args.src.peek_u48le_as_u64() >> 16) as base.u32 + + uni4_string = args.src.peek_u48le_as_u64() >> 16 uni4_value = 0 uni4_ok = 0x80 + c = hexadecimal_digits[0xFF & (uni4_string >> 0)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 12 @@ -273,15 +279,99 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { c = hexadecimal_digits[0xFF & (uni4_string >> 24)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 0 - if uni4_ok <> 0 { + + if uni4_ok == 0 { + // It wasn't 4 hexadecimal digits. No-op + // (and fall through to "#bad input"). + + } else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) { + // Not a Unicode surrogate. We're good. args.src.skip32_fast!(actual: 6, worst_case: 6) args.dst.write_fast_token!( value: 0x80_0000 | (uni4_value as base.u64), length: 6) continue.string_loop - } + } else if uni4_value >= 0xDC00 { + // Low surrogate. No-op (and fall through + // to "#bad input"). + + } else { + // High surrogate, which needs to be + // followed by a "\\u1234" low surrogate. + // We've already peeked 6 bytes for the + // high surrogate. We need another 6. + if args.src.available() < 12 { + if args.src.is_closed() { + return "#bad input" + } + yield? base."$short read" + string_length = 0 + uni4_value = 0 + continue.string_loop + } + + // Roll forward 4 bytes, so that calling + // peek_u64le will pick up the last 8. + args.src.skip32_fast!(actual: 4, worst_case: 4) + + uni4_string = args.src.peek_u64le() >> 16 + + // Look for the low surrogate's "\\u". + if ((0xFF & (uni4_string >> 0)) <> 0x5C) or + ((0xFF & (uni4_string >> 8)) <> 0x75) { + uni4_high_surrogate = 0 + uni4_value = 0 + uni4_ok = 0 + } else { + uni4_high_surrogate = + 0x1_0000 + ((uni4_value - 0xD800) << 10) + uni4_value = 0 + uni4_ok = 0x80 + uni4_string >>= 16 + + c = hexadecimal_digits[0xFF & (uni4_string >> 0)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 12 + c = hexadecimal_digits[0xFF & (uni4_string >> 8)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 8 + c = hexadecimal_digits[0xFF & (uni4_string >> 16)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 4 + c = hexadecimal_digits[0xFF & (uni4_string >> 24)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 0 + } + + if (uni4_ok <> 0) and + (0xDC00 <= uni4_value) and (uni4_value <= 0xDFFF) { + + // Emit a single token for the surrogate pair. + uni4_value -= 0xDC00 + args.src.skip32_fast!(actual: 8, worst_case: 8) + args.dst.write_fast_token!( + value: 0x80_0000 | ((uni4_high_surrogate + uni4_value) as base.u64), + length: 12) + continue.string_loop + + } else { + // Roll back the 4 bytes, then fall + // through to "#bad input". + uni4_rollback = 4 + while uni4_rollback > 0 { + uni4_rollback -= 1 + if args.src.can_undo_byte() { + args.src.undo_byte!() + } else { + return "#internal error: inconsistent I/O" + } + } + } + } + // -------- END backslash-u. } + return "#bad input" } diff --git a/test/c/std/json.c b/test/c/std/json.c index 70e44c95..094e230c 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -187,6 +187,115 @@ const char* test_wuffs_json_decode_tokens() { return NULL; } +const char* test_wuffs_json_decode_unicode4_escapes() { + CHECK_FOCUS(__func__); + + const uint32_t fail = 0xDEADBEEF; + + struct { + uint32_t want; + const char* str; + } test_cases[] = { + // Simple (non-surrogate) successes. + {.want = 0x0000000A, .str = "\"\\u000a\""}, + {.want = 0x0000005C, .str = "\"\\\\u1234\""}, // U+005C is '\\'. + {.want = 0x00001000, .str = "\"\\u10002345\""}, + {.want = 0x00001000, .str = "\"\\u1000234\""}, + {.want = 0x00001000, .str = "\"\\u100023\""}, + {.want = 0x00001000, .str = "\"\\u10002\""}, + {.want = 0x00001234, .str = "\"\\u1234\""}, + {.want = 0x0000D7FF, .str = "\"\\ud7ff\""}, + {.want = 0x0000E000, .str = "\"\\uE000\""}, + {.want = 0x0000FFFF, .str = "\"\\uFffF\""}, + + // Unicode surrogate pair. U+0001F4A9 PILE OF POO is (U+D83D, U+DCA9), + // because ((0x03D << 10) | 0x0A9) is 0xF4A9: + // - High surrogates are in the range U+D800 ..= U+DBFF. + // - Low surrogates are in the range U+DC00 ..= U+DFFF. + {.want = 0x0001F4A9, .str = "\"\\uD83D\\udca9\""}, + + // More surrogate pairs. + {.want = 0x00010000, .str = "\"\\uD800\\uDC00\""}, + {.want = 0x0010FFFF, .str = "\"\\uDBFF\\uDFFF\""}, + + // Simple (non-surrogate) failures. + {.want = fail, .str = "\"\\U1234\""}, + {.want = fail, .str = "\"\\u123"}, + {.want = fail, .str = "\"\\u123\""}, + {.want = fail, .str = "\"\\u123x\""}, + {.want = fail, .str = "\"u1234\""}, + + // Invalid surrogate pairs. + {.want = fail, .str = "\"\\uD800\""}, // High alone. + {.want = fail, .str = "\"\\uD83D?udca9\""}, // High then not "\\u". + {.want = fail, .str = "\"\\uD83D\\ud7ff\""}, // High then non-surrogate. + {.want = fail, .str = "\"\\uD83D\\udbff\""}, // High then high. + {.want = fail, .str = "\"\\uD83D\\ue000\""}, // High then non-surrogate. + {.want = fail, .str = "\"\\uDC00\""}, // Low alone. + {.want = fail, .str = "\"\\uDC00\\u0000\""}, // Low then non-surrogate. + {.want = fail, .str = "\"\\uDC00\\ud800\""}, // Low then high. + {.want = fail, .str = "\"\\uDC00\\udfff\""}, // Low then low. + {.want = fail, .str = "\"\\uDFFF1234\""}, // Low alone. + }; + + wuffs_json__decoder dec; + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + CHECK_STATUS("initialize", + wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + + wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ + .data = global_have_token_slice, + }); + size_t n = strlen(test_cases[tc].str); + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8((void*)(test_cases[tc].str), n), + .meta = wuffs_base__make_io_buffer_meta(n, 0, 0, true), + }); + + wuffs_json__decoder__decode_tokens(&dec, &tok, &src); + + uint32_t have = fail; + uint64_t total_length = 0; + size_t i; + for (i = tok.meta.ri; i < tok.meta.wi; i++) { + wuffs_base__token* t = &tok.data.ptr[i]; + total_length = + wuffs_base__u64__sat_add(total_length, wuffs_base__token__length(t)); + + // Set have to the first Unicode code point token. + if ((have == fail) && ((wuffs_base__token__value_base_category(t) == + WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT))) { + have = wuffs_base__token__value_base_detail(t); + if (have > 0x10FFFF) { // This also catches "have == fail". + RETURN_FAIL("%s: invalid Unicode code point", test_cases[tc].str); + } + + uint64_t have_length = wuffs_base__token__length(t); + uint64_t want_length = (have == 0x5C) ? 2 : ((have <= 0xFFFF) ? 6 : 12); + if (have_length != want_length) { + RETURN_FAIL("%s: token length: have %" PRIu64 ", want %" PRIu64, + test_cases[tc].str, have_length, want_length); + } + } + } + + if (have != test_cases[tc].want) { + RETURN_FAIL("%s: have 0x%" PRIX32 ", want 0x%" PRIX32, test_cases[tc].str, + have, test_cases[tc].want); + } + + if (total_length != src.meta.ri) { + RETURN_FAIL("%s: total length: have 0x%" PRIu64 ", want 0x%" PRIu64, + test_cases[tc].str, total_length, src.meta.ri); + } + } + + return NULL; +} + // ---------------- Mimic Tests #ifdef WUFFS_MIMIC @@ -217,7 +326,8 @@ proc tests[] = { // as any other place. test_strconv_parse_number_u64, // - test_wuffs_json_decode_tokens, // + test_wuffs_json_decode_tokens, // + test_wuffs_json_decode_unicode4_escapes, // #ifdef WUFFS_MIMIC