diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3bc35c96..6dce4a12 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -20767,6 +20767,18 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else { if (((uint64_t)(io2_a_src - iop_a_src)) < 12) { if (a_src && a_src->meta.closed) { + if (self->private_impl + .f_quirk_enabled_replace_invalid_utf_8) { + (iop_a_src += 6, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) + << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(6)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; + } status = wuffs_base__make_status( wuffs_json__error__bad_backslash_escape); goto exit; @@ -20837,6 +20849,21 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } } } + if (self->private_impl + .f_quirk_enabled_replace_invalid_utf_8) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 6) { + status = wuffs_base__make_status( + wuffs_json__error__internal_error_inconsistent_i_o); + goto exit; + } + (iop_a_src += 6, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; + } } status = wuffs_base__make_status( wuffs_json__error__bad_backslash_escape); @@ -20851,8 +20878,22 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + goto label__string_loop_outer__continue; + } } if (a_src && a_src->meta.closed) { + if (self->private_impl + .f_quirk_enabled_replace_invalid_utf_8) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (iop_a_src += 1, wuffs_base__make_empty_struct()); + goto label__string_loop_inner__continue; + } status = wuffs_base__make_status(wuffs_json__error__bad_utf_8); goto exit; @@ -20893,8 +20934,22 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + goto label__string_loop_outer__continue; + } } if (a_src && a_src->meta.closed) { + if (self->private_impl + .f_quirk_enabled_replace_invalid_utf_8) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (iop_a_src += 1, wuffs_base__make_empty_struct()); + goto label__string_loop_inner__continue; + } status = wuffs_base__make_status(wuffs_json__error__bad_utf_8); goto exit; @@ -20940,8 +20995,22 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + goto label__string_loop_outer__continue; + } } if (a_src && a_src->meta.closed) { + if (self->private_impl + .f_quirk_enabled_replace_invalid_utf_8) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (iop_a_src += 1, wuffs_base__make_empty_struct()); + goto label__string_loop_inner__continue; + } status = wuffs_base__make_status(wuffs_json__error__bad_utf_8); goto exit; @@ -20986,12 +21055,24 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + goto label__string_loop_outer__continue; + } } if (v_char == 128) { status = wuffs_base__make_status( wuffs_json__error__bad_c0_control_code); goto exit; } + if (self->private_impl.f_quirk_enabled_replace_invalid_utf_8) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (iop_a_src += 1, wuffs_base__make_empty_struct()); + goto label__string_loop_inner__continue; + } status = wuffs_base__make_status(wuffs_json__error__bad_utf_8); goto exit; } diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index b8152941..0f9dbd7b 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -400,6 +400,15 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { // high surrogate. We need another 6. if args.src.available() < 12 { if args.src.is_closed() { + if this.quirk_enabled_replace_invalid_utf_8 { + args.src.skip32_fast!(actual: 6, worst_case: 6) + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_FFFD, + link: 0x3, + length: 6) + continue.string_loop_outer + } return "#bad backslash-escape" } yield? base."$short read" @@ -460,7 +469,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { // through to "#invalid // backslash-escape". uni4_rollback = 4 - while uni4_rollback > 0 { + while uni4_rollback > 0, + inv args.dst.available() > 0, + { uni4_rollback -= 1 if args.src.can_undo_byte() { args.src.undo_byte!() @@ -470,6 +481,19 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { } } } + + if this.quirk_enabled_replace_invalid_utf_8 { + if args.src.available() < 6 { + return "#internal error: inconsistent I/O" + } + args.src.skip32_fast!(actual: 6, worst_case: 6) + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_FFFD, + link: 0x3, + length: 6) + continue.string_loop_outer + } // -------- END backslash-u. } @@ -484,8 +508,20 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { link: 0x3, length: string_length) string_length = 0 + if args.dst.available() <= 0 { + continue.string_loop_outer + } } if args.src.is_closed() { + if this.quirk_enabled_replace_invalid_utf_8 { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_FFFD, + link: 0x3, + length: 1) + args.src.skip32_fast!(actual: 1, worst_case: 1) + continue.string_loop_inner + } return "#bad UTF-8" } yield? base."$short read" @@ -520,8 +556,20 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { link: 0x3, length: string_length) string_length = 0 + if args.dst.available() <= 0 { + continue.string_loop_outer + } } if args.src.is_closed() { + if this.quirk_enabled_replace_invalid_utf_8 { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_FFFD, + link: 0x3, + length: 1) + args.src.skip32_fast!(actual: 1, worst_case: 1) + continue.string_loop_inner + } return "#bad UTF-8" } yield? base."$short read" @@ -561,8 +609,20 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { link: 0x3, length: string_length) string_length = 0 + if args.dst.available() <= 0 { + continue.string_loop_outer + } } if args.src.is_closed() { + if this.quirk_enabled_replace_invalid_utf_8 { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_FFFD, + link: 0x3, + length: 1) + args.src.skip32_fast!(actual: 1, worst_case: 1) + continue.string_loop_inner + } return "#bad UTF-8" } yield? base."$short read" @@ -600,10 +660,22 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { link: 0x3, length: string_length) string_length = 0 + if args.dst.available() <= 0 { + continue.string_loop_outer + } } if char == 0x80 { return "#bad C0 control code" } + if this.quirk_enabled_replace_invalid_utf_8 { + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_FFFD, + link: 0x3, + length: 1) + args.src.skip32_fast!(actual: 1, worst_case: 1) + continue.string_loop_inner + } return "#bad UTF-8" } endwhile.string_loop_inner } endwhile.string_loop_outer diff --git a/test/c/std/json.c b/test/c/std/json.c index 123426d4..a7f835b0 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -1261,6 +1261,140 @@ test_wuffs_json_decode_quirk_allow_trailing_etc() { return NULL; } +const char* // +test_wuffs_json_decode_quirk_replace_invalid_utf_8() { + CHECK_FOCUS(__func__); + + // Decoding str should produce want, with invalid UTF-8 replaced by "?". A + // proper JSON decoder (with the quirk enabled) would replace with + // "\xEF\xBF\xBD", the UTF-8 encoding of U+FFFD, but using "?" leads to + // clearer, shorter test cases. + struct { + const char* want; + const char* str; + } test_cases[] = { + // Valid UTF-8. + {.want = "abc", .str = "\"abc\""}, + {.want = "del\xCE\x94ta", .str = "\"del\\u0394ta\""}, + {.want = "del\xCE\x94ta", .str = "\"del\xCE\x94ta\""}, + + // Invalid UTF-8: right byte lengths, wrong bytes. + {.want = "1byte?yz", .str = "\"1byte\xFFyz\""}, + {.want = "2byte??yz", .str = "\"2byte\xCE\xFFyz\""}, + {.want = "3byte???yz", .str = "\"3byte\xE2\x98\xFFyz\""}, + {.want = "4byte????yz", .str = "\"4byte\xF0\x9F\x92\xFFyz\""}, + + // Invalid UTF-8: wrong byte lengths. + {.want = "?", .str = "\"\xCE\""}, + {.want = "?g", .str = "\"\xCEg\""}, + {.want = "?gh", .str = "\"\xCEgh\""}, + {.want = "j?", .str = "\"j\xE2\""}, + {.want = "j?l", .str = "\"j\xE2l\""}, + {.want = "j?lm", .str = "\"j\xE2lm\""}, + {.want = "?", .str = "\"\xF0\""}, + {.want = "?r", .str = "\"\xF0r\""}, + {.want = "?rs", .str = "\"\xF0rs\""}, + + // U+DC00 (as an unpaired surrogate) is either 1 or 3 '?'s depending on + // whether it's backslash-u or backslash-x. + {.want = "a?z", .str = "\"a\\uDC00z\""}, + {.want = "a?zzzzzz", .str = "\"a\\uDC00zzzzzz\""}, + {.want = "a???z", .str = "\"a\xED\xB0\x80z\""}, + {.want = "a???zzzzzz", .str = "\"a\xED\xB0\x80zzzzzz\""}, + + // 1 or 2 unpaired surrogates each become '?'s, but for 3 surrogates + // where consecutive surrogates make a valid pair, there's only 1 '?'. + {.want = "a?z", .str = "\"a\\uD800z\""}, + {.want = "a??z", .str = "\"a\\uD800\\uDBFFz\""}, + {.want = "a?\xF4\x8F\xBF\xBFz", .str = "\"a\\uD800\\uDBFF\\uDFFFz\""}, + {.want = "a\xF0\x90\x80\x80?z", .str = "\"a\\uD800\\uDC00\\uDFFFz\""}, + }; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + wuffs_json__decoder dec; + CHECK_STATUS("initialize", wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + wuffs_json__decoder__set_quirk_enabled( + &dec, WUFFS_JSON__QUIRK_REPLACE_INVALID_UTF_8, true); + + wuffs_base__io_buffer have = + wuffs_base__make_io_buffer_writer(global_have_slice); + wuffs_base__token_buffer tok = + wuffs_base__make_token_buffer_writer(global_have_token_slice); + wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader( + wuffs_base__make_slice_u8((void*)(test_cases[tc].str), + strlen(test_cases[tc].str)), + true); + CHECK_STATUS("decode_tokens", + wuffs_json__decoder__decode_tokens(&dec, &tok, &src)); + + uint64_t src_index = 0; + while (tok.meta.ri < tok.meta.wi) { + wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++]; + uint64_t vbc = wuffs_base__token__value_base_category(t); + uint64_t vbd = wuffs_base__token__value_base_detail(t); + uint64_t token_length = wuffs_base__token__length(t); + + if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) { + uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL]; + size_t n = wuffs_base__utf_8__encode( + wuffs_base__make_slice_u8(&u[0], + WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL), + vbd); + if (vbd == 0xFFFD) { + u[0] = '?'; + n = 1; + } + if ((have.data.len - have.meta.wi) < n) { + RETURN_FAIL("tc=%d: token too long", tc); + } + memcpy(&have.data.ptr[have.meta.wi], &u[0], n); + have.meta.wi += n; + + } else if (vbc == WUFFS_BASE__TOKEN__VBC__STRING) { + if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) { + // No-op. + } else if (vbd & + WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { + if ((have.data.len - have.meta.wi) < token_length) { + RETURN_FAIL("tc=%d: token too long", tc); + } + memcpy(&have.data.ptr[have.meta.wi], &test_cases[tc].str[src_index], + token_length); + have.meta.wi += token_length; + } else { + RETURN_FAIL("tc=%d: unexpected string-token conversion", tc); + } + + } else { + RETURN_FAIL("tc=%d: unexpected token", tc); + } + + src_index += token_length; + } + + if (src_index != src.meta.ri) { + RETURN_FAIL("tc=%d: src_index: have %zu, want %zu", tc, src_index, + src.meta.ri); + } + + if (have.meta.wi >= have.data.len) { + RETURN_FAIL("tc=%d: too many have bytes", tc); + } + have.data.ptr[have.meta.wi] = '\x00'; + size_t len = strlen(test_cases[tc].want); + if ((len != have.meta.wi) || + (memcmp(have.data.ptr, test_cases[tc].want, len) != 0)) { + RETURN_FAIL("tc=%d: have \"%s\", want \"%s\"", tc, have.data.ptr, + test_cases[tc].want); + } + } + + return NULL; +} + const char* // test_wuffs_json_decode_unicode4_escapes() { CHECK_FOCUS(__func__); @@ -1614,6 +1748,7 @@ proc tests[] = { test_wuffs_json_decode_prior_valid_utf_8, test_wuffs_json_decode_quirk_allow_leading_etc, test_wuffs_json_decode_quirk_allow_trailing_etc, + test_wuffs_json_decode_quirk_replace_invalid_utf_8, test_wuffs_json_decode_src_io_buffer_length, test_wuffs_json_decode_string, test_wuffs_json_decode_unicode4_escapes,