From 502c8eff56992cc23c003cc560e84a3bbe919ad8 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 21 Mar 2020 21:42:30 +1100 Subject: [PATCH] Add std/json quirk_allow_backslash_x --- example/jsonfindptrs/jsonfindptrs.cc | 39 ++++++++++++ release/c/wuffs-unsupported-snapshot.c | 87 ++++++++++++++++++++------ std/json/decode_json.wuffs | 50 +++++++++++++++ test/c/std/json.c | 81 ++++++++++++++++++++++++ 4 files changed, 239 insertions(+), 18 deletions(-) diff --git a/example/jsonfindptrs/jsonfindptrs.cc b/example/jsonfindptrs/jsonfindptrs.cc index e2ead122..a21a2ce9 100644 --- a/example/jsonfindptrs/jsonfindptrs.cc +++ b/example/jsonfindptrs/jsonfindptrs.cc @@ -272,6 +272,13 @@ class TokenStream { m_curr_token_end_src_index(0) { m_status = m_dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0); + + // Uncomment these lines to enable the WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X + // option, discussed in a separate comment. + // + // if (m_status.is_ok()) { + // m_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true); + // } } Result peek() { return peek_or_next(false); } @@ -584,11 +591,43 @@ JsonThing::parse_string(TokenStream& ts, TokenStream::Result tsr) { case WUFFS_BASE__TOKEN__VBC__STRING: { if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) { // No-op. + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { const char* ptr = // Convert from (uint8_t*). static_cast(static_cast(tsr.src_data.ptr)); jt.value.s.append(ptr, tsr.src_data.len); + + } else if ( + vbd & + WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X) { + // We shouldn't get here unless we enable the + // WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X option. The jsonfindptrs + // program doesn't enable that by default, but if you're copy/pasting + // this JsonThing code and your program does enable that option, + // here's how to handle it. + wuffs_base__slice_u8 encoded = tsr.src_data; + if (encoded.len & 3) { + return Result( + "main: internal error: \\x token length not a multiple of 4", + JsonThing()); + } + while (encoded.len) { + uint8_t decoded[64]; + size_t len = wuffs_base__hexadecimal__decode4( + wuffs_base__make_slice_u8(&decoded[0], 64), encoded); + if ((len > 64) || ((len * 4) > encoded.len)) { + return Result( + "main: internal error: inconsistent hexadecimal decoding", + JsonThing()); + } + const char* ptr = // Convert from (uint8_t*). + static_cast(static_cast(&decoded[0])); + jt.value.s.append(ptr, len); + encoded.ptr += len * 4; + encoded.len -= len * 4; + } + } else { return Result( "main: internal error: unexpected string-token conversion", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 0d1b1aad..52c844d7 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -20532,6 +20532,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, uint8_t v_char = 0; uint8_t v_class = 0; uint32_t v_multi_byte_utf8 = 0; + uint32_t v_backslash_x_length = 0; + uint8_t v_backslash_x_ok = 0; + uint32_t v_backslash_x_string = 0; uint8_t v_uni4_ok = 0; uint64_t v_uni4_string = 0; uint32_t v_uni4_value = 0; @@ -21008,6 +21011,54 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } + } else if ((v_c == 120) && + self->private_impl + .f_quirk_enabled_allow_backslash_x) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status( + wuffs_json__error__bad_backslash_escape); + goto exit; + } + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + v_string_length = 0; + v_char = 0; + goto label__string_loop_outer__continue; + } + v_backslash_x_length = 0; + while ((v_backslash_x_length <= 65531) && + (((uint64_t)(io2_a_src - iop_a_src)) >= 4)) { + v_backslash_x_string = + wuffs_base__load_u32le__no_bounds_check(iop_a_src); + v_backslash_x_ok = 128; + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_backslash_x_string >> 16))]; + v_backslash_x_ok &= v_c; + v_c = wuffs_json__lut_hexadecimal_digits[( + 255 & (v_backslash_x_string >> 24))]; + v_backslash_x_ok &= v_c; + if ((v_backslash_x_ok == 0) || + ((v_backslash_x_string & 65535) != 30812)) { + goto label__1__break; + } + (iop_a_src += 4, wuffs_base__make_empty_struct()); + v_backslash_x_length += 4; + } + label__1__break:; + if (v_backslash_x_length == 0) { + status = wuffs_base__make_status( + wuffs_json__error__bad_backslash_escape); + goto exit; + } + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(4194432)) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(v_backslash_x_length)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; } status = wuffs_base__make_status( wuffs_json__error__bad_backslash_escape); @@ -21044,7 +21095,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status( wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); v_string_length = 0; v_char = 0; goto label__string_loop_outer__continue; @@ -21100,7 +21151,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status( wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); v_string_length = 0; v_char = 0; goto label__string_loop_outer__continue; @@ -21161,7 +21212,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status( wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); v_string_length = 0; v_char = 0; goto label__string_loop_outer__continue; @@ -21222,7 +21273,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } } label__string_loop_outer__break:; - label__1__continue:; + label__2__continue:; while (true) { if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (a_src && a_src->meta.closed) { @@ -21231,14 +21282,14 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); - goto label__1__continue; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); + goto label__2__continue; } if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); - goto label__1__continue; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); + goto label__2__continue; } (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( @@ -21246,9 +21297,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - goto label__1__break; + goto label__2__break; } - label__1__break:; + label__2__break:; if (0 == (v_expect & 16)) { v_expect = 8; goto label__outer__continue; @@ -21293,7 +21344,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(v_number_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - goto label__2__break; + goto label__3__break; } while (v_number_length > 0) { v_number_length -= 1; @@ -21315,15 +21366,15 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status( wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17); } } } - label__2__break:; + label__3__break:; goto label__goto_parsed_a_leaf_value__break; } else if (v_class == 5) { v_vminor = 2113553; @@ -21453,7 +21504,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18); goto label__outer__continue; } } else if (v_class == 10) { @@ -21474,7 +21525,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19); goto label__outer__continue; } } else if (v_class == 11) { @@ -21495,7 +21546,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(20); goto label__outer__continue; } } @@ -21516,7 +21567,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(20); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(21); status = wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src); if (a_dst) { diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 017e6b20..db5113ef 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -117,6 +117,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var class : base.u8[..= 0x0F] var multi_byte_utf8 : base.u32 + var backslash_x_length : base.u32[..= 0xFFFF] + var backslash_x_ok : base.u8 + var backslash_x_string : base.u32 + var uni4_ok : base.u8 var uni4_string : base.u64 var uni4_value : base.u32[..= 0xFFFF] @@ -554,6 +558,52 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { continue.string_loop_outer } // -------- END backslash-capital-u. + + } else if (c == 0x78) and + this.quirk_enabled_allow_backslash_x { // 0x78 is 'x'. + // -------- BEGIN backslash-x + if args.src.available() < 4 { + if args.src.is_closed() { + return "#bad backslash-escape" + } + yield? base."$short read" + string_length = 0 + char = 0 + continue.string_loop_outer + } + + backslash_x_length = 0 + while (backslash_x_length <= 0xFFFB) and (args.src.available() >= 4), + inv args.dst.available() > 0, + { + backslash_x_string = args.src.peek_u32le() + backslash_x_ok = 0x80 + + c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 16)] + backslash_x_ok &= c + c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 24)] + backslash_x_ok &= c + + if (backslash_x_ok == 0) or + ((backslash_x_string & 0xFFFF) <> 0x785C) { + // It wasn't "\\x34", for some + // hexadecimal digits "34". + break + } + args.src.skip32_fast!(actual: 4, worst_case: 4) + backslash_x_length += 4 + } + + if backslash_x_length == 0 { + return "#bad backslash-escape" + } + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x40_0080, + link: 0x3, + length: backslash_x_length) + continue.string_loop_outer + // -------- END backslash-x } return "#bad backslash-escape" diff --git a/test/c/std/json.c b/test/c/std/json.c index 07d943cf..2d9cb1e6 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -1249,6 +1249,86 @@ test_wuffs_json_decode_quirk_allow_backslash_etc() { return NULL; } +const char* // +test_wuffs_json_decode_quirk_allow_backslash_x() { + CHECK_FOCUS(__func__); + + struct { + uint64_t want_bytes; + const char* want_status_repr; + const char* str; + } test_cases[] = { + {.want_bytes = 0x12789A, + .want_status_repr = NULL, + .str = "\"\\x12\\u3456\\x78\\x9A\""}, + {.want_bytes = 0x00, + .want_status_repr = wuffs_json__error__bad_backslash_escape, + .str = "\"a\\X6A\""}, + {.want_bytes = 0x6A6B, + .want_status_repr = NULL, + .str = "\"a\\x6A\\x6bz\""}, + {.want_bytes = 0x6A, + .want_status_repr = wuffs_json__error__bad_backslash_escape, + .str = "\"a\\x6A\\x6yz\""}, + {.want_bytes = 0x00, + .want_status_repr = wuffs_json__error__bad_backslash_escape, + .str = "\"a\\x\""}, + }; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + wuffs_json__decoder dec; + CHECK_STATUS("initialize", wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + wuffs_json__decoder__set_quirk_enabled( + &dec, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true); + + wuffs_base__token_buffer tok = + wuffs_base__make_token_buffer_writer(global_have_token_slice); + wuffs_base__slice_u8 src_slice = wuffs_base__make_slice_u8( + (void*)(test_cases[tc].str), strlen(test_cases[tc].str)); + wuffs_base__io_buffer src = + wuffs_base__make_io_buffer_reader(src_slice, true); + const char* have_status_repr = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr; + if (have_status_repr != test_cases[tc].want_status_repr) { + RETURN_FAIL("tc=%d: decode_tokens: have \"%s\", want \"%s\"", tc, + have_status_repr, test_cases[tc].want_status_repr); + } + + uint64_t src_index = 0; + uint64_t have_bytes = 0; + while (tok.meta.ri < tok.meta.wi) { + wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++]; + uint64_t vbc = wuffs_base__token__value_base_category(t); + uint64_t vbd = wuffs_base__token__value_base_detail(t); + uint64_t token_length = wuffs_base__token__length(t); + if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) && + (vbd == + WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) { + uint8_t b[8]; + size_t n = wuffs_base__hexadecimal__decode4( + wuffs_base__make_slice_u8(&b[0], 8), + wuffs_base__make_slice_u8(src_slice.ptr + src_index, token_length)); + size_t i = 0; + for (; i < n; i++) { + have_bytes <<= 8; + have_bytes |= b[i]; + } + } + + src_index += token_length; + } + if (have_bytes != test_cases[tc].want_bytes) { + RETURN_FAIL("tc=%d: have U+%08" PRIX64 ", want U+%08" PRIX64, tc, + have_bytes, test_cases[tc].want_bytes); + } + } + + return NULL; +} + const char* // test_wuffs_json_decode_quirk_allow_leading_etc() { CHECK_FOCUS(__func__); @@ -1892,6 +1972,7 @@ proc tests[] = { test_wuffs_json_decode_long_numbers, test_wuffs_json_decode_prior_valid_utf_8, test_wuffs_json_decode_quirk_allow_backslash_etc, + test_wuffs_json_decode_quirk_allow_backslash_x, test_wuffs_json_decode_quirk_allow_leading_etc, test_wuffs_json_decode_quirk_allow_trailing_etc, test_wuffs_json_decode_quirk_replace_invalid_utf_8,