From 97f7f28218d711dac0ea217cd6389ab0deb85ee7 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 19 Mar 2020 14:27:09 +1100 Subject: [PATCH] Add std/json backslash-related quirks --- release/c/wuffs-unsupported-snapshot.c | 50 +++++++++----- std/json/common_consts.wuffs | 35 ++++++++-- std/json/decode_json.wuffs | 35 ++++++---- test/c/std/json.c | 91 ++++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 36 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index c19ce807..3a8bc179 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -5977,14 +5977,9 @@ struct wuffs_json__decoder__struct { wuffs_base__vtable vtable_for__wuffs_base__token_decoder; wuffs_base__vtable null_vtable; - bool f_quirk_enabled_allow_backslash_a; + bool f_quirk_enabled_allow_backslash_etc[8]; bool f_quirk_enabled_allow_backslash_capital_u; - bool f_quirk_enabled_allow_backslash_e; - bool f_quirk_enabled_allow_backslash_question_mark; - bool f_quirk_enabled_allow_backslash_single_quote; - bool f_quirk_enabled_allow_backslash_v; bool f_quirk_enabled_allow_backslash_x; - bool f_quirk_enabled_allow_backslash_zero; bool f_quirk_enabled_allow_comment_block; bool f_quirk_enabled_allow_comment_line; bool f_quirk_enabled_allow_final_comma; @@ -20179,11 +20174,11 @@ static const uint8_t // WUFFS_BASE__POTENTIALLY_UNUSED = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 0, 0, 0, 0, 0, 0, 0, 175, 6, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, - 0, 0, 0, 136, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, - 141, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 136, 0, 0, 2, 140, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, + 141, 0, 137, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -20193,6 +20188,12 @@ static const uint8_t // 0, 0, 0, 0, 0, 0, 0, 0, 0, }; +static const uint8_t // + wuffs_json__lut_quirky_backslashes[8] // + WUFFS_BASE__POTENTIALLY_UNUSED = { + 0, 7, 27, 63, 39, 11, 0, 0, +}; + static const uint8_t // wuffs_json__lut_chars[256] // WUFFS_BASE__POTENTIALLY_UNUSED = { @@ -20383,22 +20384,21 @@ wuffs_json__decoder__set_quirk_enabled(wuffs_json__decoder* self, } if (a_quirk == 1225364480) { - self->private_impl.f_quirk_enabled_allow_backslash_a = a_enabled; + self->private_impl.f_quirk_enabled_allow_backslash_etc[1] = a_enabled; } else if (a_quirk == 1225364481) { self->private_impl.f_quirk_enabled_allow_backslash_capital_u = a_enabled; } else if (a_quirk == 1225364482) { - self->private_impl.f_quirk_enabled_allow_backslash_e = a_enabled; + self->private_impl.f_quirk_enabled_allow_backslash_etc[2] = a_enabled; } else if (a_quirk == 1225364483) { - self->private_impl.f_quirk_enabled_allow_backslash_question_mark = - a_enabled; + self->private_impl.f_quirk_enabled_allow_backslash_etc[3] = a_enabled; } else if (a_quirk == 1225364484) { - self->private_impl.f_quirk_enabled_allow_backslash_single_quote = a_enabled; + self->private_impl.f_quirk_enabled_allow_backslash_etc[4] = a_enabled; } else if (a_quirk == 1225364485) { - self->private_impl.f_quirk_enabled_allow_backslash_v = a_enabled; + self->private_impl.f_quirk_enabled_allow_backslash_etc[5] = a_enabled; } else if (a_quirk == 1225364486) { self->private_impl.f_quirk_enabled_allow_backslash_x = a_enabled; } else if (a_quirk == 1225364487) { - self->private_impl.f_quirk_enabled_allow_backslash_zero = a_enabled; + self->private_impl.f_quirk_enabled_allow_backslash_etc[6] = a_enabled; } else if (a_quirk == 1225364488) { self->private_impl.f_quirk_enabled_allow_comment_block = a_enabled; } else if (a_quirk == 1225364489) { @@ -20708,7 +20708,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, v_c = ((uint8_t)( (wuffs_base__load_u16le__no_bounds_check(iop_a_src) >> 8))); v_backslash = wuffs_json__lut_backslashes[v_c]; - if (v_backslash > 0) { + if ((v_backslash & 128) != 0) { (iop_a_src += 2, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)( @@ -20717,6 +20717,20 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; + } else if (v_backslash != 0) { + if (self->private_impl.f_quirk_enabled_allow_backslash_etc[( + v_backslash & 7)]) { + (iop_a_src += 2, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)( + (6291456 | + ((uint32_t)(wuffs_json__lut_quirky_backslashes[( + v_backslash & 7)]))))) + << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; + } } else if (v_c == 117) { if (((uint64_t)(io2_a_src - iop_a_src)) < 6) { if (a_src && a_src->meta.closed) { diff --git a/std/json/common_consts.wuffs b/std/json/common_consts.wuffs index 553768d0..5414d70a 100644 --- a/std/json/common_consts.wuffs +++ b/std/json/common_consts.wuffs @@ -56,6 +56,24 @@ pri const decoder_number_length_max_incl base.u64 = 99 // Look-Up Tables (LUTs). +// lut_backslashes[i] helps decode "\i", for various 'i's. +// +// If the element's 0x80 bit is set then "\i" is unconditionally a valid +// single-output-byte backslash-escape. The low 7 bits are the unescaped value. +// For example, lut_backslashes['n'] is (0x80 | 0x0A), because "\n" is U+000A. +// +// If the element is non-zero (but the 0x80 bit is not set) then "\i"'s +// validity depends on the relevant quirk. The element's value is an enum: +// - 1: "\a", U+0007, quirk_allow_backslash_a. +// - 2: "\e", U+001B, quirk_allow_backslash_e. +// - 3: "\?", U+003F, quirk_allow_backslash_question_mark. +// - 4: "\'", U+0027, quirk_allow_backslash_single_quote. +// - 5: "\v", U+000B, quirk_allow_backslash_v. +// - 6: "\0", U+0000, quirk_allow_backslash_zero. +// The U+1234 values are held in lut_quirky_backslashes, below. +// +// If the element is zero then "\i" is invalid, or it is a special case, the +// start of "\x12", "\u1234" or "\U12345678". pri const lut_backslashes array[256] base.u8 = [ // 0 1 2 3 4 5 6 7 // 8 9 A B C D E F @@ -63,18 +81,18 @@ pri const lut_backslashes array[256] base.u8 = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F. - 0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. '"'. + 0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x04, // 0x20 ..= 0x27. '"', '\''. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAF, // 0x28 ..= 0x2F. '/'. - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. '0'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // 0x38 ..= 0x3F. '?' 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. 0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. '\\'. - 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8C, 0x00, // 0x60 ..= 0x67. 'b', 'f'. + 0x00, 0x01, 0x88, 0x00, 0x00, 0x02, 0x8C, 0x00, // 0x60 ..= 0x67. 'a', 'b', 'e', 'f'. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x00, // 0x68 ..= 0x6F. 'n'. - 0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. 'r', 't'. + 0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x05, 0x00, // 0x70 ..= 0x77. 'r', 't', 'v'. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87. @@ -98,6 +116,13 @@ pri const lut_backslashes array[256] base.u8 = [ // 8 9 A B C D E F ] +// lut_quirky_backslashes is discussed in the lut_backslashes comment. The +// first and last elements (indexes 0 and 7) are not used, but 8 is a round +// power of 2, so enforcing index-in-bounds is a simple "&7" operation. +pri const lut_quirky_backslashes array[8] base.u8 = [ + 0x00, 0x07, 0x1B, 0x3F, 0x27, 0x0B, 0x00, 0x00, +] + // lut_chars helps decode bytes within a string: // - 0x00 is 1-byte UTF-8 (ASCII) but not '"', '\\' or a C0 control code. // - 0x01 is '"'. diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index f24e00a8..51edc4fe 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -13,14 +13,12 @@ // limitations under the License. pub struct decoder? implements base.token_decoder( - quirk_enabled_allow_backslash_a : base.bool, + // quirk_enabled_allow_backslash_etc, an 8-element array, is indexed by the + // same enum as lut_quirky_backslashes. + quirk_enabled_allow_backslash_etc : array[8] base.bool, + quirk_enabled_allow_backslash_capital_u : base.bool, - quirk_enabled_allow_backslash_e : base.bool, - quirk_enabled_allow_backslash_question_mark : base.bool, - quirk_enabled_allow_backslash_single_quote : base.bool, - quirk_enabled_allow_backslash_v : base.bool, quirk_enabled_allow_backslash_x : base.bool, - quirk_enabled_allow_backslash_zero : base.bool, quirk_enabled_allow_comment_block : base.bool, quirk_enabled_allow_comment_line : base.bool, quirk_enabled_allow_final_comma : base.bool, @@ -68,21 +66,21 @@ pub struct decoder? implements base.token_decoder( pub func decoder.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) { if args.quirk == quirk_allow_backslash_a { - this.quirk_enabled_allow_backslash_a = args.enabled + this.quirk_enabled_allow_backslash_etc[1] = args.enabled } else if args.quirk == quirk_allow_backslash_capital_u { this.quirk_enabled_allow_backslash_capital_u = args.enabled } else if args.quirk == quirk_allow_backslash_e { - this.quirk_enabled_allow_backslash_e = args.enabled + this.quirk_enabled_allow_backslash_etc[2] = args.enabled } else if args.quirk == quirk_allow_backslash_question_mark { - this.quirk_enabled_allow_backslash_question_mark = args.enabled + this.quirk_enabled_allow_backslash_etc[3] = args.enabled } else if args.quirk == quirk_allow_backslash_single_quote { - this.quirk_enabled_allow_backslash_single_quote = args.enabled + this.quirk_enabled_allow_backslash_etc[4] = args.enabled } else if args.quirk == quirk_allow_backslash_v { - this.quirk_enabled_allow_backslash_v = args.enabled + this.quirk_enabled_allow_backslash_etc[5] = args.enabled } else if args.quirk == quirk_allow_backslash_x { this.quirk_enabled_allow_backslash_x = args.enabled } else if args.quirk == quirk_allow_backslash_zero { - this.quirk_enabled_allow_backslash_zero = args.enabled + this.quirk_enabled_allow_backslash_etc[6] = args.enabled } else if args.quirk == quirk_allow_comment_block { this.quirk_enabled_allow_comment_block = args.enabled } else if args.quirk == quirk_allow_comment_line { @@ -335,7 +333,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { } c = (args.src.peek_u16le() >> 8) as base.u8 backslash = lut_backslashes[c] - if backslash > 0 { + if (backslash & 0x80) <> 0 { args.src.skip32_fast!(actual: 2, worst_case: 2) args.dst.write_fast_token!( value_major: 0, @@ -344,6 +342,17 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { length: 2) continue.string_loop_outer + } else if backslash <> 0 { + if this.quirk_enabled_allow_backslash_etc[backslash & 7] { + args.src.skip32_fast!(actual: 2, worst_case: 2) + args.dst.write_fast_token!( + value_major: 0, + value_minor: 0x60_0000 | (lut_quirky_backslashes[backslash & 7] as base.u32), + link: 0x3, + length: 2) + continue.string_loop_outer + } + } else if c == 0x75 { // 0x75 is 'u'. // -------- BEGIN backslash-u. if args.src.available() < 6 { diff --git a/test/c/std/json.c b/test/c/std/json.c index a7f835b0..eb5d018d 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -1105,6 +1105,96 @@ test_wuffs_json_decode_prior_valid_utf_8() { return NULL; } +const char* // +test_wuffs_json_decode_quirk_allow_backslash_etc() { + CHECK_FOCUS(__func__); + + struct { + uint32_t want; + const char* str; + uint32_t quirk; + } test_cases[] = { + { + .want = 0x07, + .str = "\"\\a\"", + .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_A, + }, + { + .want = 0x1B, + .str = "\"\\e\"", + .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_E, + }, + { + .want = 0x3F, + .str = "\"\\?\"", + .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_QUESTION_MARK, + }, + { + .want = 0x27, + .str = "\"\\'\"", + .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE, + }, + { + .want = 0x0B, + .str = "\"\\v\"", + .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_V, + }, + { + .want = 0x00, + .str = "\"\\0\"", + .quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO, + }, + }; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + int q; + for (q = 0; q < 2; q++) { + wuffs_json__decoder dec; + CHECK_STATUS("initialize", wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + wuffs_json__decoder__set_quirk_enabled(&dec, test_cases[tc].quirk, q); + + wuffs_base__token_buffer tok = + wuffs_base__make_token_buffer_writer(global_have_token_slice); + wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader( + wuffs_base__make_slice_u8((void*)(test_cases[tc].str), + strlen(test_cases[tc].str)), + true); + + const char* have_status_repr = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr; + const char* want_status_repr = + q ? NULL : wuffs_json__error__bad_backslash_escape; + if (have_status_repr != want_status_repr) { + RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc, + q, have_status_repr, want_status_repr); + } + if (want_status_repr != NULL) { + continue; + } + + uint32_t have = 0; + while (tok.meta.ri < tok.meta.wi) { + wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++]; + uint64_t vbc = wuffs_base__token__value_base_category(t); + uint64_t vbd = wuffs_base__token__value_base_detail(t); + if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) { + have = vbd; + break; + } + } + if (have != test_cases[tc].want) { + RETURN_FAIL("tc=%d, q=%d: Unicode code point: have U+%04" PRIX32 + ", want U+%04" PRIX32, + tc, q, have, test_cases[tc].want); + } + } + } + return NULL; +} + const char* // test_wuffs_json_decode_quirk_allow_leading_etc() { CHECK_FOCUS(__func__); @@ -1746,6 +1836,7 @@ proc tests[] = { test_wuffs_json_decode_interface, test_wuffs_json_decode_long_numbers, test_wuffs_json_decode_prior_valid_utf_8, + test_wuffs_json_decode_quirk_allow_backslash_etc, test_wuffs_json_decode_quirk_allow_leading_etc, test_wuffs_json_decode_quirk_allow_trailing_etc, test_wuffs_json_decode_quirk_replace_invalid_utf_8,