mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
Add std/json backslash-related quirks
This commit is contained in:
1 parent
b86373aff5
commit
97f7f28218
4 files changed
+175
-36
No files matched your search
@@ -5977,14 +5977,9 @@ struct wuffs_json__decoder__struct {
|
||||
wuffs_base__vtable vtable_for__wuffs_base__token_decoder;
|
||||
wuffs_base__vtable null_vtable;
|
||||
|
||||
bool f_quirk_enabled_allow_backslash_a;
|
||||
bool f_quirk_enabled_allow_backslash_etc[8];
|
||||
bool f_quirk_enabled_allow_backslash_capital_u;
|
||||
bool f_quirk_enabled_allow_backslash_e;
|
||||
bool f_quirk_enabled_allow_backslash_question_mark;
|
||||
bool f_quirk_enabled_allow_backslash_single_quote;
|
||||
bool f_quirk_enabled_allow_backslash_v;
|
||||
bool f_quirk_enabled_allow_backslash_x;
|
||||
bool f_quirk_enabled_allow_backslash_zero;
|
||||
bool f_quirk_enabled_allow_comment_block;
|
||||
bool f_quirk_enabled_allow_comment_line;
|
||||
bool f_quirk_enabled_allow_final_comma;
|
||||
@@ -20179,11 +20174,11 @@ static const uint8_t //
|
||||
WUFFS_BASE__POTENTIALLY_UNUSED = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 4, 0, 0, 0, 0, 0, 0, 0, 175, 6, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0,
|
||||
0, 0, 0, 136, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0,
|
||||
141, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 1, 136, 0, 0, 2, 140, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0,
|
||||
141, 0, 137, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
@@ -20193,6 +20188,12 @@ static const uint8_t //
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
};
|
||||
|
||||
static const uint8_t //
|
||||
wuffs_json__lut_quirky_backslashes[8] //
|
||||
WUFFS_BASE__POTENTIALLY_UNUSED = {
|
||||
0, 7, 27, 63, 39, 11, 0, 0,
|
||||
};
|
||||
|
||||
static const uint8_t //
|
||||
wuffs_json__lut_chars[256] //
|
||||
WUFFS_BASE__POTENTIALLY_UNUSED = {
|
||||
@@ -20383,22 +20384,21 @@ wuffs_json__decoder__set_quirk_enabled(wuffs_json__decoder* self,
|
||||
}
|
||||
|
||||
if (a_quirk == 1225364480) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_a = a_enabled;
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_etc[1] = a_enabled;
|
||||
} else if (a_quirk == 1225364481) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_capital_u = a_enabled;
|
||||
} else if (a_quirk == 1225364482) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_e = a_enabled;
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_etc[2] = a_enabled;
|
||||
} else if (a_quirk == 1225364483) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_question_mark =
|
||||
a_enabled;
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_etc[3] = a_enabled;
|
||||
} else if (a_quirk == 1225364484) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_single_quote = a_enabled;
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_etc[4] = a_enabled;
|
||||
} else if (a_quirk == 1225364485) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_v = a_enabled;
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_etc[5] = a_enabled;
|
||||
} else if (a_quirk == 1225364486) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_x = a_enabled;
|
||||
} else if (a_quirk == 1225364487) {
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_zero = a_enabled;
|
||||
self->private_impl.f_quirk_enabled_allow_backslash_etc[6] = a_enabled;
|
||||
} else if (a_quirk == 1225364488) {
|
||||
self->private_impl.f_quirk_enabled_allow_comment_block = a_enabled;
|
||||
} else if (a_quirk == 1225364489) {
|
||||
@@ -20708,7 +20708,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
v_c = ((uint8_t)(
|
||||
(wuffs_base__load_u16le__no_bounds_check(iop_a_src) >> 8)));
|
||||
v_backslash = wuffs_json__lut_backslashes[v_c];
|
||||
if (v_backslash > 0) {
|
||||
if ((v_backslash & 128) != 0) {
|
||||
(iop_a_src += 2, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(
|
||||
@@ -20717,6 +20717,20 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
} else if (v_backslash != 0) {
|
||||
if (self->private_impl.f_quirk_enabled_allow_backslash_etc[(
|
||||
v_backslash & 7)]) {
|
||||
(iop_a_src += 2, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(
|
||||
(6291456 |
|
||||
((uint32_t)(wuffs_json__lut_quirky_backslashes[(
|
||||
v_backslash & 7)])))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
}
|
||||
} else if (v_c == 117) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 6) {
|
||||
if (a_src && a_src->meta.closed) {
|
||||
|
||||
@@ -56,6 +56,24 @@ pri const decoder_number_length_max_incl base.u64 = 99
|
||||
|
||||
// Look-Up Tables (LUTs).
|
||||
|
||||
// lut_backslashes[i] helps decode "\i", for various 'i's.
|
||||
//
|
||||
// If the element's 0x80 bit is set then "\i" is unconditionally a valid
|
||||
// single-output-byte backslash-escape. The low 7 bits are the unescaped value.
|
||||
// For example, lut_backslashes['n'] is (0x80 | 0x0A), because "\n" is U+000A.
|
||||
//
|
||||
// If the element is non-zero (but the 0x80 bit is not set) then "\i"'s
|
||||
// validity depends on the relevant quirk. The element's value is an enum:
|
||||
// - 1: "\a", U+0007, quirk_allow_backslash_a.
|
||||
// - 2: "\e", U+001B, quirk_allow_backslash_e.
|
||||
// - 3: "\?", U+003F, quirk_allow_backslash_question_mark.
|
||||
// - 4: "\'", U+0027, quirk_allow_backslash_single_quote.
|
||||
// - 5: "\v", U+000B, quirk_allow_backslash_v.
|
||||
// - 6: "\0", U+0000, quirk_allow_backslash_zero.
|
||||
// The U+1234 values are held in lut_quirky_backslashes, below.
|
||||
//
|
||||
// If the element is zero then "\i" is invalid, or it is a special case, the
|
||||
// start of "\x12", "\u1234" or "\U12345678".
|
||||
pri const lut_backslashes array[256] base.u8 = [
|
||||
// 0 1 2 3 4 5 6 7
|
||||
// 8 9 A B C D E F
|
||||
@@ -63,18 +81,18 @@ pri const lut_backslashes array[256] base.u8 = [
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
|
||||
0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. '"'.
|
||||
0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x04, // 0x20 ..= 0x27. '"', '\''.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAF, // 0x28 ..= 0x2F. '/'.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.
|
||||
0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. '0'.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // 0x38 ..= 0x3F. '?'
|
||||
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
|
||||
0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. '\\'.
|
||||
0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8C, 0x00, // 0x60 ..= 0x67. 'b', 'f'.
|
||||
0x00, 0x01, 0x88, 0x00, 0x00, 0x02, 0x8C, 0x00, // 0x60 ..= 0x67. 'a', 'b', 'e', 'f'.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x00, // 0x68 ..= 0x6F. 'n'.
|
||||
0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. 'r', 't'.
|
||||
0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x05, 0x00, // 0x70 ..= 0x77. 'r', 't', 'v'.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
|
||||
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87.
|
||||
@@ -98,6 +116,13 @@ pri const lut_backslashes array[256] base.u8 = [
|
||||
// 8 9 A B C D E F
|
||||
]
|
||||
|
||||
// lut_quirky_backslashes is discussed in the lut_backslashes comment. The
|
||||
// first and last elements (indexes 0 and 7) are not used, but 8 is a round
|
||||
// power of 2, so enforcing index-in-bounds is a simple "&7" operation.
|
||||
pri const lut_quirky_backslashes array[8] base.u8 = [
|
||||
0x00, 0x07, 0x1B, 0x3F, 0x27, 0x0B, 0x00, 0x00,
|
||||
]
|
||||
|
||||
// lut_chars helps decode bytes within a string:
|
||||
// - 0x00 is 1-byte UTF-8 (ASCII) but not '"', '\\' or a C0 control code.
|
||||
// - 0x01 is '"'.
|
||||
|
||||
+22
-13
@@ -13,14 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub struct decoder? implements base.token_decoder(
|
||||
quirk_enabled_allow_backslash_a : base.bool,
|
||||
// quirk_enabled_allow_backslash_etc, an 8-element array, is indexed by the
|
||||
// same enum as lut_quirky_backslashes.
|
||||
quirk_enabled_allow_backslash_etc : array[8] base.bool,
|
||||
|
||||
quirk_enabled_allow_backslash_capital_u : base.bool,
|
||||
quirk_enabled_allow_backslash_e : base.bool,
|
||||
quirk_enabled_allow_backslash_question_mark : base.bool,
|
||||
quirk_enabled_allow_backslash_single_quote : base.bool,
|
||||
quirk_enabled_allow_backslash_v : base.bool,
|
||||
quirk_enabled_allow_backslash_x : base.bool,
|
||||
quirk_enabled_allow_backslash_zero : base.bool,
|
||||
quirk_enabled_allow_comment_block : base.bool,
|
||||
quirk_enabled_allow_comment_line : base.bool,
|
||||
quirk_enabled_allow_final_comma : base.bool,
|
||||
@@ -68,21 +66,21 @@ pub struct decoder? implements base.token_decoder(
|
||||
|
||||
pub func decoder.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) {
|
||||
if args.quirk == quirk_allow_backslash_a {
|
||||
this.quirk_enabled_allow_backslash_a = args.enabled
|
||||
this.quirk_enabled_allow_backslash_etc[1] = args.enabled
|
||||
} else if args.quirk == quirk_allow_backslash_capital_u {
|
||||
this.quirk_enabled_allow_backslash_capital_u = args.enabled
|
||||
} else if args.quirk == quirk_allow_backslash_e {
|
||||
this.quirk_enabled_allow_backslash_e = args.enabled
|
||||
this.quirk_enabled_allow_backslash_etc[2] = args.enabled
|
||||
} else if args.quirk == quirk_allow_backslash_question_mark {
|
||||
this.quirk_enabled_allow_backslash_question_mark = args.enabled
|
||||
this.quirk_enabled_allow_backslash_etc[3] = args.enabled
|
||||
} else if args.quirk == quirk_allow_backslash_single_quote {
|
||||
this.quirk_enabled_allow_backslash_single_quote = args.enabled
|
||||
this.quirk_enabled_allow_backslash_etc[4] = args.enabled
|
||||
} else if args.quirk == quirk_allow_backslash_v {
|
||||
this.quirk_enabled_allow_backslash_v = args.enabled
|
||||
this.quirk_enabled_allow_backslash_etc[5] = args.enabled
|
||||
} else if args.quirk == quirk_allow_backslash_x {
|
||||
this.quirk_enabled_allow_backslash_x = args.enabled
|
||||
} else if args.quirk == quirk_allow_backslash_zero {
|
||||
this.quirk_enabled_allow_backslash_zero = args.enabled
|
||||
this.quirk_enabled_allow_backslash_etc[6] = args.enabled
|
||||
} else if args.quirk == quirk_allow_comment_block {
|
||||
this.quirk_enabled_allow_comment_block = args.enabled
|
||||
} else if args.quirk == quirk_allow_comment_line {
|
||||
@@ -335,7 +333,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
}
|
||||
c = (args.src.peek_u16le() >> 8) as base.u8
|
||||
backslash = lut_backslashes[c]
|
||||
if backslash > 0 {
|
||||
if (backslash & 0x80) <> 0 {
|
||||
args.src.skip32_fast!(actual: 2, worst_case: 2)
|
||||
args.dst.write_fast_token!(
|
||||
value_major: 0,
|
||||
@@ -344,6 +342,17 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
length: 2)
|
||||
continue.string_loop_outer
|
||||
|
||||
} else if backslash <> 0 {
|
||||
if this.quirk_enabled_allow_backslash_etc[backslash & 7] {
|
||||
args.src.skip32_fast!(actual: 2, worst_case: 2)
|
||||
args.dst.write_fast_token!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0000 | (lut_quirky_backslashes[backslash & 7] as base.u32),
|
||||
link: 0x3,
|
||||
length: 2)
|
||||
continue.string_loop_outer
|
||||
}
|
||||
|
||||
} else if c == 0x75 { // 0x75 is 'u'.
|
||||
// -------- BEGIN backslash-u.
|
||||
if args.src.available() < 6 {
|
||||
|
||||
@@ -1105,6 +1105,96 @@ test_wuffs_json_decode_prior_valid_utf_8() {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_json_decode_quirk_allow_backslash_etc() {
|
||||
CHECK_FOCUS(__func__);
|
||||
|
||||
struct {
|
||||
uint32_t want;
|
||||
const char* str;
|
||||
uint32_t quirk;
|
||||
} test_cases[] = {
|
||||
{
|
||||
.want = 0x07,
|
||||
.str = "\"\\a\"",
|
||||
.quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_A,
|
||||
},
|
||||
{
|
||||
.want = 0x1B,
|
||||
.str = "\"\\e\"",
|
||||
.quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_E,
|
||||
},
|
||||
{
|
||||
.want = 0x3F,
|
||||
.str = "\"\\?\"",
|
||||
.quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_QUESTION_MARK,
|
||||
},
|
||||
{
|
||||
.want = 0x27,
|
||||
.str = "\"\\'\"",
|
||||
.quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE,
|
||||
},
|
||||
{
|
||||
.want = 0x0B,
|
||||
.str = "\"\\v\"",
|
||||
.quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_V,
|
||||
},
|
||||
{
|
||||
.want = 0x00,
|
||||
.str = "\"\\0\"",
|
||||
.quirk = WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO,
|
||||
},
|
||||
};
|
||||
|
||||
int tc;
|
||||
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
|
||||
int q;
|
||||
for (q = 0; q < 2; q++) {
|
||||
wuffs_json__decoder dec;
|
||||
CHECK_STATUS("initialize", wuffs_json__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
|
||||
wuffs_json__decoder__set_quirk_enabled(&dec, test_cases[tc].quirk, q);
|
||||
|
||||
wuffs_base__token_buffer tok =
|
||||
wuffs_base__make_token_buffer_writer(global_have_token_slice);
|
||||
wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader(
|
||||
wuffs_base__make_slice_u8((void*)(test_cases[tc].str),
|
||||
strlen(test_cases[tc].str)),
|
||||
true);
|
||||
|
||||
const char* have_status_repr =
|
||||
wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr;
|
||||
const char* want_status_repr =
|
||||
q ? NULL : wuffs_json__error__bad_backslash_escape;
|
||||
if (have_status_repr != want_status_repr) {
|
||||
RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc,
|
||||
q, have_status_repr, want_status_repr);
|
||||
}
|
||||
if (want_status_repr != NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
uint32_t have = 0;
|
||||
while (tok.meta.ri < tok.meta.wi) {
|
||||
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
|
||||
uint64_t vbc = wuffs_base__token__value_base_category(t);
|
||||
uint64_t vbd = wuffs_base__token__value_base_detail(t);
|
||||
if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) {
|
||||
have = vbd;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (have != test_cases[tc].want) {
|
||||
RETURN_FAIL("tc=%d, q=%d: Unicode code point: have U+%04" PRIX32
|
||||
", want U+%04" PRIX32,
|
||||
tc, q, have, test_cases[tc].want);
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_json_decode_quirk_allow_leading_etc() {
|
||||
CHECK_FOCUS(__func__);
|
||||
@@ -1746,6 +1836,7 @@ proc tests[] = {
|
||||
test_wuffs_json_decode_interface,
|
||||
test_wuffs_json_decode_long_numbers,
|
||||
test_wuffs_json_decode_prior_valid_utf_8,
|
||||
test_wuffs_json_decode_quirk_allow_backslash_etc,
|
||||
test_wuffs_json_decode_quirk_allow_leading_etc,
|
||||
test_wuffs_json_decode_quirk_allow_trailing_etc,
|
||||
test_wuffs_json_decode_quirk_replace_invalid_utf_8,
|
||||
|
||||
Reference in new issue
Block a user