Add std/json quirk_allow_ascii_control_codes

This commit is contained in:
Nigel Tao committed 2020-04-07 17:50:47 +10:00
1 parent 6f67bbc4b2
commit d83bd8d036
5 files changed
+70 -19

No files matched your search

+23 -8
View File
@@ -6048,6 +6048,8 @@ extern const char* wuffs_json__error__unsupported_recursion_depth;
#define WUFFS_JSON__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL 100
#define WUFFS_JSON__QUIRK_ALLOW_ASCII_CONTROL_CODES 1225364480
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_A 1225364481
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_CAPITAL_U 1225364482
@@ -6151,6 +6153,7 @@ struct wuffs_json__decoder__struct {
wuffs_base__vtable null_vtable;
bool f_quirk_enabled_allow_backslash_etc[8];
bool f_quirk_enabled_allow_ascii_control_codes;
bool f_quirk_enabled_allow_backslash_capital_u;
bool f_quirk_enabled_allow_backslash_x;
bool f_quirk_enabled_allow_comment_block;
@@ -20842,9 +20845,9 @@ static const uint8_t //
static const uint8_t //
wuffs_json__lut_chars[256] //
WUFFS_BASE__POTENTIALLY_UNUSED = {
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141,
142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155,
156, 157, 158, 159, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -20855,12 +20858,12 @@ static const uint8_t //
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 129, 129, 3, 3,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 32, 32, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 5, 5, 5, 5, 5, 129, 129, 129, 129, 129, 129, 129,
129, 129, 129, 129,
4, 4, 5, 5, 5, 5, 5, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32,
};
static const uint8_t //
@@ -21041,7 +21044,9 @@ wuffs_json__decoder__set_quirk_enabled(wuffs_json__decoder* self,
return wuffs_base__make_empty_struct();
}
if (a_quirk == 1225364481) {
if (a_quirk == 1225364480) {
self->private_impl.f_quirk_enabled_allow_ascii_control_codes = a_enabled;
} else if (a_quirk == 1225364481) {
self->private_impl.f_quirk_enabled_allow_backslash_etc[1] = a_enabled;
} else if (a_quirk == 1225364482) {
self->private_impl.f_quirk_enabled_allow_backslash_capital_u = a_enabled;
@@ -21862,7 +21867,17 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto label__string_loop_outer__continue;
}
}
if (v_char == 128) {
if ((v_char & 128) != 0) {
if (self->private_impl
.f_quirk_enabled_allow_ascii_control_codes) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((6291456 | ((uint32_t)((v_char & 127))))))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
goto label__string_loop_outer__continue;
}
status = wuffs_base__make_status(
wuffs_json__error__bad_c0_control_code);
goto exit;
+9 -9
View File
@@ -134,8 +134,8 @@ pri const lut_quirky_backslashes array[8] base.u8 = [
// - 0x04 is the start of 3-byte UTF-8.
// - 0x05 is the start of 4-byte UTF-8.
// - 0x10 is a UTF-8 tail byte.
// - 0x80 is invalid JSON (C0 control codes).
// - 0x81 is invalid UTF-8.
// - 0x20 is invalid UTF-8.
// - 0x80 and above is invalid JSON (C0 control codes).
//
// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:
// UTF8-1 = %x00-7F
@@ -148,10 +148,10 @@ pri const lut_quirky_backslashes array[8] base.u8 = [
pri const lut_chars array[256] base.u8 = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 ..= 0x07. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 ..= 0x0F. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 ..= 0x17. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 ..= 0x1F. C0 control codes.
0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, // 0x00 ..= 0x07. C0 control codes.
0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, // 0x08 ..= 0x0F. C0 control codes.
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, // 0x10 ..= 0x17. C0 control codes.
0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, // 0x18 ..= 0x1F. C0 control codes.
0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. UTF-8-1; '"'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. UTF-8-1.
@@ -175,14 +175,14 @@ pri const lut_chars array[256] base.u8 = [
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB0 ..= 0xB7. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB8 ..= 0xBF. UTF-8 tail.
0x81, 0x81, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2.
0x20, 0x20, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC8 ..= 0xCF. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD0 ..= 0xD7. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD8 ..= 0xDF. UTF-8-2.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE0 ..= 0xE7. UTF-8-3.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE8 ..= 0xEF. UTF-8-3.
0x05, 0x05, 0x05, 0x05, 0x05, 0x81, 0x81, 0x81, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8.
0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, // 0xF8 ..= 0xFF. Invalid UTF-8.
0x05, 0x05, 0x05, 0x05, 0x05, 0x20, 0x20, 0x20, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8.
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, // 0xF8 ..= 0xFF. Invalid UTF-8.
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
]
+14 -2
View File
@@ -17,6 +17,7 @@ pub struct decoder? implements base.token_decoder(
// same enum as lut_quirky_backslashes.
quirk_enabled_allow_backslash_etc : array[8] base.bool,
quirk_enabled_allow_ascii_control_codes : base.bool,
quirk_enabled_allow_backslash_capital_u : base.bool,
quirk_enabled_allow_backslash_x : base.bool,
quirk_enabled_allow_comment_block : base.bool,
@@ -67,7 +68,9 @@ pub struct decoder? implements base.token_decoder(
)
pub func decoder.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) {
if args.quirk == quirk_allow_backslash_a {
if args.quirk == quirk_allow_ascii_control_codes {
this.quirk_enabled_allow_ascii_control_codes = args.enabled
} else if args.quirk == quirk_allow_backslash_a {
this.quirk_enabled_allow_backslash_etc[1] = args.enabled
} else if args.quirk == quirk_allow_backslash_capital_u {
this.quirk_enabled_allow_backslash_capital_u = args.enabled
@@ -788,7 +791,16 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
continue.string_loop_outer
}
}
if char == 0x80 {
if (char & 0x80) <> 0 {
if this.quirk_enabled_allow_ascii_control_codes {
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | ((char & 0x7F) as base.u32),
link: 0x3,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_outer
}
return "#bad C0 control code"
}
if this.quirk_enabled_replace_invalid_unicode {
+14
View File
@@ -21,6 +21,20 @@
// --------
// When this quirk is enabled, literal (not escaped) ASCII control codes, such
// as backspaces, new lines and NUL bytes, are accepted in JSON strings.
//
// Regardless of whether this quirk is enabled, escaped ASCII control codes,
// such as "\b" or "\u0008", are always accepted.
//
// This allows for multi-line strings, where a literal new line in the encoding
// becomes a literal new line in the decoding. A backslash before a new line is
// still a syntax error, unless combined with quirk_allow_backslash_new_line.
//
// Any indentation following a new line is not stripped, but remains part of
// the decoded string.
pub const quirk_allow_ascii_control_codes base.u32 = 0x4909_9400 | 0x00
// When this quirk is enabled, e.g. "abc\az" is accepted as a JSON string,
// equivalent to "abc\u0007z", containing an ASCII Bell control character.
pub const quirk_allow_backslash_a base.u32 = 0x4909_9400 | 0x01
+10
View File
@@ -1413,6 +1413,11 @@ test_wuffs_json_decode_quirk_allow_backslash_etc() {
const char* str;
uint32_t quirk;
} test_cases[] = {
{
.want = 0x09,
.str = "\"\t\"",
.quirk = WUFFS_JSON__QUIRK_ALLOW_ASCII_CONTROL_CODES,
},
{
.want = 0x07,
.str = "\"\\a\"",
@@ -1475,6 +1480,11 @@ test_wuffs_json_decode_quirk_allow_backslash_etc() {
.repr;
const char* want_status_repr =
q ? NULL : wuffs_json__error__bad_backslash_escape;
if ((test_cases[tc].quirk ==
WUFFS_JSON__QUIRK_ALLOW_ASCII_CONTROL_CODES) &&
want_status_repr) {
want_status_repr = wuffs_json__error__bad_c0_control_code;
}
if (have_status_repr != want_status_repr) {
RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc,
q, have_status_repr, want_status_repr);