Add std/json quirk_allow_ascii_control_codes

This commit is contained in:
Nigel Tao committed 2020-04-07 17:50:47 +10:00
1 parent 6f67bbc4b2
commit d83bd8d036
5 files changed
+70 -19

No files matched your search

+9 -9
View File
@@ -134,8 +134,8 @@ pri const lut_quirky_backslashes array[8] base.u8 = [
// - 0x04 is the start of 3-byte UTF-8.
// - 0x05 is the start of 4-byte UTF-8.
// - 0x10 is a UTF-8 tail byte.
// - 0x80 is invalid JSON (C0 control codes).
// - 0x81 is invalid UTF-8.
// - 0x20 is invalid UTF-8.
// - 0x80 and above is invalid JSON (C0 control codes).
//
// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:
// UTF8-1 = %x00-7F
@@ -148,10 +148,10 @@ pri const lut_quirky_backslashes array[8] base.u8 = [
pri const lut_chars array[256] base.u8 = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 ..= 0x07. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 ..= 0x0F. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 ..= 0x17. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 ..= 0x1F. C0 control codes.
0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, // 0x00 ..= 0x07. C0 control codes.
0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, // 0x08 ..= 0x0F. C0 control codes.
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, // 0x10 ..= 0x17. C0 control codes.
0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, // 0x18 ..= 0x1F. C0 control codes.
0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. UTF-8-1; '"'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. UTF-8-1.
@@ -175,14 +175,14 @@ pri const lut_chars array[256] base.u8 = [
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB0 ..= 0xB7. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB8 ..= 0xBF. UTF-8 tail.
0x81, 0x81, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2.
0x20, 0x20, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC8 ..= 0xCF. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD0 ..= 0xD7. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD8 ..= 0xDF. UTF-8-2.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE0 ..= 0xE7. UTF-8-3.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE8 ..= 0xEF. UTF-8-3.
0x05, 0x05, 0x05, 0x05, 0x05, 0x81, 0x81, 0x81, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8.
0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, // 0xF8 ..= 0xFF. Invalid UTF-8.
0x05, 0x05, 0x05, 0x05, 0x05, 0x20, 0x20, 0x20, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8.
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, // 0xF8 ..= 0xFF. Invalid UTF-8.
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
]
+14 -2
View File
@@ -17,6 +17,7 @@ pub struct decoder? implements base.token_decoder(
// same enum as lut_quirky_backslashes.
quirk_enabled_allow_backslash_etc : array[8] base.bool,
quirk_enabled_allow_ascii_control_codes : base.bool,
quirk_enabled_allow_backslash_capital_u : base.bool,
quirk_enabled_allow_backslash_x : base.bool,
quirk_enabled_allow_comment_block : base.bool,
@@ -67,7 +68,9 @@ pub struct decoder? implements base.token_decoder(
)
pub func decoder.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) {
if args.quirk == quirk_allow_backslash_a {
if args.quirk == quirk_allow_ascii_control_codes {
this.quirk_enabled_allow_ascii_control_codes = args.enabled
} else if args.quirk == quirk_allow_backslash_a {
this.quirk_enabled_allow_backslash_etc[1] = args.enabled
} else if args.quirk == quirk_allow_backslash_capital_u {
this.quirk_enabled_allow_backslash_capital_u = args.enabled
@@ -788,7 +791,16 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
continue.string_loop_outer
}
}
if char == 0x80 {
if (char & 0x80) <> 0 {
if this.quirk_enabled_allow_ascii_control_codes {
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | ((char & 0x7F) as base.u32),
link: 0x3,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_outer
}
return "#bad C0 control code"
}
if this.quirk_enabled_replace_invalid_unicode {
+14
View File
@@ -21,6 +21,20 @@
// --------
// When this quirk is enabled, literal (not escaped) ASCII control codes, such
// as backspaces, new lines and NUL bytes, are accepted in JSON strings.
//
// Regardless of whether this quirk is enabled, escaped ASCII control codes,
// such as "\b" or "\u0008", are always accepted.
//
// This allows for multi-line strings, where a literal new line in the encoding
// becomes a literal new line in the decoding. A backslash before a new line is
// still a syntax error, unless combined with quirk_allow_backslash_new_line.
//
// Any indentation following a new line is not stripped, but remains part of
// the decoded string.
pub const quirk_allow_ascii_control_codes base.u32 = 0x4909_9400 | 0x00
// When this quirk is enabled, e.g. "abc\az" is accepted as a JSON string,
// equivalent to "abc\u0007z", containing an ASCII Bell control character.
pub const quirk_allow_backslash_a base.u32 = 0x4909_9400 | 0x01