Add std/json backslash-related quirks

This commit is contained in:
Nigel Tao committed 2020-03-19 15:56:21 +11:00
1 parent b86373aff5
commit 97f7f28218
4 files changed
+175 -36

No files matched your search

+30 -5
View File
@@ -56,6 +56,24 @@ pri const decoder_number_length_max_incl base.u64 = 99
// Look-Up Tables (LUTs).
// lut_backslashes[i] helps decode "\i", for various 'i's.
//
// If the element's 0x80 bit is set then "\i" is unconditionally a valid
// single-output-byte backslash-escape. The low 7 bits are the unescaped value.
// For example, lut_backslashes['n'] is (0x80 | 0x0A), because "\n" is U+000A.
//
// If the element is non-zero (but the 0x80 bit is not set) then "\i"'s
// validity depends on the relevant quirk. The element's value is an enum:
// - 1: "\a", U+0007, quirk_allow_backslash_a.
// - 2: "\e", U+001B, quirk_allow_backslash_e.
// - 3: "\?", U+003F, quirk_allow_backslash_question_mark.
// - 4: "\'", U+0027, quirk_allow_backslash_single_quote.
// - 5: "\v", U+000B, quirk_allow_backslash_v.
// - 6: "\0", U+0000, quirk_allow_backslash_zero.
// The U+1234 values are held in lut_quirky_backslashes, below.
//
// If the element is zero then "\i" is invalid, or it is a special case, the
// start of "\x12", "\u1234" or "\U12345678".
pri const lut_backslashes array[256] base.u8 = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
@@ -63,18 +81,18 @@ pri const lut_backslashes array[256] base.u8 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. '"'.
0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x04, // 0x20 ..= 0x27. '"', '\''.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAF, // 0x28 ..= 0x2F. '/'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.
0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. '0'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // 0x38 ..= 0x3F. '?'
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. '\\'.
0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8C, 0x00, // 0x60 ..= 0x67. 'b', 'f'.
0x00, 0x01, 0x88, 0x00, 0x00, 0x02, 0x8C, 0x00, // 0x60 ..= 0x67. 'a', 'b', 'e', 'f'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x00, // 0x68 ..= 0x6F. 'n'.
0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. 'r', 't'.
0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x05, 0x00, // 0x70 ..= 0x77. 'r', 't', 'v'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87.
@@ -98,6 +116,13 @@ pri const lut_backslashes array[256] base.u8 = [
// 8 9 A B C D E F
]
// lut_quirky_backslashes is discussed in the lut_backslashes comment. The
// first and last elements (indexes 0 and 7) are not used, but 8 is a round
// power of 2, so enforcing index-in-bounds is a simple "&7" operation.
pri const lut_quirky_backslashes array[8] base.u8 = [
0x00, 0x07, 0x1B, 0x3F, 0x27, 0x0B, 0x00, 0x00,
]
// lut_chars helps decode bytes within a string:
// - 0x00 is 1-byte UTF-8 (ASCII) but not '"', '\\' or a C0 control code.
// - 0x01 is '"'.
+22 -13
View File
@@ -13,14 +13,12 @@
// limitations under the License.
pub struct decoder? implements base.token_decoder(
quirk_enabled_allow_backslash_a : base.bool,
// quirk_enabled_allow_backslash_etc, an 8-element array, is indexed by the
// same enum as lut_quirky_backslashes.
quirk_enabled_allow_backslash_etc : array[8] base.bool,
quirk_enabled_allow_backslash_capital_u : base.bool,
quirk_enabled_allow_backslash_e : base.bool,
quirk_enabled_allow_backslash_question_mark : base.bool,
quirk_enabled_allow_backslash_single_quote : base.bool,
quirk_enabled_allow_backslash_v : base.bool,
quirk_enabled_allow_backslash_x : base.bool,
quirk_enabled_allow_backslash_zero : base.bool,
quirk_enabled_allow_comment_block : base.bool,
quirk_enabled_allow_comment_line : base.bool,
quirk_enabled_allow_final_comma : base.bool,
@@ -68,21 +66,21 @@ pub struct decoder? implements base.token_decoder(
pub func decoder.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) {
if args.quirk == quirk_allow_backslash_a {
this.quirk_enabled_allow_backslash_a = args.enabled
this.quirk_enabled_allow_backslash_etc[1] = args.enabled
} else if args.quirk == quirk_allow_backslash_capital_u {
this.quirk_enabled_allow_backslash_capital_u = args.enabled
} else if args.quirk == quirk_allow_backslash_e {
this.quirk_enabled_allow_backslash_e = args.enabled
this.quirk_enabled_allow_backslash_etc[2] = args.enabled
} else if args.quirk == quirk_allow_backslash_question_mark {
this.quirk_enabled_allow_backslash_question_mark = args.enabled
this.quirk_enabled_allow_backslash_etc[3] = args.enabled
} else if args.quirk == quirk_allow_backslash_single_quote {
this.quirk_enabled_allow_backslash_single_quote = args.enabled
this.quirk_enabled_allow_backslash_etc[4] = args.enabled
} else if args.quirk == quirk_allow_backslash_v {
this.quirk_enabled_allow_backslash_v = args.enabled
this.quirk_enabled_allow_backslash_etc[5] = args.enabled
} else if args.quirk == quirk_allow_backslash_x {
this.quirk_enabled_allow_backslash_x = args.enabled
} else if args.quirk == quirk_allow_backslash_zero {
this.quirk_enabled_allow_backslash_zero = args.enabled
this.quirk_enabled_allow_backslash_etc[6] = args.enabled
} else if args.quirk == quirk_allow_comment_block {
this.quirk_enabled_allow_comment_block = args.enabled
} else if args.quirk == quirk_allow_comment_line {
@@ -335,7 +333,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
c = (args.src.peek_u16le() >> 8) as base.u8
backslash = lut_backslashes[c]
if backslash > 0 {
if (backslash & 0x80) <> 0 {
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value_major: 0,
@@ -344,6 +342,17 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
length: 2)
continue.string_loop_outer
} else if backslash <> 0 {
if this.quirk_enabled_allow_backslash_etc[backslash & 7] {
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value_major: 0,
value_minor: 0x60_0000 | (lut_quirky_backslashes[backslash & 7] as base.u32),
link: 0x3,
length: 2)
continue.string_loop_outer
}
} else if c == 0x75 { // 0x75 is 'u'.
// -------- BEGIN backslash-u.
if args.src.available() < 6 {