Have std/json decode string escapes

This commit is contained in:
Nigel Tao committed 2020-02-21 14:13:37 +11:00
1 parent 93b09674a9
commit b5461bdd5e
3 files changed
+426 -43

No files matched your search

+171 -11
View File
@@ -20,6 +20,82 @@ pub status "#unsupported recursion depth"
pri status "#internal error: inconsistent I/O"
pri const backslashes array[256] base.u8 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. '"'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAF, // 0x28 ..= 0x2F. '/'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. '\\'.
0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8C, 0x00, // 0x60 ..= 0x67. 'b', 'f'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x00, // 0x68 ..= 0x6F. 'n'.
0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. 'r', 't'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x88 ..= 0x8F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x90 ..= 0x97.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x98 ..= 0x9F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA0 ..= 0xA7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA8 ..= 0xAF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB0 ..= 0xB7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB8 ..= 0xBF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC0 ..= 0xC7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC8 ..= 0xCF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD0 ..= 0xD7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD8 ..= 0xDF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE0 ..= 0xE7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF.
]
pri const hexadecimal_digits array[256] base.u8 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F.
0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, // 0x30 ..= 0x37. '0'-'7'.
0x88, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. '8'-'9'.
0x00, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x00, // 0x40 ..= 0x47. 'A'-'F'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F.
0x00, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x00, // 0x60 ..= 0x67. 'a'-'f'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x88 ..= 0x8F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x90 ..= 0x97.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x98 ..= 0x9F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA0 ..= 0xA7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA8 ..= 0xAF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB0 ..= 0xB7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB8 ..= 0xBF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC0 ..= 0xC7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC8 ..= 0xCF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD0 ..= 0xD7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD8 ..= 0xDF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE0 ..= 0xE7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF.
]
pub struct decoder?(
)(
// stack is conceptually an array of bits, implemented as an array of u32.
@@ -63,6 +139,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var stack_bit : base.u32[..= 31]
var match : base.u32[..= 2]
var c : base.u8
var backslash : base.u8
var uni4_ok : base.u8
var uni4_string : base.u32
var uni4_value : base.u32[..= 0xFFFF]
// expect is a bitmask of what the next token can be:
// - 0x01 EXPECT_NON_STRING_VALUE.
@@ -138,7 +218,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.outer
}
whitespace_length += 1
}
} // while.ws
// Emit whitespace.
if whitespace_length > 0 {
@@ -171,6 +251,27 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
// If the string contents starts with an escape code, issue
// a zero-length string token (one with value_base_category
// 3) before issuing a token with value_base_category 4.
while true {
if args.src.available() <= 0 {
yield? base."$short read"
continue
}
if args.src.peek_u8() <> 0x5C { // 0x5C is '\\'.
break
}
if args.dst.available() <= 0 {
yield? base."$short write"
continue
}
args.dst.write_fast_token!(
value: 0x60_0001,
length: 0)
break
}
while.string_loop true {
if args.dst.available() <= 0 {
yield? base."$short write"
@@ -196,7 +297,6 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.string_loop
}
c = args.src.peek_u8()
args.src.skip32_fast!(actual: 1, worst_case: 1)
if c <= 0x22 { // 0x22 is '"'.
if c == 0x22 { // 0x22 is '"'.
@@ -217,11 +317,65 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
} else if c == 0x5C { // 0x5C is '\\'.
// TODO: escapes.
return "#bad input"
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x60_0001,
length: string_length as base.u64)
string_length = 0
if args.dst.available() <= 0 {
continue.string_loop
}
}
assert args.dst.available() > 0
if args.src.available() < 2 {
yield? base."$short read"
string_length = 0
continue.string_loop
}
backslash = (args.src.peek_u16le() >> 8) as base.u8
c = backslashes[backslash]
if c > 0 {
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value: 0x80_0000 | ((c & 0x7F) as base.u64),
length: 2)
continue.string_loop
} else if backslash == 0x75 { // 0x75 is 'u'.
if args.src.available() < 6 {
yield? base."$short read"
string_length = 0
continue.string_loop
}
uni4_string = (args.src.peek_u48le_as_u64() >> 16) as base.u32
uni4_value = 0
uni4_ok = 0x80
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 12
c = hexadecimal_digits[0xFF & (uni4_string >> 8)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 8
c = hexadecimal_digits[0xFF & (uni4_string >> 16)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 4
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 0
if uni4_ok <> 0 {
args.src.skip32_fast!(actual: 6, worst_case: 6)
args.dst.write_fast_token!(
value: 0x80_0000 | (uni4_value as base.u64),
length: 6)
continue.string_loop
}
}
return "#bad input"
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
args.dst.write_fast_token!(
value: 0x60_0001,
@@ -231,17 +385,23 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
string_length += 1
}
}
} // while.string_loop
// The trailing '"' is filler.
while true {
if args.dst.available() > 0 {
args.dst.write_fast_token!(
value: 0,
length: 1)
break
if args.src.available() <= 0 {
yield? base."$short read"
continue
}
yield? base."$short write"
if args.dst.available() <= 0 {
yield? base."$short write"
continue
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
args.dst.write_fast_token!(
value: 0,
length: 1)
break
}
// As above, expect must have contained