Add std/json character classes

This commit is contained in:
Nigel Tao committed 2020-02-22 08:07:26 +11:00
1 parent 9263c80cfb
commit 48e7004636
3 files changed
+116 -40

No files matched your search

+35 -21
View File
@@ -18458,6 +18458,26 @@ static const uint8_t //
0, 0, 0, 0, 0, 0, 0, 0, 0,
};
static const uint8_t //
wuffs_json__classes[256] //
WUFFS_BASE__POTENTIALLY_UNUSED = {
15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 15, 15, 0, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 15, 1, 15,
15, 15, 15, 15, 15, 15, 15, 15, 2, 4, 15, 15, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 3, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 7, 15, 8, 15, 15, 15, 15, 15, 15, 15, 15, 9, 15, 15, 15, 15, 15,
15, 15, 11, 15, 15, 15, 15, 15, 10, 15, 15, 15, 15, 15, 15, 5, 15, 6,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15,
};
static const uint8_t //
wuffs_json__hexadecimal_digits[256] //
WUFFS_BASE__POTENTIALLY_UNUSED = {
@@ -18585,6 +18605,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
uint32_t v_stack_bit = 0;
uint32_t v_match = 0;
uint8_t v_c = 0;
uint8_t v_class = 0;
uint8_t v_backslash = 0;
uint8_t v_uni4_ok = 0;
uint32_t v_uni4_string = 0;
@@ -18637,6 +18658,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
v_whitespace_length = 0;
v_c = 0;
v_class = 0;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (v_whitespace_length > 0) {
@@ -18657,17 +18679,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto label_0_continue;
}
v_c = wuffs_base__load_u8be(iop_a_src);
if (v_c > 32) {
v_class = wuffs_json__classes[v_c];
if (v_class != 0) {
goto label_1_break;
} else if ((v_c != 32) && (v_c != 10) && (v_c != 9) && (v_c != 13)) {
if (v_whitespace_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_whitespace_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
}
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_whitespace_length >= 65534) {
@@ -18691,7 +18705,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
}
if (v_c <= 57) {
if (v_c == 34) {
if (v_class == 1) {
if (0 == (v_expect & 2)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -18888,7 +18902,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto label_0_continue;
}
goto label_5_break;
} else if (v_c == 44) {
} else if (v_class == 2) {
if (0 == (v_expect & 4)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -18903,7 +18917,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
v_expect = 3;
}
goto label_0_continue;
} else if ((v_c >= 48) || (v_c == 45)) {
} else if (v_class == 4) {
if (0 == (v_expect & 1)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -18961,7 +18975,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
label_6_break:;
goto label_5_break;
}
} else if (v_c == 58) {
} else if (v_class == 3) {
if (0 == (v_expect & 8)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -18972,7 +18986,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_expect = 3;
goto label_0_continue;
} else if (v_c == 91) {
} else if (v_class == 7) {
if (0 == (v_expect & 1)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -18994,7 +19008,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
v_expect = 19;
v_expect_after_value = 20;
goto label_0_continue;
} else if (v_c == 93) {
} else if (v_class == 8) {
if (0 == (v_expect & 16)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -19024,7 +19038,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
v_expect_after_value = 36;
}
goto label_0_continue;
} else if (v_c == 123) {
} else if (v_class == 5) {
if (0 == (v_expect & 1)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -19046,7 +19060,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
v_expect = 34;
v_expect_after_value = 36;
goto label_0_continue;
} else if (v_c == 125) {
} else if (v_class == 6) {
if (0 == (v_expect & 32)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
@@ -19082,7 +19096,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto exit;
}
v_match = 2;
if (v_c == 110) {
if (v_class == 11) {
v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src,
465676103172);
if (v_match == 0) {
@@ -19097,7 +19111,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(iop_a_src += 4, wuffs_base__make_empty_struct());
goto label_5_break;
}
} else if (v_c == 102) {
} else if (v_class == 9) {
v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src,
111546413966853);
if (v_match == 0) {
@@ -19112,7 +19126,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(iop_a_src += 5, wuffs_base__make_empty_struct());
goto label_5_break;
}
} else if (v_c == 116) {
} else if (v_class == 10) {
v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src,
435762131972);
if (v_match == 0) {
+66
View File
@@ -19,6 +19,8 @@ pub status "#unsupported recursion depth"
pri status "#internal error: inconsistent I/O"
pri const backslashes array[256] base.u8 = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
@@ -54,9 +56,71 @@ pri const backslashes array[256] base.u8 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF.
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
]
// - 0x00 is CLASS_WHITESPACE.
// - 0x01 is CLASS_STRING.
// - 0x02 is CLASS_COMMA.
// - 0x03 is CLASS_COLON.
// - 0x04 is CLASS_NUMBER.
// - 0x05 is CLASS_OPEN_CURLY_BRACE.
// - 0x06 is CLASS_CLOSE_CURLY_BRACE.
// - 0x07 is CLASS_OPEN_SQUARE_BRACKET.
// - 0x08 is CLASS_CLOSE_SQUARE_BRACKET.
// - 0x09 is CLASS_FALSE.
// - 0x0A is CLASS_TRUE.
// - 0x0B is CLASS_NULL.
// - 0x0C is reserved.
// - 0x0D is reserved.
// - 0x0E is reserved.
// - 0x0F is CLASS_BAD_INPUT.
pri const classes array[256] base.u8[..= 0x0F] = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x00 ..= 0x07.
0x0F, 0x00, 0x00, 0x0F, 0x0F, 0x00, 0x0F, 0x0F, // 0x08 ..= 0x0F. '\t', '\n', '\r'.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x10 ..= 0x17.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x18 ..= 0x1F.
0x00, 0x0F, 0x01, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x20 ..= 0x27. ' ', '"'.
0x0F, 0x0F, 0x0F, 0x0F, 0x02, 0x04, 0x0F, 0x0F, // 0x28 ..= 0x2F. ',', '-'.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0x30 ..= 0x37. '0'-'7'.
0x04, 0x04, 0x03, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x38 ..= 0x3F. '8'-'9', ':'.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x40 ..= 0x47.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x48 ..= 0x4F.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x50 ..= 0x57.
0x0F, 0x0F, 0x0F, 0x07, 0x0F, 0x08, 0x0F, 0x0F, // 0x58 ..= 0x5F. '[', ']'.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x09, 0x0F, // 0x60 ..= 0x67. 'f'.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0B, 0x0F, // 0x68 ..= 0x6F. 'n'.
0x0F, 0x0F, 0x0F, 0x0F, 0x0A, 0x0F, 0x0F, 0x0F, // 0x70 ..= 0x77. 't'.
0x0F, 0x0F, 0x0F, 0x05, 0x0F, 0x06, 0x0F, 0x0F, // 0x78 ..= 0x7F. '{', '}'.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x80 ..= 0x87.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x88 ..= 0x8F.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x90 ..= 0x97.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0x98 ..= 0x9F.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xA0 ..= 0xA7.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xA8 ..= 0xAF.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xB0 ..= 0xB7.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xB8 ..= 0xBF.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xC0 ..= 0xC7.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xC8 ..= 0xCF.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xD0 ..= 0xD7.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xD8 ..= 0xDF.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xE0 ..= 0xE7.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xE8 ..= 0xEF.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xF0 ..= 0xF7.
0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, // 0xF8 ..= 0xFF.
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
]
pri const hexadecimal_digits array[256] base.u8 = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
@@ -92,4 +156,6 @@ pri const hexadecimal_digits array[256] base.u8 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF.
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
]
+15 -19
View File
@@ -57,6 +57,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var stack_bit : base.u32[..= 31]
var match : base.u32[..= 2]
var c : base.u8
var class : base.u8[..= 0x0F]
var backslash : base.u8
var uni4_ok : base.u8
var uni4_string : base.u32
@@ -96,6 +97,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// Consume whitespace.
whitespace_length = 0
c = 0
class = 0
while.ws true,
inv args.dst.available() > 0,
post args.src.available() > 0,
@@ -116,15 +118,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
c = args.src.peek_u8()
if c > 0x20 { // 0x20 is ' '.
class = classes[c]
if class <> 0x00 { // 0x00 is CLASS_WHITESPACE.
break.ws
} else if (c <> 0x20) and (c <> 0x0A) and (c <> 0x09) and (c <> 0x0D) {
if whitespace_length > 0 {
args.dst.write_fast_token!(
value: 0,
length: whitespace_length as base.u64)
}
return "#bad input"
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
@@ -157,7 +153,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
assert args.src.available() > 0
if c <= 0x39 { // 0x39 is '9'.
if c == 0x22 { // 0x22 is '"'.
if class == 0x01 { // 0x01 is CLASS_STRING.
// -------- BEGIN parse strings.
if 0 == (expect & 0x02) { // 0x02 is EXPECT_STRING_VALUE.
return "#bad input"
@@ -333,7 +329,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
break.goto_parsed_a_leaf_value
// -------- END parse strings.
} else if c == 0x2C { // 0x2C is ','.
} else if class == 0x02 { // 0x02 is CLASS_COMMA.
if 0 == (expect & 0x04) { // 0x01 is EXPECT_COMMA.
return "#bad input"
}
@@ -351,7 +347,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
continue.outer
} else if (c >= 0x30) or (c == 0x2D) { // 0x30 is '0', 0x2D is '-'.
} else if class == 0x04 { // 0x04 is CLASS_NUMBER.
// -------- BEGIN parse numbers.
if 0 == (expect & 0x01) { // 0x02 is EXPECT_NON_STRING_VALUE.
return "#bad input"
@@ -399,7 +395,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// -------- END parse numbers.
}
} else if c == 0x3A { // 0x3A is ':'.
} else if class == 0x03 { // 0x03 is CLASS_COLON.
if 0 == (expect & 0x08) { // 0x08 is EXPECT_COLON.
return "#bad input"
}
@@ -411,7 +407,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
expect = 0x03 // 0x03 is EXPECT_VALUE.
continue.outer
} else if c == 0x5B { // 0x5B is '['.
} else if class == 0x07 { // 0x07 is CLASS_OPEN_SQUARE_BRACKET.
if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE.
return "#bad input"
}
@@ -431,7 +427,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
expect_after_value = 0x14 // 0x14 is (EXPECT_CLOSE_BRACKET | EXPECT_COMMA).
continue.outer
} else if c == 0x5D { // 0x5D is ']'.
} else if class == 0x08 { // 0x08 is CLASS_CLOSE_SQUARE_BRACKET.
if 0 == (expect & 0x10) { // 0x10 is EXPECT_CLOSE_BRACKET
return "#bad input"
}
@@ -462,7 +458,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
continue.outer
} else if c == 0x7B { // 0x7B is '{'.
} else if class == 0x05 { // 0x05 is CLASS_OPEN_CURLY_BRACE.
if 0 == (expect & 0x01) { // 0x01 is EXPECT_NON_STRING_VALUE.
return "#bad input"
}
@@ -482,7 +478,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
expect_after_value = 0x24 // 0x24 is (EXPECT_CLOSE_BRACE | EXPECT_COMMA).
continue.outer
} else if c == 0x7D { // 0x7D is '}'.
} else if class == 0x06 { // 0x06 is CLASS_CLOSE_CURLY_BRACE.
if 0 == (expect & 0x20) { // 0x20 is EXPECT_CLOSE_BRACE.
return "#bad input"
}
@@ -518,7 +514,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
return "#bad input"
}
match = 2
if c == 0x6E { // 0x6E is 'n'.
if class == 0x0B { // 0x0B is CLASS_NULL.
match = args.src.match7(a: 0x6C_6C75_6E04) // 4 bytes "null".
if match == 0 {
args.dst.write_fast_token!(
@@ -531,7 +527,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
break.goto_parsed_a_leaf_value
}
} else if c == 0x66 { // 0x66 is 'f'.
} else if class == 0x09 { // 0x09 is CLASS_FALSE.
match = args.src.match7(a: 0x6573_6C61_6605) // 5 bytes "false".
if match == 0 {
args.dst.write_fast_token!(
@@ -544,7 +540,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
break.goto_parsed_a_leaf_value
}
} else if c == 0x74 { // 0x74 is 't'.
} else if class == 0x0A { // 0x0A is CLASS_TRUE.
match = args.src.match7(a: 0x65_7572_7404) // 4 bytes "true".
if match == 0 {
args.dst.write_fast_token!(